]> git.saurik.com Git - apple/xnu.git/blob - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blob
? search:


58d4836c4998691b8f4e5498148dfc6be73e7a65
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
1 /*
2 * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29
30 /*
31
32 Sleep:
33
34 - PMRootDomain calls IOHibernateSystemSleep() before system sleep
35 (devices awake, normal execution context)
36 - IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
37 grabs its extents and searches for a polling driver willing to work with that IOMedia.
38 The BSD code makes an ioctl to the storage driver to get the partition base offset to
39 the disk, and other ioctls to get the transfer constraints
40 If successful, the file is written to make sure its initially not bootable (in case of
41 later failure) and nvram set to point to the first block of the file. (Has to be done
42 here so blocking is possible in nvram support).
43 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
44 page out any pages it wants to (currently zero, but probably some percentage of memory).
45 Its assumed just allocating pages will cause the VM system to naturally select the best
46 pages for eviction. It also copies processor flags needed for the restore path and sets
47 a flag in the boot processor proc info.
48 gIOHibernateState = kIOHibernateStateHibernating.
49 - Regular sleep progresses - some drivers may inspect the root domain property
50 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
51 as usual but leaves motherboard I/O on.
52 - Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
53 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
54 all ppc RC bits out of the hash table and caches into the mapping structures.
55 - hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
56 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
57 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
58 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
59 The image header and block list are written. The header includes the second file extent so
60 only the header block is needed to read the file, regardless of filesystem.
61 The kernel section "__HIB" is written uncompressed to the image. This section of code and data
62 (only) is used to decompress the image during wake/boot.
63 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
64 The bitmaps are written to the image.
65 More areas are removed from the bitmaps (after they have been written to the image) - the
66 section "__HIB" pages and interrupt stack.
67 Each wired page is compressed and written and then each non-wired page. Compression and
68 disk writes are in parallel.
69 The image header is written to the start of the file and the polling driver closed.
70 The machine powers down (or sleeps).
71
72 Boot/Wake:
73
74 - BootX sees the boot-image nvram variable containing the device and block number of the image,
75 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
76 - BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
77 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
78 that is in the kernel's __HIB section.
79 - BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
80 hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
81 only code & data in that section is safe to call since all the other wired pages are still
82 compressed in the image.
83 - hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
84 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
85 location directly, and copies those that can't to interim free pages. When the image has been
86 completed, the copies are uncompressed, overwriting the wired image pages.
87 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
88 is used to get pages into place for 64bit.
89 - the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
90 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
91 are removed from the software strutures, and the hash table is reinitialized.
92 - After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
93 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
94 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
95 for the remaining non wired pages.
96 - The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
97 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
98 is closed via bsd.
99
100 Polled Mode I/O:
101
102 IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
103 registry, specifying an object of calls IOPolledInterface.
104
105 Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
106 partition) that the image is going to live, looking for polled interface properties. If it finds
107 one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
108 interfaces found are kept in an ordered list.
109
110 There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
111 few different contexts things happen in:
112
113 - there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
114 up and running) and after wake - this is safe to allocate memory and do anything. The device
115 ignores sleep requests from that point since its a waste of time if it goes to sleep and
116 immediately wakes back up for the image write.
117
118 - there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
119 immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
120 from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
121 used to flush and set the disk to sleep.
122
123 - there is an Open/Close (AfterSleep) pair made around the image read operations that happen
124 immediately after sleep. These can't block or allocate memory. This is happening after the platform
125 expert has woken the low level bits of the system, but most of the I/O system has not. There is only
126 one thread running.
127
128 For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
129 (prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
130 the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
131 Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
132 that is called for the hardware to check for events, and complete the I/O via the callback.
133 The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
134 to restrict I/O ops.
135 */
136
137 #include <sys/systm.h>
138
139 #include <IOKit/IOWorkLoop.h>
140 #include <IOKit/IOCommandGate.h>
141 #include <IOKit/IOTimerEventSource.h>
142 #include <IOKit/IOPlatformExpert.h>
143 #include <IOKit/IOKitDebug.h>
144 #include <IOKit/IOTimeStamp.h>
145 #include <IOKit/pwr_mgt/RootDomain.h>
146 #include <IOKit/pwr_mgt/IOPMPrivate.h>
147 #include <IOKit/IOMessage.h>
148 #include <IOKit/IODeviceTreeSupport.h>
149 #include <IOKit/IOBSD.h>
150 #include "RootDomainUserClient.h"
151 #include <IOKit/pwr_mgt/IOPowerConnection.h>
152 #include "IOPMPowerStateQueue.h"
153 #include <IOKit/IOBufferMemoryDescriptor.h>
154 #include <crypto/aes.h>
155
156 #include <sys/uio.h>
157 #include <sys/conf.h>
158 #include <sys/stat.h>
159 #include <sys/fcntl.h> // (FWRITE, ...)
160 #include <sys/sysctl.h>
161
162 #include <IOKit/IOHibernatePrivate.h>
163 #include <IOKit/IOPolledInterface.h>
164 #include <IOKit/IONVRAM.h>
165 #include "IOHibernateInternal.h"
166 #include "WKdm.h"
167 #include "IOKitKernelInternal.h"
168
169 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
170
171 OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
172
173 OSMetaClassDefineReservedUnused(IOPolledInterface, 0);
174 OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
175 OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
176 OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
177 OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
178 OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
179 OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
180 OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
181 OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
182 OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
183 OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
184 OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
185 OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
186 OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
187 OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
188 OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
189
190 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
191
192 extern uint32_t gIOHibernateState;
193 uint32_t gIOHibernateMode;
194 static char gIOHibernateBootSignature[256+1];
195 static char gIOHibernateFilename[MAXPATHLEN+1];
196 static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
197 uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
198
199 static IODTNVRAM * gIOOptionsEntry;
200 static IORegistryEntry * gIOChosenEntry;
201 #ifdef __i386__
202 static const OSSymbol * gIOCreateEFIDevicePathSymbol;
203 #endif
204
205 static IOPolledFileIOVars gFileVars;
206 static IOHibernateVars gIOHibernateVars;
207 static struct kern_direct_file_io_ref_t * gIOHibernateFileRef;
208 static hibernate_cryptvars_t gIOHibernateCryptWakeContext;
209
210 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
211
212 enum { kXPRamAudioVolume = 8 };
213 enum { kDefaultIOSize = 128 * 1024 };
214 enum { kVideoMapSize = 32 * 1024 * 1024 };
215
216 #ifndef kIOMediaPreferredBlockSizeKey
217 #define kIOMediaPreferredBlockSizeKey "Preferred Block Size"
218 #endif
219
220 #ifndef kIOBootPathKey
221 #define kIOBootPathKey "bootpath"
222 #endif
223 #ifndef kIOSelectedBootDeviceKey
224 #define kIOSelectedBootDeviceKey "boot-device"
225 #endif
226
227
228 enum { kIOHibernateMinPollersNeeded = 2 };
229
230 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
231
232 // copy from phys addr to MD
233
234 static IOReturn
235 IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
236 IOByteCount offset, addr64_t bytes, IOByteCount length)
237 {
238 addr64_t srcAddr = bytes;
239 IOByteCount remaining;
240
241 remaining = length = min(length, md->getLength() - offset);
242 while (remaining) { // (process another target segment?)
243 addr64_t dstAddr64;
244 IOByteCount dstLen;
245
246 dstAddr64 = md->getPhysicalSegment64(offset, &dstLen);
247 if (!dstAddr64)
248 break;
249
250 // Clip segment length to remaining
251 if (dstLen > remaining)
252 dstLen = remaining;
253
254 #if 1
255 bcopy_phys(srcAddr, dstAddr64, dstLen);
256 #else
257 copypv(srcAddr, dstAddr64, dstLen,
258 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
259 #endif
260 srcAddr += dstLen;
261 offset += dstLen;
262 remaining -= dstLen;
263 }
264
265 assert(!remaining);
266
267 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
268 }
269
270 // copy from MD to phys addr
271
272 static IOReturn
273 IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
274 IOByteCount offset, addr64_t bytes, IOByteCount length)
275 {
276 addr64_t dstAddr = bytes;
277 IOByteCount remaining;
278
279 remaining = length = min(length, md->getLength() - offset);
280 while (remaining) { // (process another target segment?)
281 addr64_t srcAddr64;
282 IOByteCount dstLen;
283
284 srcAddr64 = md->getPhysicalSegment64(offset, &dstLen);
285 if (!srcAddr64)
286 break;
287
288 // Clip segment length to remaining
289 if (dstLen > remaining)
290 dstLen = remaining;
291
292 #if 1
293 bcopy_phys(srcAddr64, dstAddr, dstLen);
294 #else
295 copypv(srcAddr, dstAddr64, dstLen,
296 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
297 #endif
298 dstAddr += dstLen;
299 offset += dstLen;
300 remaining -= dstLen;
301 }
302
303 assert(!remaining);
304
305 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
306 }
307
308 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
309
310 void
311 hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
312 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
313 {
314 count += ppnum;
315 switch (kind)
316 {
317 case kIOHibernatePageStateUnwiredSave:
318 // unwired save
319 for (; ppnum < count; ppnum++)
320 {
321 hibernate_page_bitset(page_list, FALSE, ppnum);
322 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
323 }
324 break;
325 case kIOHibernatePageStateWiredSave:
326 // wired save
327 for (; ppnum < count; ppnum++)
328 {
329 hibernate_page_bitset(page_list, FALSE, ppnum);
330 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
331 }
332 break;
333 case kIOHibernatePageStateFree:
334 // free page
335 for (; ppnum < count; ppnum++)
336 {
337 hibernate_page_bitset(page_list, TRUE, ppnum);
338 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
339 }
340 break;
341 default:
342 panic("hibernate_set_page_state");
343 }
344 }
345
346 static vm_offset_t
347 hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
348 {
349 uint32_t page = *pPage;
350 uint32_t count;
351 hibernate_bitmap_t * bitmap;
352
353 while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
354 {
355 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
356 if (!count)
357 break;
358 page += count;
359 if (page <= bitmap->last_page)
360 break;
361 }
362
363 *pPage = page;
364 if (bitmap)
365 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
366 else
367 count = 0;
368
369 return (count);
370 }
371
372 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
373
374 static IOReturn
375 IOHibernatePollerProbe(IOPolledFileIOVars * vars, IOService * target)
376 {
377 IOReturn err = kIOReturnError;
378 int32_t idx;
379 IOPolledInterface * poller;
380
381 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
382 {
383 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
384 err = poller->probe(target);
385 if (err)
386 {
387 HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
388 break;
389 }
390 }
391
392 return (err);
393 }
394
395 static IOReturn
396 IOHibernatePollerOpen(IOPolledFileIOVars * vars, uint32_t state, IOMemoryDescriptor * md)
397 {
398 IOReturn err = kIOReturnError;
399 int32_t idx;
400 IOPolledInterface * poller;
401
402 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
403 {
404 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
405 err = poller->open(state, md);
406 if (err)
407 {
408 HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
409 break;
410 }
411 }
412
413 return (err);
414 }
415
416 static IOReturn
417 IOHibernatePollerClose(IOPolledFileIOVars * vars, uint32_t state)
418 {
419 IOReturn err = kIOReturnError;
420 int32_t idx;
421 IOPolledInterface * poller;
422
423 for (idx = 0;
424 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
425 idx++)
426 {
427 err = poller->close(state);
428 if (err)
429 HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
430 }
431
432 return (err);
433 }
434
435 static void
436 IOHibernatePollerIOComplete(void * target,
437 void * parameter,
438 IOReturn status,
439 UInt64 actualByteCount)
440 {
441 IOPolledFileIOVars * vars = (IOPolledFileIOVars *) parameter;
442
443 vars->ioStatus = status;
444 }
445
446 static IOReturn
447 IOHibernatePollerIO(IOPolledFileIOVars * vars,
448 uint32_t operation, uint32_t bufferOffset,
449 uint64_t deviceOffset, uint64_t length)
450 {
451
452 IOReturn err = kIOReturnError;
453 IOPolledInterface * poller;
454 IOPolledCompletion completion;
455
456 completion.target = 0;
457 completion.action = &IOHibernatePollerIOComplete;
458 completion.parameter = vars;
459
460 vars->ioStatus = -1;
461
462 poller = (IOPolledInterface *) vars->pollers->getObject(0);
463 err = poller->startIO(operation, bufferOffset, deviceOffset + vars->block0, length, completion);
464 if (err)
465 HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
466
467 return (err);
468 }
469
470 static IOReturn
471 IOHibernatePollerIODone(IOPolledFileIOVars * vars, bool abortable)
472 {
473 IOReturn err = kIOReturnSuccess;
474 int32_t idx = 0;
475 IOPolledInterface * poller;
476
477 while (-1 == vars->ioStatus)
478 {
479 for (idx = 0;
480 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
481 idx++)
482 {
483 IOReturn newErr;
484 newErr = poller->checkForWork();
485 if ((newErr == kIOReturnAborted) && !abortable)
486 newErr = kIOReturnSuccess;
487 if (kIOReturnSuccess == err)
488 err = newErr;
489 }
490 }
491
492 if (err)
493 {
494 HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
495 }
496 else
497 {
498 err = vars->ioStatus;
499 if (kIOReturnSuccess != err)
500 HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
501 }
502
503 return (err);
504 }
505
506 IOReturn
507 IOPolledInterface::checkAllForWork(void)
508 {
509 IOReturn err = kIOReturnNotReady;
510 int32_t idx;
511 IOPolledInterface * poller;
512
513 IOHibernateVars * vars = &gIOHibernateVars;
514
515 if (!vars->fileVars || !vars->fileVars->pollers)
516 return (err);
517
518 for (idx = 0;
519 (poller = (IOPolledInterface *) vars->fileVars->pollers->getObject(idx));
520 idx++)
521 {
522 err = poller->checkForWork();
523 if (err)
524 HIBLOG("IOPolledInterface::checkAllForWork[%d] 0x%x\n", idx, err);
525 }
526
527 return (err);
528 }
529
530 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
531
532 struct _OpenFileContext
533 {
534 OSData * extents;
535 uint64_t size;
536 };
537
538 static void
539 file_extent_callback(void * ref, uint64_t start, uint64_t length)
540 {
541 _OpenFileContext * ctx = (_OpenFileContext *) ref;
542 IOPolledFileExtent extent;
543
544 extent.start = start;
545 extent.length = length;
546
547 ctx->extents->appendBytes(&extent, sizeof(extent));
548 ctx->size += length;
549 }
550
551 IOReturn
552 IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
553 IOPolledFileIOVars ** fileVars, OSData ** fileExtents,
554 OSData ** imagePath)
555 {
556 IOReturn err = kIOReturnError;
557 IOPolledFileIOVars * vars;
558 _OpenFileContext ctx;
559 OSData * extentsData;
560 OSNumber * num;
561 IORegistryEntry * part = 0;
562 OSDictionary * matching;
563 OSIterator * iter;
564 dev_t hibernate_image_dev;
565 uint64_t maxiobytes;
566
567 vars = &gFileVars;
568 do
569 {
570 HIBLOG("sizeof(IOHibernateImageHeader) == %ld\n", sizeof(IOHibernateImageHeader));
571 if (sizeof(IOHibernateImageHeader) != 512)
572 continue;
573
574 vars->io = false;
575 vars->buffer = (uint8_t *) ioBuffer->getBytesNoCopy();
576 vars->bufferHalf = 0;
577 vars->bufferOffset = 0;
578 vars->bufferSize = ioBuffer->getLength() >> 1;
579
580 extentsData = OSData::withCapacity(32);
581
582 ctx.extents = extentsData;
583 ctx.size = 0;
584 vars->fileRef = kern_open_file_for_direct_io(filename,
585 &file_extent_callback, &ctx,
586 &hibernate_image_dev,
587 &vars->block0,
588 &maxiobytes);
589 if (!vars->fileRef)
590 {
591 err = kIOReturnNoSpace;
592 break;
593 }
594 HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx\n", filename, ctx.size,
595 vars->block0, maxiobytes);
596 if (ctx.size < 1*1024*1024) // check against image size estimate!
597 {
598 err = kIOReturnNoSpace;
599 break;
600 }
601
602 if (maxiobytes < vars->bufferSize)
603 vars->bufferSize = maxiobytes;
604
605 vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
606
607 matching = IOService::serviceMatching("IOMedia");
608 num = OSNumber::withNumber(major(hibernate_image_dev), 32);
609 matching->setObject(kIOBSDMajorKey, num);
610 num->release();
611 num = OSNumber::withNumber(minor(hibernate_image_dev), 32);
612 matching->setObject(kIOBSDMinorKey, num);
613 num->release();
614 iter = IOService::getMatchingServices(matching);
615 matching->release();
616 if (iter)
617 {
618 part = (IORegistryEntry *) iter->getNextObject();
619 part->retain();
620 iter->release();
621 }
622 if (!part)
623 break;
624
625 int minor, major;
626 IORegistryEntry * next;
627 IORegistryEntry * child;
628 OSData * data;
629
630 num = (OSNumber *) part->getProperty(kIOBSDMajorKey);
631 if (!num)
632 break;
633 major = num->unsigned32BitValue();
634 num = (OSNumber *) part->getProperty(kIOBSDMinorKey);
635 if (!num)
636 break;
637 minor = num->unsigned32BitValue();
638
639 hibernate_image_dev = makedev(major, minor);
640
641 vars->pollers = OSArray::withCapacity(4);
642 if (!vars->pollers)
643 break;
644
645 vars->blockSize = 512;
646 next = part;
647 do
648 {
649 IOPolledInterface * poller;
650 OSObject * obj;
651
652 obj = next->getProperty(kIOPolledInterfaceSupportKey);
653 if (kOSBooleanFalse == obj)
654 {
655 vars->pollers->flushCollection();
656 break;
657 }
658 else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
659 vars->pollers->setObject(poller);
660 if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey))))
661 vars->blockSize = num->unsigned32BitValue();
662 child = next;
663 }
664 while ((next = child->getParentEntry(gIOServicePlane))
665 && child->isParent(next, gIOServicePlane, true));
666
667 HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
668 major, minor, vars->blockSize, vars->pollers->getCount());
669 if (vars->pollers->getCount() < kIOHibernateMinPollersNeeded)
670 continue;
671
672 err = IOHibernatePollerProbe(vars, (IOService *) part);
673 if (kIOReturnSuccess != err)
674 break;
675
676 err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer);
677 if (kIOReturnSuccess != err)
678 break;
679
680 *fileVars = vars;
681 *fileExtents = extentsData;
682
683 // make imagePath
684
685 if ((extentsData->getLength() >= sizeof(IOPolledFileExtent)))
686 {
687 char str2[24];
688
689 #if __i386__
690 if (!gIOCreateEFIDevicePathSymbol)
691 gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
692
693 snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
694
695 err = IOService::getPlatform()->callPlatformFunction(
696 gIOCreateEFIDevicePathSymbol, false,
697 (void *) part, (void *) str2, (void *) true,
698 (void *) &data);
699 #else
700 char str1[256];
701 int len = sizeof(str1);
702
703 if (!part->getPath(str1, &len, gIODTPlane))
704 err = kIOReturnNotFound;
705 else
706 {
707 snprintf(str2, sizeof(str2), ",%qx", vars->extentMap[0].start);
708 // (strip the plane name)
709 char * tail = strchr(str1, ':');
710 if (!tail)
711 tail = str1 - 1;
712 data = OSData::withBytes(tail + 1, strlen(tail + 1));
713 data->appendBytes(str2, strlen(str2));
714 }
715 #endif
716 if (kIOReturnSuccess == err)
717 *imagePath = data;
718 else
719 HIBLOG("error 0x%x getting path\n", err);
720 }
721 }
722 while (false);
723
724 if (kIOReturnSuccess != err)
725 {
726 HIBLOG("error 0x%x opening hibernation file\n", err);
727 if (vars->fileRef)
728 kern_close_file_for_direct_io(vars->fileRef);
729 }
730
731 if (part)
732 part->release();
733
734 return (err);
735 }
736
737 IOReturn
738 IOPolledFileClose( IOPolledFileIOVars * vars )
739 {
740 if (vars->pollers)
741 {
742 IOHibernatePollerClose(vars, kIOPolledPostflightState);
743 vars->pollers->release();
744 }
745
746 gIOHibernateFileRef = vars->fileRef;
747
748 bzero(vars, sizeof(IOPolledFileIOVars));
749
750 return (kIOReturnSuccess);
751 }
752
753 static IOReturn
754 IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
755 {
756 IOPolledFileExtent * extentMap;
757
758 extentMap = vars->extentMap;
759
760 vars->position = position;
761
762 while (position >= extentMap->length)
763 {
764 position -= extentMap->length;
765 extentMap++;
766 }
767
768 vars->currentExtent = extentMap;
769 vars->extentRemaining = extentMap->length - position;
770 vars->extentPosition = vars->position - position;
771
772 if (vars->bufferSize <= vars->extentRemaining)
773 vars->bufferLimit = vars->bufferSize;
774 else
775 vars->bufferLimit = vars->extentRemaining;
776
777 return (kIOReturnSuccess);
778 }
779
780 static IOReturn
781 IOPolledFileWrite(IOPolledFileIOVars * vars,
782 const uint8_t * bytes, IOByteCount size,
783 hibernate_cryptvars_t * cryptvars)
784 {
785 IOReturn err = kIOReturnSuccess;
786 IOByteCount copy;
787 bool flush = false;
788
789 do
790 {
791 if (!bytes && !size)
792 {
793 // seek to end of block & flush
794 size = vars->position & (vars->blockSize - 1);
795 if (size)
796 size = vars->blockSize - size;
797 flush = true;
798 // use some garbage for the fill
799 bytes = vars->buffer + vars->bufferOffset;
800 }
801
802 copy = vars->bufferLimit - vars->bufferOffset;
803 if (copy > size)
804 copy = size;
805 else
806 flush = true;
807
808 if (bytes)
809 {
810 bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
811 bytes += copy;
812 }
813 else
814 bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
815
816 size -= copy;
817 vars->bufferOffset += copy;
818 vars->position += copy;
819
820 if (flush && vars->bufferOffset)
821 {
822 uint64_t offset = (vars->position - vars->bufferOffset
823 - vars->extentPosition + vars->currentExtent->start);
824 uint32_t length = (vars->bufferOffset);
825
826 #if CRYPTO
827 if (cryptvars && vars->encryptStart && (vars->position > vars->encryptStart))
828 {
829 uint32_t encryptLen, encryptStart;
830 encryptLen = vars->position - vars->encryptStart;
831 if (encryptLen > length)
832 encryptLen = length;
833 encryptStart = length - encryptLen;
834
835 // encrypt the buffer
836 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
837 &cryptvars->aes_iv[0],
838 encryptLen / AES_BLOCK_SIZE,
839 vars->buffer + vars->bufferHalf + encryptStart,
840 &cryptvars->ctx.encrypt);
841 // save initial vector for following encrypts
842 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
843 &cryptvars->aes_iv[0],
844 AES_BLOCK_SIZE);
845 }
846 #endif /* CRYPTO */
847
848 if (vars->io)
849 {
850 err = IOHibernatePollerIODone(vars, true);
851 if (kIOReturnSuccess != err)
852 break;
853 }
854
855 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
856 //if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
857
858 err = IOHibernatePollerIO(vars, kIOPolledWrite, vars->bufferHalf, offset, length);
859 if (kIOReturnSuccess != err)
860 break;
861 vars->io = true;
862
863 vars->extentRemaining -= vars->bufferOffset;
864 if (!vars->extentRemaining)
865 {
866 vars->currentExtent++;
867 vars->extentRemaining = vars->currentExtent->length;
868 vars->extentPosition = vars->position;
869 if (!vars->extentRemaining)
870 {
871 err = kIOReturnOverrun;
872 break;
873 }
874 }
875
876 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
877 vars->bufferOffset = 0;
878 if (vars->bufferSize <= vars->extentRemaining)
879 vars->bufferLimit = vars->bufferSize;
880 else
881 vars->bufferLimit = vars->extentRemaining;
882
883 flush = false;
884 }
885 }
886 while (size);
887
888 return (err);
889 }
890
891 static IOReturn
892 IOPolledFileRead(IOPolledFileIOVars * vars,
893 uint8_t * bytes, IOByteCount size,
894 hibernate_cryptvars_t * cryptvars)
895 {
896 IOReturn err = kIOReturnSuccess;
897 IOByteCount copy;
898
899 // bytesWritten += size;
900
901 do
902 {
903 copy = vars->bufferLimit - vars->bufferOffset;
904 if (copy > size)
905 copy = size;
906
907 if (bytes)
908 {
909 bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
910 bytes += copy;
911 }
912 size -= copy;
913 vars->bufferOffset += copy;
914 // vars->position += copy;
915
916 if (vars->bufferOffset == vars->bufferLimit)
917 {
918 if (vars->io)
919 {
920 err = IOHibernatePollerIODone(vars, false);
921 if (kIOReturnSuccess != err)
922 break;
923 }
924 else
925 cryptvars = 0;
926
927 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
928
929 vars->position += vars->lastRead;
930 vars->extentRemaining -= vars->lastRead;
931 vars->bufferLimit = vars->lastRead;
932
933 if (!vars->extentRemaining)
934 {
935 vars->currentExtent++;
936 vars->extentRemaining = vars->currentExtent->length;
937 vars->extentPosition = vars->position;
938 if (!vars->extentRemaining)
939 {
940 err = kIOReturnOverrun;
941 break;
942 }
943 }
944
945 if (vars->extentRemaining <= vars->bufferSize)
946 vars->lastRead = vars->extentRemaining;
947 else
948 vars->lastRead = vars->bufferSize;
949
950 uint64_t offset = (vars->position
951 - vars->extentPosition + vars->currentExtent->start);
952 uint64_t length = (vars->lastRead);
953
954 //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
955
956 err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
957 if (kIOReturnSuccess != err)
958 break;
959 vars->io = true;
960
961 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
962 vars->bufferOffset = 0;
963
964 #if CRYPTO
965 if (cryptvars)
966 {
967 uint8_t thisVector[AES_BLOCK_SIZE];
968 // save initial vector for following decrypts
969 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
970 bcopy(vars->buffer + vars->bufferHalf + vars->lastRead - AES_BLOCK_SIZE,
971 &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
972 // decrypt the buffer
973 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
974 &thisVector[0],
975 vars->lastRead / AES_BLOCK_SIZE,
976 vars->buffer + vars->bufferHalf,
977 &cryptvars->ctx.decrypt);
978 }
979 #endif CRYPTO
980 }
981 }
982 while (size);
983
984 return (err);
985 }
986
987 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
988
989 IOReturn
990 IOHibernateSystemSleep(void)
991 {
992 IOReturn err;
993 OSData * data;
994 OSObject * obj;
995 OSString * str;
996 OSNumber * num;
997 OSDictionary *sleepOverrideOptions;
998
999 IOHibernateVars * vars = &gIOHibernateVars;
1000
1001 if (vars->fileVars && vars->fileVars->fileRef)
1002 // already on the way down
1003 return (kIOReturnSuccess);
1004
1005 gIOHibernateState = kIOHibernateStateInactive;
1006
1007 /* The invocation of IOPMSleepSystemWithOptions() may override
1008 * existing hibernation settings.
1009 */
1010 sleepOverrideOptions = (OSDictionary *)OSDynamicCast( OSDictionary,
1011 IOService::getPMRootDomain()->copyProperty(kRootDomainSleepOptionsKey));
1012
1013
1014 /* Hibernate mode overriden by sleep otions ? */
1015 obj = NULL;
1016
1017 if (sleepOverrideOptions) {
1018 obj = sleepOverrideOptions->getObject(kIOHibernateModeKey);
1019 if (obj) obj->retain();
1020 }
1021
1022 if(!obj) {
1023 obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateModeKey);
1024 }
1025
1026 if (obj && (num = OSDynamicCast(OSNumber, obj)) )
1027 {
1028 gIOHibernateMode = num->unsigned32BitValue();
1029 if (kIOHibernateModeSleep & gIOHibernateMode)
1030 // default to discard clean for safe sleep
1031 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
1032 | kIOHibernateModeDiscardCleanActive);
1033 }
1034
1035 if (obj) obj->release();
1036
1037 /* Hibernate free rotio overriden by sleep options ? */
1038 obj = NULL;
1039
1040 if (sleepOverrideOptions) {
1041 obj = sleepOverrideOptions->getObject(kIOHibernateFreeRatioKey);
1042 if (obj) obj->retain();
1043 }
1044
1045 if(!obj) {
1046 obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeRatioKey);
1047 }
1048 if (obj && (num = OSDynamicCast(OSNumber, obj)))
1049 {
1050 gIOHibernateFreeRatio = num->unsigned32BitValue();
1051 }
1052 if (obj) obj->release();
1053
1054 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeTimeKey)))
1055 {
1056 if ((num = OSDynamicCast(OSNumber, obj)))
1057 gIOHibernateFreeTime = num->unsigned32BitValue();
1058 obj->release();
1059 }
1060 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
1061 {
1062 if ((str = OSDynamicCast(OSString, obj)))
1063 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
1064 sizeof(gIOHibernateFilename));
1065 obj->release();
1066 }
1067
1068 if (sleepOverrideOptions)
1069 sleepOverrideOptions->release();
1070
1071 if (!gIOHibernateMode || !gIOHibernateFilename[0])
1072 return (kIOReturnUnsupported);
1073
1074 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
1075
1076
1077 do
1078 {
1079 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1080 4 * page_size, page_size);
1081 vars->ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1082 2 * kDefaultIOSize, page_size);
1083
1084 if (!vars->srcBuffer || !vars->ioBuffer)
1085 {
1086 err = kIOReturnNoMemory;
1087 break;
1088 }
1089
1090 err = IOPolledFileOpen(gIOHibernateFilename, vars->ioBuffer,
1091 &vars->fileVars, &vars->fileExtents, &data);
1092 if (KERN_SUCCESS != err)
1093 {
1094 HIBLOG("IOPolledFileOpen(%x)\n", err);
1095 break;
1096 }
1097 if (vars->fileVars->fileRef)
1098 {
1099 // invalidate the image file
1100 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1101 int err = kern_write_file(vars->fileVars->fileRef, 0,
1102 (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1103 if (KERN_SUCCESS != err)
1104 HIBLOG("kern_write_file(%d)\n", err);
1105 }
1106
1107 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1108
1109 boolean_t encryptedswap;
1110 err = hibernate_setup(gIOHibernateCurrentHeader,
1111 gIOHibernateFreeRatio, gIOHibernateFreeTime,
1112 &vars->page_list, &vars->page_list_wired, &encryptedswap);
1113 if (KERN_SUCCESS != err)
1114 {
1115 HIBLOG("hibernate_setup(%d)\n", err);
1116 break;
1117 }
1118
1119 if (encryptedswap)
1120 gIOHibernateMode ^= kIOHibernateModeEncrypt;
1121
1122 vars->videoAllocSize = kVideoMapSize;
1123 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
1124 vars->videoMapping = 0;
1125
1126 // generate crypt keys
1127 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
1128 vars->wiredCryptKey[i] = random();
1129 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
1130 vars->cryptKey[i] = random();
1131
1132 // set nvram
1133
1134 IORegistryEntry * regEntry;
1135 if (!gIOOptionsEntry)
1136 {
1137 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
1138 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
1139 if (regEntry && !gIOOptionsEntry)
1140 regEntry->release();
1141 }
1142 if (!gIOChosenEntry)
1143 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1144
1145 if (gIOOptionsEntry)
1146 {
1147 const OSSymbol * sym;
1148
1149 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1150 if (sym)
1151 {
1152 gIOOptionsEntry->setProperty(sym, data);
1153 sym->release();
1154 }
1155 data->release();
1156
1157 #ifdef __ppc__
1158 size_t len;
1159 char valueString[16];
1160
1161 vars->saveBootDevice = gIOOptionsEntry->copyProperty(kIOSelectedBootDeviceKey);
1162 if (gIOChosenEntry)
1163 {
1164 OSData * bootDevice = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOBootPathKey));
1165 if (bootDevice)
1166 {
1167 sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
1168 OSString * str2 = OSString::withCStringNoCopy((const char *) bootDevice->getBytesNoCopy());
1169 if (sym && str2)
1170 gIOOptionsEntry->setProperty(sym, str2);
1171 if (sym)
1172 sym->release();
1173 if (str2)
1174 str2->release();
1175 }
1176 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMemorySignatureKey));
1177 if (data)
1178 {
1179 vars->haveFastBoot = true;
1180
1181 len = sprintf(valueString, "0x%lx", *((UInt32 *)data->getBytesNoCopy()));
1182 data = OSData::withBytes(valueString, len + 1);
1183 sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
1184 if (sym && data)
1185 gIOOptionsEntry->setProperty(sym, data);
1186 if (sym)
1187 sym->release();
1188 if (data)
1189 data->release();
1190 }
1191 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
1192 if (data)
1193 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
1194 }
1195 #endif /* __ppc__ */
1196 #ifdef __i386__
1197 struct AppleRTCHibernateVars
1198 {
1199 uint8_t signature[4];
1200 uint32_t revision;
1201 uint8_t booterSignature[20];
1202 uint8_t wiredCryptKey[16];
1203 };
1204 AppleRTCHibernateVars rtcVars;
1205
1206 rtcVars.signature[0] = 'A';
1207 rtcVars.signature[1] = 'A';
1208 rtcVars.signature[2] = 'P';
1209 rtcVars.signature[3] = 'L';
1210 rtcVars.revision = 1;
1211 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
1212 if (gIOHibernateBootSignature[0])
1213 {
1214 char c;
1215 uint8_t value = 0;
1216 for (uint32_t i = 0;
1217 (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
1218 i++)
1219 {
1220 if (c >= 'a')
1221 c -= 'a' - 10;
1222 else if (c >= 'A')
1223 c -= 'A' - 10;
1224 else if (c >= '0')
1225 c -= '0';
1226 else
1227 continue;
1228 value = (value << 4) | c;
1229 if (i & 1)
1230 rtcVars.booterSignature[i >> 1] = value;
1231 }
1232 }
1233 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
1234 if (data)
1235 {
1236 IOService::getPMRootDomain()->setProperty(kIOHibernateRTCVariablesKey, data);
1237
1238 if( gIOOptionsEntry )
1239 {
1240 if( gIOHibernateMode & kIOHibernateModeSwitch )
1241 {
1242 const OSSymbol *sym;
1243 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSwitchVarsKey);
1244 if( sym )
1245 {
1246 gIOOptionsEntry->setProperty(sym, data); /* intentional insecure backup of rtc boot vars */
1247 sym->release();
1248 }
1249 }
1250 }
1251
1252 data->release();
1253 }
1254 if (gIOChosenEntry)
1255 {
1256 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
1257 if (data)
1258 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
1259 }
1260 #else /* !__i386__ */
1261 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1262 {
1263 data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1264 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
1265 if (sym && data)
1266 gIOOptionsEntry->setProperty(sym, data);
1267 if (sym)
1268 sym->release();
1269 if (data)
1270 data->release();
1271 if (false && gIOHibernateBootSignature[0])
1272 {
1273 data = OSData::withCapacity(16);
1274 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
1275 if (sym && data)
1276 {
1277 char c;
1278 uint8_t value = 0;
1279 for (uint32_t i = 0; (c = gIOHibernateBootSignature[i]); i++)
1280 {
1281 if (c >= 'a')
1282 c -= 'a' - 10;
1283 else if (c >= 'A')
1284 c -= 'A' - 10;
1285 else if (c >= '0')
1286 c -= '0';
1287 else
1288 continue;
1289 value = (value << 4) | c;
1290 if (i & 1)
1291 data->appendBytes(&value, sizeof(value));
1292 }
1293 gIOOptionsEntry->setProperty(sym, data);
1294 }
1295 if (sym)
1296 sym->release();
1297 if (data)
1298 data->release();
1299 }
1300 }
1301 if (!vars->haveFastBoot)
1302 {
1303 // set boot volume to zero
1304 IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
1305 if (platform && (kIOReturnSuccess == platform->readXPRAM(kXPRamAudioVolume,
1306 &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume))))
1307 {
1308 uint8_t newVolume;
1309 newVolume = vars->saveBootAudioVolume & 0xf8;
1310 platform->writeXPRAM(kXPRamAudioVolume,
1311 &newVolume, sizeof(newVolume));
1312 }
1313 }
1314 #endif /* !__i386__ */
1315 }
1316 // --
1317
1318 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
1319 gIOHibernateState = kIOHibernateStateHibernating;
1320 }
1321 while (false);
1322
1323 return (err);
1324 }
1325
1326 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1327
1328 DECLARE_IOHIBERNATEPROGRESSALPHA
1329
1330 static void
1331 ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
1332 {
1333 uint32_t rowBytes, pixelShift;
1334 uint32_t x, y;
1335 int32_t blob;
1336 uint32_t alpha, in, color, result;
1337 uint8_t * out;
1338 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1339
1340 rowBytes = display->rowBytes;
1341 pixelShift = display->depth >> 4;
1342 if (pixelShift < 1) return;
1343
1344 screen += ((display->width
1345 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1346 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1347
1348 for (y = 0; y < kIOHibernateProgressHeight; y++)
1349 {
1350 out = screen + y * rowBytes;
1351 for (blob = 0; blob < kIOHibernateProgressCount; blob++)
1352 {
1353 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
1354 for (x = 0; x < kIOHibernateProgressWidth; x++)
1355 {
1356 alpha = gIOHibernateProgressAlpha[y][x];
1357 result = color;
1358 if (alpha)
1359 {
1360 if (0xff != alpha)
1361 {
1362 if (1 == pixelShift)
1363 {
1364 in = *((uint16_t *)out) & 0x1f; // 16
1365 in = (in << 3) | (in >> 2);
1366 }
1367 else
1368 in = *((uint32_t *)out) & 0xff; // 32
1369 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
1370 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
1371 }
1372 if (1 == pixelShift)
1373 {
1374 result >>= 3;
1375 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1376 }
1377 else
1378 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1379 }
1380 out += (1 << pixelShift);
1381 }
1382 out += (kIOHibernateProgressSpacing << pixelShift);
1383 }
1384 }
1385 }
1386
1387
1388 static void
1389 ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
1390 {
1391 uint32_t rowBytes, pixelShift;
1392 uint32_t x, y;
1393 int32_t blob, lastBlob;
1394 uint32_t alpha, in, color, result;
1395 uint8_t * out;
1396 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1397
1398 pixelShift = display->depth >> 4;
1399 if (pixelShift < 1)
1400 return;
1401
1402 rowBytes = display->rowBytes;
1403
1404 screen += ((display->width
1405 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1406 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1407
1408 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
1409
1410 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
1411
1412 for (y = 0; y < kIOHibernateProgressHeight; y++)
1413 {
1414 out = screen + y * rowBytes;
1415 for (blob = firstBlob; blob <= lastBlob; blob++)
1416 {
1417 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
1418 for (x = 0; x < kIOHibernateProgressWidth; x++)
1419 {
1420 alpha = gIOHibernateProgressAlpha[y][x];
1421 result = color;
1422 if (alpha)
1423 {
1424 if (0xff != alpha)
1425 {
1426 in = display->progressSaveUnder[blob][saveindex[blob]++];
1427 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
1428 }
1429 if (1 == pixelShift)
1430 {
1431 result >>= 3;
1432 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1433 }
1434 else
1435 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1436 }
1437 out += (1 << pixelShift);
1438 }
1439 out += (kIOHibernateProgressSpacing << pixelShift);
1440 }
1441 }
1442 }
1443
1444 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1445
1446 IOReturn
1447 IOHibernateSystemHasSlept(void)
1448 {
1449 IOHibernateVars * vars = &gIOHibernateVars;
1450 OSObject * obj;
1451 OSData * data;
1452
1453 obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
1454 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1455 if (obj && !vars->previewBuffer)
1456 obj->release();
1457
1458 vars->consoleMapping = NULL;
1459 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
1460 {
1461 vars->previewBuffer->release();
1462 vars->previewBuffer = 0;
1463 }
1464
1465 if (vars->previewBuffer && (data = OSDynamicCast(OSData,
1466 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
1467 {
1468 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
1469 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", flags);
1470
1471 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1472
1473 if (kIOHibernatePreviewUpdates & flags)
1474 {
1475 PE_Video consoleInfo;
1476 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1477
1478 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1479
1480 graphicsInfo->width = consoleInfo.v_width;
1481 graphicsInfo->height = consoleInfo.v_height;
1482 graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
1483 graphicsInfo->depth = consoleInfo.v_depth;
1484 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1485
1486 HIBPRINT("video %p %d %d %d\n",
1487 vars->consoleMapping, gIOHibernateGraphicsInfo->depth,
1488 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height);
1489 if (vars->consoleMapping)
1490 ProgressInit(graphicsInfo, vars->consoleMapping,
1491 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1492 }
1493 }
1494
1495 if (gIOOptionsEntry)
1496 gIOOptionsEntry->sync();
1497
1498 return (kIOReturnSuccess);
1499 }
1500
1501 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1502
1503 IOReturn
1504 IOHibernateSystemWake(void)
1505 {
1506 IOHibernateVars * vars = &gIOHibernateVars;
1507
1508 hibernate_teardown(vars->page_list, vars->page_list_wired);
1509
1510 if (vars->videoMapping)
1511 {
1512 if (vars->videoMapSize)
1513 // remove mappings
1514 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1515 if (vars->videoAllocSize)
1516 // dealloc range
1517 kmem_free(kernel_map, trunc_page_32(vars->videoMapping), vars->videoAllocSize);
1518 }
1519
1520 if (vars->previewBuffer)
1521 {
1522 vars->previewBuffer->release();
1523 vars->previewBuffer = 0;
1524 }
1525
1526 if (vars->fileVars)
1527 {
1528 IOPolledFileClose(vars->fileVars);
1529 }
1530
1531 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1532
1533 #ifdef __ppc__
1534 OSData * data = OSData::withCapacity(4);
1535 if (gIOOptionsEntry && data)
1536 {
1537 const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1538 if (sym)
1539 {
1540 gIOOptionsEntry->setProperty(sym, data);
1541 sym->release();
1542 }
1543 sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
1544 if (sym)
1545 {
1546 if (vars->saveBootDevice)
1547 {
1548 gIOOptionsEntry->setProperty(sym, vars->saveBootDevice);
1549 vars->saveBootDevice->release();
1550 }
1551 sym->release();
1552 }
1553 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
1554 if (sym)
1555 {
1556 gIOOptionsEntry->setProperty(sym, data);
1557 sym->release();
1558 }
1559 sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
1560 if (sym)
1561 {
1562 gIOOptionsEntry->removeProperty(sym);
1563 sym->release();
1564 }
1565 }
1566 if (data)
1567 data->release();
1568
1569 if (gIOOptionsEntry)
1570 {
1571 if (!vars->haveFastBoot)
1572 {
1573 // reset boot audio volume
1574 IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
1575 if (platform)
1576 platform->writeXPRAM(kXPRamAudioVolume,
1577 &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume));
1578 }
1579
1580 // sync now to hardware if the booter has not
1581 if (kIOHibernateStateInactive == gIOHibernateState)
1582 gIOOptionsEntry->sync();
1583 else
1584 // just sync the variables in case a later panic syncs nvram (it won't sync variables)
1585 gIOOptionsEntry->syncOFVariables();
1586 }
1587 #endif
1588
1589 #ifdef __i386__
1590 IOService::getPMRootDomain()->removeProperty(kIOHibernateRTCVariablesKey);
1591
1592 /*
1593 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1594 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1595 */
1596 if (gIOOptionsEntry) {
1597 const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1598
1599 if (sym) {
1600 gIOOptionsEntry->removeProperty(sym);
1601 sym->release();
1602 }
1603 }
1604 #endif
1605
1606 if (vars->srcBuffer)
1607 vars->srcBuffer->release();
1608 if (vars->ioBuffer)
1609 vars->ioBuffer->release();
1610 if (vars->fileExtents)
1611 vars->fileExtents->release();
1612
1613 bzero(vars, sizeof(*vars));
1614
1615 // gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1616
1617 return (kIOReturnSuccess);
1618 }
1619
1620 IOReturn
1621 IOHibernateSystemPostWake(void)
1622 {
1623 if (gIOHibernateFileRef)
1624 {
1625 // invalidate the image file
1626 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1627 int err = kern_write_file(gIOHibernateFileRef, 0,
1628 (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1629 if (KERN_SUCCESS != err)
1630 HIBLOG("kern_write_file(%d)\n", err);
1631
1632 kern_close_file_for_direct_io(gIOHibernateFileRef);
1633 gIOHibernateFileRef = 0;
1634 }
1635 return (kIOReturnSuccess);
1636 }
1637
1638 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1639
1640 SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
1641 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
1642 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
1643 SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
1644 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
1645 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
1646 SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
1647 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
1648 &gIOHibernateMode, 0, "");
1649
1650 void
1651 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1652 {
1653 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
1654 if (data)
1655 {
1656 rootDomain->setProperty(kIOHibernateStateKey, data);
1657 data->release();
1658 }
1659
1660 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))
1661 gIOHibernateMode = kIOHibernateModeOn;
1662 else
1663 gIOHibernateFilename[0] = 0;
1664
1665 sysctl_register_oid(&sysctl__kern_hibernatefile);
1666 sysctl_register_oid(&sysctl__kern_bootsignature);
1667 sysctl_register_oid(&sysctl__kern_hibernatemode);
1668 }
1669
1670
1671 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1672
1673 static void
1674 hibernate_setup_for_wake(void)
1675 {
1676 #if __ppc__
1677 // go slow (state needed for wake)
1678 ml_set_processor_speed(1);
1679 #endif
1680 }
1681
1682 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1683
1684 #define C_ASSERT(e) typedef char __C_ASSERT__[(e) ? 1 : -1]
1685
1686 extern "C" uint32_t
1687 hibernate_write_image(void)
1688 {
1689 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
1690 IOHibernateVars * vars = &gIOHibernateVars;
1691 IOPolledFileExtent * fileExtents;
1692
1693 C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
1694
1695 uint32_t pageCount, pagesDone;
1696 IOReturn err;
1697 vm_offset_t ppnum;
1698 IOItemCount page, count;
1699 uint8_t * src;
1700 uint8_t * data;
1701 IOByteCount pageCompressedSize;
1702 uint64_t compressedSize, uncompressedSize;
1703 uint64_t image1Size = 0;
1704 uint32_t bitmap_size;
1705 bool iterDone, pollerOpen, needEncryptStart;
1706 uint32_t restore1Sum, sum, sum1, sum2;
1707 uint32_t tag;
1708 uint32_t pageType;
1709 uint32_t pageAndCount[2];
1710
1711 AbsoluteTime startTime, endTime;
1712 AbsoluteTime allTime, compTime, decoTime;
1713 uint64_t nsec;
1714 uint32_t lastProgressStamp = 0;
1715 uint32_t progressStamp;
1716 uint32_t blob, lastBlob = (uint32_t) -1L;
1717
1718 hibernate_cryptvars_t _cryptvars;
1719 hibernate_cryptvars_t * cryptvars = 0;
1720
1721 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
1722 return (false /* sleep */ );
1723
1724 restore1Sum = sum1 = sum2 = 0;
1725
1726 #if CRYPTO
1727 // encryption data. "iv" is the "initial vector".
1728 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1729 {
1730 static const unsigned char first_iv[AES_BLOCK_SIZE]
1731 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
1732 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
1733
1734 cryptvars = &gIOHibernateCryptWakeContext;
1735 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
1736 aes_encrypt_key(vars->cryptKey,
1737 kIOHibernateAESKeySize,
1738 &cryptvars->ctx.encrypt);
1739 aes_decrypt_key(vars->cryptKey,
1740 kIOHibernateAESKeySize,
1741 &cryptvars->ctx.decrypt);
1742
1743 cryptvars = &_cryptvars;
1744 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
1745 aes_encrypt_key(vars->wiredCryptKey,
1746 kIOHibernateAESKeySize,
1747 &cryptvars->ctx.encrypt);
1748
1749 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1750 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1751 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
1752 bzero(gIOHibernateCryptWakeVars, sizeof(hibernate_cryptwakevars_t));
1753 }
1754 #endif /* CRYPTO */
1755
1756 hibernate_setup_for_wake();
1757
1758 hibernate_page_list_setall(vars->page_list,
1759 vars->page_list_wired,
1760 &pageCount);
1761
1762 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
1763
1764 fileExtents = (IOPolledFileExtent *) vars->fileExtents->getBytesNoCopy();
1765
1766 #if 0
1767 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
1768 for (page = 0; page < count; page++)
1769 {
1770 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
1771 fileExtents[page].start, fileExtents[page].length,
1772 fileExtents[page].start + fileExtents[page].length);
1773 }
1774 #endif
1775
1776 needEncryptStart = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
1777
1778 AbsoluteTime_to_scalar(&compTime) = 0;
1779 AbsoluteTime_to_scalar(&decoTime) = 0;
1780
1781 clock_get_uptime(&allTime);
1782
1783 do
1784 {
1785 compressedSize = 0;
1786 uncompressedSize = 0;
1787 iterDone = false;
1788 pageType = 0; // wired pages first
1789
1790 IOPolledFileSeek(vars->fileVars, sizeof(IOHibernateImageHeader));
1791
1792 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
1793 ml_get_interrupts_enabled());
1794 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledBeforeSleepState, vars->ioBuffer);
1795 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
1796 pollerOpen = (kIOReturnSuccess == err);
1797 if (!pollerOpen)
1798 break;
1799
1800 // copy file block extent list if larger than header
1801
1802 count = vars->fileExtents->getLength();
1803 if (count > sizeof(header->fileExtentMap))
1804 {
1805 count -= sizeof(header->fileExtentMap);
1806 err = IOPolledFileWrite(vars->fileVars,
1807 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
1808 if (kIOReturnSuccess != err)
1809 break;
1810 }
1811
1812 // copy out restore1 code
1813
1814 page = atop_32(sectHIBB);
1815 count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
1816 header->restore1CodePage = page;
1817 header->restore1PageCount = count;
1818 header->restore1CodeOffset = ((uint32_t) &hibernate_machine_entrypoint) - sectHIBB;
1819 header->restore1StackOffset = ((uint32_t) &gIOHibernateRestoreStackEnd[0]) - 64 - sectHIBB;
1820
1821 // sum __HIB sect, with zeros for the stack
1822 src = (uint8_t *) trunc_page(sectHIBB);
1823 for (page = 0; page < count; page++)
1824 {
1825 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
1826 restore1Sum += hibernate_sum(src, page_size);
1827 else
1828 restore1Sum += 0x10000001;
1829 src += page_size;
1830 }
1831 sum1 = restore1Sum;
1832
1833 // write the __HIB sect, with zeros for the stack
1834
1835 src = (uint8_t *) trunc_page(sectHIBB);
1836 count = ((uint32_t) &gIOHibernateRestoreStack[0]) - trunc_page(sectHIBB);
1837 if (count)
1838 {
1839 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
1840 if (kIOReturnSuccess != err)
1841 break;
1842 }
1843 err = IOPolledFileWrite(vars->fileVars,
1844 (uint8_t *) 0,
1845 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
1846 cryptvars);
1847 if (kIOReturnSuccess != err)
1848 break;
1849 src = &gIOHibernateRestoreStackEnd[0];
1850 count = round_page(sectHIBB + sectSizeHIB) - ((uint32_t) src);
1851 if (count)
1852 {
1853 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
1854 if (kIOReturnSuccess != err)
1855 break;
1856 }
1857
1858 // write the preview buffer
1859
1860 addr64_t phys64;
1861 IOByteCount segLen;
1862
1863 if (vars->previewBuffer)
1864 {
1865 ppnum = 0;
1866 count = 0;
1867 do
1868 {
1869 phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen);
1870 pageAndCount[0] = atop_64(phys64);
1871 pageAndCount[1] = atop_32(segLen);
1872 err = IOPolledFileWrite(vars->fileVars,
1873 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1874 cryptvars);
1875 if (kIOReturnSuccess != err)
1876 break;
1877 count += segLen;
1878 ppnum += sizeof(pageAndCount);
1879 }
1880 while (phys64);
1881 if (kIOReturnSuccess != err)
1882 break;
1883
1884 src = (uint8_t *) vars->previewBuffer->getSourceSegment(0, NULL);
1885 count = vars->previewBuffer->getLength();
1886
1887 header->previewPageListSize = ppnum;
1888 header->previewSize = count + ppnum;
1889
1890 for (page = 0; page < count; page += page_size)
1891 sum1 += hibernate_sum(src + page, page_size);
1892
1893 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
1894 if (kIOReturnSuccess != err)
1895 break;
1896 }
1897
1898 // mark areas for no save
1899
1900 for (count = 0;
1901 (phys64 = vars->ioBuffer->getPhysicalSegment64(count, &segLen));
1902 count += segLen)
1903 {
1904 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1905 atop_64(phys64), atop_32(segLen),
1906 kIOHibernatePageStateFree);
1907 pageCount -= atop_32(segLen);
1908 }
1909
1910 for (count = 0;
1911 (phys64 = vars->srcBuffer->getPhysicalSegment64(count, &segLen));
1912 count += segLen)
1913 {
1914 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1915 atop_64(phys64), atop_32(segLen),
1916 kIOHibernatePageStateFree);
1917 pageCount -= atop_32(segLen);
1918 }
1919
1920 // copy out bitmap of pages available for trashing during restore
1921
1922 bitmap_size = vars->page_list_wired->list_size;
1923 src = (uint8_t *) vars->page_list_wired;
1924 err = IOPolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
1925 if (kIOReturnSuccess != err)
1926 break;
1927
1928 // mark more areas for no save, but these are not available
1929 // for trashing during restore
1930
1931 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
1932
1933 page = atop_32(sectHIBB);
1934 count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
1935 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1936 page, count,
1937 kIOHibernatePageStateFree);
1938 pageCount -= count;
1939
1940 if (vars->previewBuffer) for (count = 0;
1941 (phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen));
1942 count += segLen)
1943 {
1944 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1945 atop_64(phys64), atop_32(segLen),
1946 kIOHibernatePageStateFree);
1947 pageCount -= atop_32(segLen);
1948 }
1949
1950 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
1951
1952 ppnum = 0;
1953 pagesDone = 0;
1954 lastBlob = 0;
1955
1956 HIBLOG("writing %d pages\n", pageCount);
1957
1958 do
1959 {
1960 count = hibernate_page_list_iterate(pageType ? vars->page_list : vars->page_list_wired,
1961 &ppnum);
1962 // kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
1963
1964 iterDone = !count;
1965
1966 pageAndCount[0] = ppnum;
1967 pageAndCount[1] = count;
1968 err = IOPolledFileWrite(vars->fileVars,
1969 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1970 cryptvars);
1971 if (kIOReturnSuccess != err)
1972 break;
1973
1974 for (page = 0; page < count; page++)
1975 {
1976 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(ppnum), page_size);
1977 if (err)
1978 {
1979 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%d] %x\n", __LINE__, ppnum, err);
1980 break;
1981 }
1982
1983 sum = hibernate_sum(src, page_size);
1984
1985 clock_get_uptime(&startTime);
1986
1987 pageCompressedSize = WKdm_compress ((WK_word*) src, (WK_word*) (src + page_size), PAGE_SIZE_IN_WORDS);
1988
1989 clock_get_uptime(&endTime);
1990 ADD_ABSOLUTETIME(&compTime, &endTime);
1991 SUB_ABSOLUTETIME(&compTime, &startTime);
1992
1993 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1994 pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
1995
1996 if (pageCompressedSize > page_size)
1997 {
1998 // HIBLOG("------------lose: %d\n", pageCompressedSize);
1999 pageCompressedSize = page_size;
2000 }
2001
2002 if (pageCompressedSize != page_size)
2003 data = (src + page_size);
2004 else
2005 data = src;
2006
2007 tag = pageCompressedSize | kIOHibernateTagSignature;
2008
2009 if (pageType)
2010 sum2 += sum;
2011 else
2012 sum1 += sum;
2013
2014 if (needEncryptStart && (ppnum >= atop_32(sectDATAB)))
2015 {
2016 // start encrypting partway into the data about to be written
2017 vars->fileVars->encryptStart = (vars->fileVars->position + AES_BLOCK_SIZE - 1)
2018 & ~(AES_BLOCK_SIZE - 1);
2019 needEncryptStart = false;
2020 }
2021
2022 err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
2023 if (kIOReturnSuccess != err)
2024 break;
2025
2026 err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
2027 if (kIOReturnSuccess != err)
2028 break;
2029
2030 compressedSize += pageCompressedSize;
2031 if (pageCompressedSize)
2032 uncompressedSize += page_size;
2033 ppnum++;
2034 pagesDone++;
2035
2036 if (vars->consoleMapping && (0 == (1023 & pagesDone)))
2037 {
2038 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
2039 if (blob != lastBlob)
2040 {
2041 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
2042 lastBlob = blob;
2043 }
2044 }
2045 if (0 == (8191 & pagesDone))
2046 {
2047 clock_get_uptime(&endTime);
2048 SUB_ABSOLUTETIME(&endTime, &allTime);
2049 absolutetime_to_nanoseconds(endTime, &nsec);
2050 progressStamp = nsec / 750000000ULL;
2051 if (progressStamp != lastProgressStamp)
2052 {
2053 lastProgressStamp = progressStamp;
2054 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
2055 }
2056 }
2057 }
2058 if (kIOReturnSuccess != err)
2059 break;
2060 if (iterDone && !pageType)
2061 {
2062 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2063 if (kIOReturnSuccess != err)
2064 break;
2065
2066 iterDone = false;
2067 pageType = 1;
2068 ppnum = 0;
2069 image1Size = vars->fileVars->position;
2070 if (cryptvars)
2071 {
2072 bcopy(&cryptvars->aes_iv[0],
2073 &gIOHibernateCryptWakeContext.aes_iv[0],
2074 sizeof(cryptvars->aes_iv));
2075 cryptvars = &gIOHibernateCryptWakeContext;
2076 }
2077 HIBLOG("image1Size %qd\n", image1Size);
2078 }
2079 }
2080 while (!iterDone);
2081 if (kIOReturnSuccess != err)
2082 break;
2083 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2084 if (kIOReturnSuccess != err)
2085 break;
2086
2087 // Header:
2088
2089 header->imageSize = vars->fileVars->position;
2090 header->image1Size = image1Size;
2091 header->bitmapSize = bitmap_size;
2092 header->pageCount = pageCount;
2093 header->encryptStart = vars->fileVars->encryptStart;
2094
2095 header->restore1Sum = restore1Sum;
2096 header->image1Sum = sum1;
2097 header->image2Sum = sum2;
2098
2099 count = vars->fileExtents->getLength();
2100 if (count > sizeof(header->fileExtentMap))
2101 {
2102 header->fileExtentMapSize = count;
2103 count = sizeof(header->fileExtentMap);
2104 }
2105 else
2106 header->fileExtentMapSize = sizeof(header->fileExtentMap);
2107 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
2108
2109 IOPolledFileSeek(vars->fileVars, 0);
2110 err = IOPolledFileWrite(vars->fileVars,
2111 (uint8_t *) header, sizeof(IOHibernateImageHeader),
2112 cryptvars);
2113 if (kIOReturnSuccess != err)
2114 break;
2115 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2116 if (kIOReturnSuccess != err)
2117 break;
2118 err = IOHibernatePollerIODone(vars->fileVars, true);
2119 if (kIOReturnSuccess != err)
2120 break;
2121 }
2122 while (false);
2123
2124 clock_get_uptime(&endTime);
2125 SUB_ABSOLUTETIME(&endTime, &allTime);
2126 absolutetime_to_nanoseconds(endTime, &nsec);
2127 HIBLOG("all time: %qd ms, ",
2128 nsec / 1000000ULL);
2129
2130 absolutetime_to_nanoseconds(compTime, &nsec);
2131 HIBLOG("comp time: %qd ms, ",
2132 nsec / 1000000ULL);
2133
2134 absolutetime_to_nanoseconds(decoTime, &nsec);
2135 HIBLOG("deco time: %qd ms, ",
2136 nsec / 1000000ULL);
2137
2138 HIBLOG("\nimage %qd, uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
2139 header->imageSize,
2140 uncompressedSize, atop_32(uncompressedSize), compressedSize,
2141 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
2142 sum1, sum2);
2143
2144 if (pollerOpen)
2145 IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
2146
2147 if (vars->consoleMapping)
2148 ProgressUpdate(gIOHibernateGraphicsInfo,
2149 vars->consoleMapping, 0, kIOHibernateProgressCount);
2150
2151 HIBLOG("hibernate_write_image done(%x)\n", err);
2152
2153 // should we come back via regular wake, set the state in memory.
2154 gIOHibernateState = kIOHibernateStateInactive;
2155
2156 if (kIOReturnSuccess == err)
2157 {
2158 if (kIOHibernateModeSleep & gIOHibernateMode)
2159 {
2160 return (kIOHibernatePostWriteSleep);
2161 }
2162 else if(kIOHibernateModeRestart & gIOHibernateMode)
2163 {
2164 return (kIOHibernatePostWriteRestart);
2165 }
2166 else
2167 {
2168 /* by default, power down */
2169 return (kIOHibernatePostWriteHalt);
2170 }
2171 }
2172 else if (kIOReturnAborted == err)
2173 {
2174 return (kIOHibernatePostWriteWake);
2175 }
2176 else
2177 {
2178 /* on error, sleep */
2179 return (kIOHibernatePostWriteSleep);
2180 }
2181 }
2182
2183 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2184
2185 extern "C" void
2186 hibernate_machine_init(void)
2187 {
2188 IOReturn err;
2189 uint32_t sum;
2190 uint32_t pagesDone;
2191 AbsoluteTime allTime, endTime;
2192 uint64_t nsec;
2193 uint32_t lastProgressStamp = 0;
2194 uint32_t progressStamp;
2195 uint64_t progressZeroPosition = 0;
2196 uint32_t blob, lastBlob = (uint32_t) -1L;
2197 hibernate_cryptvars_t * cryptvars = 0;
2198
2199 IOHibernateVars * vars = &gIOHibernateVars;
2200
2201 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
2202 return;
2203
2204 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2205 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2206
2207 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, image1Size %qx, conflictCount %d, nextFree %x\n",
2208 gIOHibernateState, pagesDone, sum, gIOHibernateCurrentHeader->image1Size,
2209 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
2210
2211 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
2212 {
2213 HIBLOG("regular wake\n");
2214 return;
2215 }
2216
2217 HIBPRINT("diag %x %x %x %x\n",
2218 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
2219 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
2220
2221 HIBPRINT("video %x %d %d %d\n",
2222 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2223 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height);
2224
2225 if ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)
2226 hibernate_page_list_discard(vars->page_list);
2227
2228 boot_args *args = (boot_args *) PE_state.bootArgs;
2229
2230 if (vars->videoMapping
2231 && gIOHibernateGraphicsInfo->physicalAddress
2232 && (args->Video.v_baseAddr == gIOHibernateGraphicsInfo->physicalAddress))
2233 {
2234 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2235 * gIOHibernateGraphicsInfo->rowBytes);
2236 IOMapPages(kernel_map,
2237 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2238 vars->videoMapSize, kIOMapInhibitCache );
2239 }
2240
2241 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2242
2243 if (gIOHibernateWakeMapSize)
2244 {
2245 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(gIOHibernateWakeMap),
2246 gIOHibernateWakeMapSize);
2247 if (kIOReturnSuccess == err)
2248 hibernate_newruntime_map(src, gIOHibernateWakeMapSize,
2249 gIOHibernateCurrentHeader->systemTableOffset);
2250 gIOHibernateWakeMap = 0;
2251 gIOHibernateWakeMapSize = 0;
2252 }
2253
2254 uint32_t decoOffset;
2255
2256 clock_get_uptime(&allTime);
2257
2258 HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2259 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0);
2260 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
2261
2262 if (gIOHibernateCurrentHeader->previewSize)
2263 progressZeroPosition = gIOHibernateCurrentHeader->previewSize
2264 + gIOHibernateCurrentHeader->fileExtentMapSize
2265 - sizeof(gIOHibernateCurrentHeader->fileExtentMap)
2266 + ptoa_64(gIOHibernateCurrentHeader->restore1PageCount);
2267
2268 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2269
2270 if (vars->videoMapSize)
2271 {
2272 lastBlob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
2273 / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
2274 ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, 0, lastBlob);
2275 }
2276
2277 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
2278 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2279 {
2280 cryptvars = &gIOHibernateCryptWakeContext;
2281 bcopy(&gIOHibernateCryptWakeVars->aes_iv[0],
2282 &cryptvars->aes_iv[0],
2283 sizeof(cryptvars->aes_iv));
2284 }
2285
2286 // kick off the read ahead
2287 vars->fileVars->io = false;
2288 vars->fileVars->bufferHalf = 0;
2289 vars->fileVars->bufferLimit = 0;
2290 vars->fileVars->lastRead = 0;
2291 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2292
2293 IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
2294 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2295 // --
2296
2297 HIBLOG("hibernate_machine_init reading\n");
2298
2299 uint32_t * header = (uint32_t *) src;
2300 sum = 0;
2301
2302 do
2303 {
2304 unsigned int count;
2305 unsigned int page;
2306 uint32_t tag;
2307 vm_offset_t ppnum, compressedSize;
2308
2309 IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2310
2311 ppnum = header[0];
2312 count = header[1];
2313
2314 // HIBPRINT("(%x, %x)\n", ppnum, count);
2315
2316 if (!count)
2317 break;
2318
2319 for (page = 0; page < count; page++)
2320 {
2321 IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2322
2323 compressedSize = kIOHibernateTagLength & tag;
2324 if (!compressedSize)
2325 {
2326 ppnum++;
2327 pagesDone++;
2328 continue;
2329 }
2330
2331 IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2332
2333 if (compressedSize != page_size)
2334 {
2335 decoOffset = page_size;
2336 WKdm_decompress((WK_word*) src, (WK_word*) (src + decoOffset), PAGE_SIZE_IN_WORDS);
2337 }
2338 else
2339 decoOffset = 0;
2340
2341 sum += hibernate_sum((src + decoOffset), page_size);
2342
2343 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2344 if (err)
2345 HIBLOG("IOMemoryDescriptorReadToPhysical [%d] %x\n", ppnum, err);
2346
2347 ppnum++;
2348 pagesDone++;
2349
2350 if (vars->videoMapSize && (0 == (1023 & pagesDone)))
2351 {
2352 blob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
2353 / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
2354 if (blob != lastBlob)
2355 {
2356 ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, lastBlob, blob);
2357 lastBlob = blob;
2358 }
2359 }
2360
2361 if (0 == (8191 & pagesDone))
2362 {
2363 clock_get_uptime(&endTime);
2364 SUB_ABSOLUTETIME(&endTime, &allTime);
2365 absolutetime_to_nanoseconds(endTime, &nsec);
2366 progressStamp = nsec / 750000000ULL;
2367 if (progressStamp != lastProgressStamp)
2368 {
2369 lastProgressStamp = progressStamp;
2370 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2371 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2372 }
2373 }
2374 }
2375 }
2376 while (true);
2377
2378 gIOHibernateCurrentHeader->actualImage2Sum = sum;
2379
2380 if (vars->fileVars->io)
2381 (void) IOHibernatePollerIODone(vars->fileVars, false);
2382
2383 err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
2384
2385 if (vars->videoMapSize)
2386 ProgressUpdate(gIOHibernateGraphicsInfo,
2387 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2388
2389 clock_get_uptime(&endTime);
2390 SUB_ABSOLUTETIME(&endTime, &allTime);
2391 absolutetime_to_nanoseconds(endTime, &nsec);
2392
2393 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %qd ms\n",
2394 pagesDone, sum, nsec / 1000000ULL);
2395 }
2396
2397 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
mirror of XNU