]> git.saurik.com Git - apple/xnu.git/blob - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blob
? search:


39e1ddba03101be29b460f0f758d4b43920e3a2c
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
1 /*
2 * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the
10 * License may not be used to create, or enable the creation or
11 * redistribution of, unlawful or unlicensed copies of an Apple operating
12 * system, or to circumvent, violate, or enable the circumvention or
13 * violation of, any terms of an Apple operating system software license
14 * agreement.
15 *
16 * Please obtain a copy of the License at
17 * http://www.opensource.apple.com/apsl/ and read it before using this
18 * file.
19 *
20 * The Original Code and all software distributed under the License are
21 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
22 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
23 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
24 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
25 * Please see the License for the specific language governing rights and
26 * limitations under the License.
27 *
28 * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
29 */
30
31
32 /*
33
34 Sleep:
35
36 - PMRootDomain calls IOHibernateSystemSleep() before system sleep
37 (devices awake, normal execution context)
38 - IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
39 grabs its extents and searches for a polling driver willing to work with that IOMedia.
40 The BSD code makes an ioctl to the storage driver to get the partition base offset to
41 the disk, and other ioctls to get the transfer constraints
42 If successful, the file is written to make sure its initially not bootable (in case of
43 later failure) and nvram set to point to the first block of the file. (Has to be done
44 here so blocking is possible in nvram support).
45 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
46 page out any pages it wants to (currently zero, but probably some percentage of memory).
47 Its assumed just allocating pages will cause the VM system to naturally select the best
48 pages for eviction. It also copies processor flags needed for the restore path and sets
49 a flag in the boot processor proc info.
50 gIOHibernateState = kIOHibernateStateHibernating.
51 - Regular sleep progresses - some drivers may inspect the root domain property
52 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
53 as usual but leaves motherboard I/O on.
54 - Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
55 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
56 all ppc RC bits out of the hash table and caches into the mapping structures.
57 - hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
58 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
59 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
60 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
61 The image header and block list are written. The header includes the second file extent so
62 only the header block is needed to read the file, regardless of filesystem.
63 The kernel section "__HIB" is written uncompressed to the image. This section of code and data
64 (only) is used to decompress the image during wake/boot.
65 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
66 The bitmaps are written to the image.
67 More areas are removed from the bitmaps (after they have been written to the image) - the
68 section "__HIB" pages and interrupt stack.
69 Each wired page is compressed and written and then each non-wired page. Compression and
70 disk writes are in parallel.
71 The image header is written to the start of the file and the polling driver closed.
72 The machine powers down (or sleeps).
73
74 Boot/Wake:
75
76 - BootX sees the boot-image nvram variable containing the device and block number of the image,
77 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
78 - BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
79 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
80 that is in the kernel's __HIB section.
81 - BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
82 hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
83 only code & data in that section is safe to call since all the other wired pages are still
84 compressed in the image.
85 - hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
86 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
87 location directly, and copies those that can't to interim free pages. When the image has been
88 completed, the copies are uncompressed, overwriting the wired image pages.
89 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
90 is used to get pages into place for 64bit.
91 - the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
92 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
93 are removed from the software strutures, and the hash table is reinitialized.
94 - After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
95 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
96 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
97 for the remaining non wired pages.
98 - The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
99 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
100 is closed via bsd.
101
102 Polled Mode I/O:
103
104 IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
105 registry, specifying an object of calls IOPolledInterface.
106
107 Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
108 partition) that the image is going to live, looking for polled interface properties. If it finds
109 one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
110 interfaces found are kept in an ordered list.
111
112 There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
113 few different contexts things happen in:
114
115 - there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
116 up and running) and after wake - this is safe to allocate memory and do anything. The device
117 ignores sleep requests from that point since its a waste of time if it goes to sleep and
118 immediately wakes back up for the image write.
119
120 - there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
121 immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
122 from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
123 used to flush and set the disk to sleep.
124
125 - there is an Open/Close (AfterSleep) pair made around the image read operations that happen
126 immediately after sleep. These can't block or allocate memory. This is happening after the platform
127 expert has woken the low level bits of the system, but most of the I/O system has not. There is only
128 one thread running.
129
130 For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
131 (prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
132 the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
133 Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
134 that is called for the hardware to check for events, and complete the I/O via the callback.
135 The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
136 to restrict I/O ops.
137 */
138
139 #include <sys/systm.h>
140
141 #include <IOKit/IOWorkLoop.h>
142 #include <IOKit/IOCommandGate.h>
143 #include <IOKit/IOTimerEventSource.h>
144 #include <IOKit/IOPlatformExpert.h>
145 #include <IOKit/IOKitDebug.h>
146 #include <IOKit/IOTimeStamp.h>
147 #include <IOKit/pwr_mgt/RootDomain.h>
148 #include <IOKit/pwr_mgt/IOPMPrivate.h>
149 #include <IOKit/IOMessage.h>
150 #include <IOKit/IODeviceTreeSupport.h>
151 #include <IOKit/IOBSD.h>
152 #include "RootDomainUserClient.h"
153 #include <IOKit/pwr_mgt/IOPowerConnection.h>
154 #include "IOPMPowerStateQueue.h"
155 #include <IOKit/IOBufferMemoryDescriptor.h>
156 #include <crypto/aes.h>
157
158 #include <sys/uio.h>
159 #include <sys/conf.h>
160 #include <sys/stat.h>
161 #include <sys/fcntl.h> // (FWRITE, ...)
162 #include <sys/sysctl.h>
163
164 #include <IOKit/IOHibernatePrivate.h>
165 #include <IOKit/IOPolledInterface.h>
166 #include <IOKit/IONVRAM.h>
167 #include "IOHibernateInternal.h"
168 #include "WKdm.h"
169 #include "IOKitKernelInternal.h"
170
171 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
172
173 OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
174
175 OSMetaClassDefineReservedUnused(IOPolledInterface, 0);
176 OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
177 OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
178 OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
179 OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
180 OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
181 OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
182 OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
183 OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
184 OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
185 OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
186 OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
187 OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
188 OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
189 OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
190 OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
191
192 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
193
194 extern uint32_t gIOHibernateState;
195 uint32_t gIOHibernateMode;
196 static char gIOHibernateBootSignature[256+1];
197 static char gIOHibernateFilename[MAXPATHLEN+1];
198 static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
199 uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
200
201 static IODTNVRAM * gIOOptionsEntry;
202 static IORegistryEntry * gIOChosenEntry;
203
204 static IOPolledFileIOVars gFileVars;
205 static IOHibernateVars gIOHibernateVars;
206 static struct kern_direct_file_io_ref_t * gIOHibernateFileRef;
207 static hibernate_cryptvars_t gIOHibernateCryptWakeContext;
208
209 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
210
211 enum { kXPRamAudioVolume = 8 };
212 enum { kDefaultIOSize = 128 * 1024 };
213 enum { kVideoMapSize = 32 * 1024 * 1024 };
214
215 #ifndef kIOMediaPreferredBlockSizeKey
216 #define kIOMediaPreferredBlockSizeKey "Preferred Block Size"
217 #endif
218
219 #ifndef kIOBootPathKey
220 #define kIOBootPathKey "bootpath"
221 #endif
222 #ifndef kIOSelectedBootDeviceKey
223 #define kIOSelectedBootDeviceKey "boot-device"
224 #endif
225
226
227 enum { kIOHibernateMinPollersNeeded = 2 };
228
229 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
230
231 // copy from phys addr to MD
232
233 static IOReturn
234 IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
235 IOByteCount offset, addr64_t bytes, IOByteCount length)
236 {
237 addr64_t srcAddr = bytes;
238 IOByteCount remaining;
239
240 remaining = length = min(length, md->getLength() - offset);
241 while (remaining) { // (process another target segment?)
242 addr64_t dstAddr64;
243 IOByteCount dstLen;
244
245 dstAddr64 = md->getPhysicalSegment64(offset, &dstLen);
246 if (!dstAddr64)
247 break;
248
249 // Clip segment length to remaining
250 if (dstLen > remaining)
251 dstLen = remaining;
252
253 #if 1
254 bcopy_phys(srcAddr, dstAddr64, dstLen);
255 #else
256 copypv(srcAddr, dstAddr64, dstLen,
257 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
258 #endif
259 srcAddr += dstLen;
260 offset += dstLen;
261 remaining -= dstLen;
262 }
263
264 assert(!remaining);
265
266 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
267 }
268
269 // copy from MD to phys addr
270
271 static IOReturn
272 IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
273 IOByteCount offset, addr64_t bytes, IOByteCount length)
274 {
275 addr64_t dstAddr = bytes;
276 IOByteCount remaining;
277
278 remaining = length = min(length, md->getLength() - offset);
279 while (remaining) { // (process another target segment?)
280 addr64_t srcAddr64;
281 IOByteCount dstLen;
282
283 srcAddr64 = md->getPhysicalSegment64(offset, &dstLen);
284 if (!srcAddr64)
285 break;
286
287 // Clip segment length to remaining
288 if (dstLen > remaining)
289 dstLen = remaining;
290
291 #if 1
292 bcopy_phys(srcAddr64, dstAddr, dstLen);
293 #else
294 copypv(srcAddr, dstAddr64, dstLen,
295 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
296 #endif
297 dstAddr += dstLen;
298 offset += dstLen;
299 remaining -= dstLen;
300 }
301
302 assert(!remaining);
303
304 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
305 }
306
307 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
308
309 void
310 hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
311 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
312 {
313 count += ppnum;
314 switch (kind)
315 {
316 case kIOHibernatePageStateUnwiredSave:
317 // unwired save
318 for (; ppnum < count; ppnum++)
319 {
320 hibernate_page_bitset(page_list, FALSE, ppnum);
321 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
322 }
323 break;
324 case kIOHibernatePageStateWiredSave:
325 // wired save
326 for (; ppnum < count; ppnum++)
327 {
328 hibernate_page_bitset(page_list, FALSE, ppnum);
329 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
330 }
331 break;
332 case kIOHibernatePageStateFree:
333 // free page
334 for (; ppnum < count; ppnum++)
335 {
336 hibernate_page_bitset(page_list, TRUE, ppnum);
337 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
338 }
339 break;
340 default:
341 panic("hibernate_set_page_state");
342 }
343 }
344
345 static vm_offset_t
346 hibernate_page_list_iterate(hibernate_page_list_t * list,
347 void ** iterator, vm_offset_t * ppnum)
348 {
349 uint32_t count, idx;
350
351 idx = (uint32_t) *iterator;
352
353 if (!idx)
354 idx = hibernate_page_list_count(list, TRUE, idx);
355
356 *ppnum = idx;
357 count = hibernate_page_list_count(list, FALSE, idx);
358 idx += count;
359 idx += hibernate_page_list_count(list, TRUE, idx);
360 *iterator = (void *) idx;
361
362 return (count);
363 }
364
365 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
366
367 static IOReturn
368 IOHibernatePollerProbe(IOPolledFileIOVars * vars, IOService * target)
369 {
370 IOReturn err = kIOReturnError;
371 int32_t idx;
372 IOPolledInterface * poller;
373
374 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
375 {
376 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
377 err = poller->probe(target);
378 if (err)
379 {
380 HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
381 break;
382 }
383 }
384
385 return (err);
386 }
387
388 static IOReturn
389 IOHibernatePollerOpen(IOPolledFileIOVars * vars, uint32_t state, IOMemoryDescriptor * md)
390 {
391 IOReturn err = kIOReturnError;
392 int32_t idx;
393 IOPolledInterface * poller;
394
395 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
396 {
397 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
398 err = poller->open(state, md);
399 if (err)
400 {
401 HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
402 break;
403 }
404 }
405
406 return (err);
407 }
408
409 static IOReturn
410 IOHibernatePollerClose(IOPolledFileIOVars * vars, uint32_t state)
411 {
412 IOReturn err = kIOReturnError;
413 int32_t idx;
414 IOPolledInterface * poller;
415
416 for (idx = 0;
417 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
418 idx++)
419 {
420 err = poller->close(state);
421 if (err)
422 HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
423 }
424
425 return (err);
426 }
427
428 static void
429 IOHibernatePollerIOComplete(void * target,
430 void * parameter,
431 IOReturn status,
432 UInt64 actualByteCount)
433 {
434 IOPolledFileIOVars * vars = (IOPolledFileIOVars *) parameter;
435
436 vars->ioStatus = status;
437 }
438
439 static IOReturn
440 IOHibernatePollerIO(IOPolledFileIOVars * vars,
441 uint32_t operation, uint32_t bufferOffset,
442 uint64_t deviceOffset, uint64_t length)
443 {
444
445 IOReturn err = kIOReturnError;
446 IOPolledInterface * poller;
447 IOPolledCompletion completion;
448
449 completion.target = 0;
450 completion.action = &IOHibernatePollerIOComplete;
451 completion.parameter = vars;
452
453 vars->ioStatus = -1;
454
455 poller = (IOPolledInterface *) vars->pollers->getObject(0);
456 err = poller->startIO(operation, bufferOffset, deviceOffset + vars->block0, length, completion);
457 if (err)
458 HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
459
460 return (err);
461 }
462
463 static IOReturn
464 IOHibernatePollerIODone(IOPolledFileIOVars * vars)
465 {
466 IOReturn err = kIOReturnError;
467 int32_t idx;
468 IOPolledInterface * poller;
469
470 while (-1 == vars->ioStatus)
471 {
472 for (idx = 0;
473 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
474 idx++)
475 {
476 err = poller->checkForWork();
477 if (err)
478 HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
479 }
480 }
481
482 if (kIOReturnSuccess != vars->ioStatus)
483 HIBLOG("IOPolledInterface::ioStatus 0x%x\n", vars->ioStatus);
484
485 return (vars->ioStatus);
486 }
487
488 IOReturn
489 IOPolledInterface::checkAllForWork(void)
490 {
491 IOReturn err = kIOReturnNotReady;
492 int32_t idx;
493 IOPolledInterface * poller;
494
495 IOHibernateVars * vars = &gIOHibernateVars;
496
497 if (!vars->fileVars || !vars->fileVars->pollers)
498 return (err);
499
500 for (idx = 0;
501 (poller = (IOPolledInterface *) vars->fileVars->pollers->getObject(idx));
502 idx++)
503 {
504 err = poller->checkForWork();
505 if (err)
506 HIBLOG("IOPolledInterface::checkAllForWork[%d] 0x%x\n", idx, err);
507 }
508
509 return (err);
510 }
511
512 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
513
514 struct _OpenFileContext
515 {
516 OSData * extents;
517 uint64_t size;
518 };
519
520 static void
521 file_extent_callback(void * ref, uint64_t start, uint64_t length)
522 {
523 _OpenFileContext * ctx = (_OpenFileContext *) ref;
524 IOPolledFileExtent extent;
525
526 extent.start = start;
527 extent.length = length;
528
529 ctx->extents->appendBytes(&extent, sizeof(extent));
530 ctx->size += length;
531 }
532
533 IOReturn
534 IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
535 IOPolledFileIOVars ** fileVars, OSData ** fileExtents,
536 OSData ** imagePath)
537 {
538 IOReturn err = kIOReturnError;
539 IOPolledFileIOVars * vars;
540 _OpenFileContext ctx;
541 OSData * extentsData;
542 OSNumber * num;
543 IORegistryEntry * part = 0;
544 OSDictionary * matching;
545 OSIterator * iter;
546 dev_t hibernate_image_dev;
547 uint64_t maxiobytes;
548
549 vars = &gFileVars;
550 do
551 {
552 HIBLOG("sizeof(IOHibernateImageHeader) == %ld\n", sizeof(IOHibernateImageHeader));
553 if (sizeof(IOHibernateImageHeader) != 512)
554 continue;
555
556 vars->io = false;
557 vars->buffer = (uint8_t *) ioBuffer->getBytesNoCopy();
558 vars->bufferHalf = 0;
559 vars->bufferOffset = 0;
560 vars->bufferSize = ioBuffer->getLength() >> 1;
561
562 extentsData = OSData::withCapacity(32);
563
564 ctx.extents = extentsData;
565 ctx.size = 0;
566 vars->fileRef = kern_open_file_for_direct_io(filename,
567 &file_extent_callback, &ctx,
568 &hibernate_image_dev,
569 &vars->block0,
570 &maxiobytes);
571 if (!vars->fileRef)
572 {
573 err = kIOReturnNoSpace;
574 break;
575 }
576 HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx\n", filename, ctx.size,
577 vars->block0, maxiobytes);
578 if (ctx.size < 1*1024*1024) // check against image size estimate!
579 {
580 err = kIOReturnNoSpace;
581 break;
582 }
583
584 if (maxiobytes < vars->bufferSize)
585 vars->bufferSize = maxiobytes;
586
587 vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
588
589 matching = IOService::serviceMatching("IOMedia");
590 num = OSNumber::withNumber(major(hibernate_image_dev), 32);
591 matching->setObject(kIOBSDMajorKey, num);
592 num->release();
593 num = OSNumber::withNumber(minor(hibernate_image_dev), 32);
594 matching->setObject(kIOBSDMinorKey, num);
595 num->release();
596 iter = IOService::getMatchingServices(matching);
597 matching->release();
598 if (iter)
599 {
600 part = (IORegistryEntry *) iter->getNextObject();
601 part->retain();
602 iter->release();
603 }
604
605 int minor, major;
606 IORegistryEntry * next;
607 IORegistryEntry * child;
608 OSData * data;
609
610 num = (OSNumber *) part->getProperty(kIOBSDMajorKey);
611 if (!num)
612 break;
613 major = num->unsigned32BitValue();
614 num = (OSNumber *) part->getProperty(kIOBSDMinorKey);
615 if (!num)
616 break;
617 minor = num->unsigned32BitValue();
618
619 hibernate_image_dev = makedev(major, minor);
620
621 vars->pollers = OSArray::withCapacity(4);
622 if (!vars->pollers)
623 break;
624
625 vars->blockSize = 512;
626 next = part;
627 do
628 {
629 IOPolledInterface * poller;
630 OSObject * obj;
631
632 obj = next->getProperty(kIOPolledInterfaceSupportKey);
633 if (kOSBooleanFalse == obj)
634 {
635 vars->pollers->flushCollection();
636 break;
637 }
638 else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
639 vars->pollers->setObject(poller);
640 if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey))))
641 vars->blockSize = num->unsigned32BitValue();
642 child = next;
643 }
644 while ((next = child->getParentEntry(gIOServicePlane))
645 && child->isParent(next, gIOServicePlane, true));
646
647 HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
648 major, minor, vars->blockSize, vars->pollers->getCount());
649 if (vars->pollers->getCount() < kIOHibernateMinPollersNeeded)
650 continue;
651
652 err = IOHibernatePollerProbe(vars, (IOService *) part);
653 if (kIOReturnSuccess != err)
654 break;
655
656 err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer);
657 if (kIOReturnSuccess != err)
658 break;
659
660 *fileVars = vars;
661 *fileExtents = extentsData;
662
663 // make imagePath
664 char str1[256];
665 char str2[24];
666 int len = sizeof(str1);
667
668 if ((extentsData->getLength() >= sizeof(IOPolledFileExtent))
669 && part->getPath(str1, &len, gIODTPlane))
670 {
671 // (strip the plane name)
672 char * tail = strchr(str1, ':');
673 if (!tail)
674 tail = str1 - 1;
675 data = OSData::withBytes(tail + 1, strlen(tail + 1));
676 sprintf(str2, ",%qx", vars->extentMap[0]);
677 data->appendBytes(str2, strlen(str2));
678 *imagePath = data;
679 }
680 }
681 while (false);
682
683 if (kIOReturnSuccess != err)
684 {
685 HIBLOG("error 0x%x opening hibernation file\n", err);
686 if (vars->fileRef)
687 kern_close_file_for_direct_io(vars->fileRef);
688 }
689
690 if (part)
691 part->release();
692
693 return (err);
694 }
695
696 IOReturn
697 IOPolledFileClose( IOPolledFileIOVars * vars )
698 {
699 if (vars->pollers)
700 {
701 IOHibernatePollerClose(vars, kIOPolledPostflightState);
702 vars->pollers->release();
703 }
704
705 gIOHibernateFileRef = vars->fileRef;
706
707 bzero(vars, sizeof(IOPolledFileIOVars));
708
709 return (kIOReturnSuccess);
710 }
711
712 static IOReturn
713 IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
714 {
715 IOPolledFileExtent * extentMap;
716
717 extentMap = vars->extentMap;
718
719 vars->position = position;
720
721 while (position >= extentMap->length)
722 {
723 position -= extentMap->length;
724 extentMap++;
725 }
726
727 vars->currentExtent = extentMap;
728 vars->extentRemaining = extentMap->length - position;
729 vars->extentPosition = vars->position - position;
730
731 if (vars->bufferSize <= vars->extentRemaining)
732 vars->bufferLimit = vars->bufferSize;
733 else
734 vars->bufferLimit = vars->extentRemaining;
735
736 return (kIOReturnSuccess);
737 }
738
739 static IOReturn
740 IOPolledFileWrite(IOPolledFileIOVars * vars,
741 const uint8_t * bytes, IOByteCount size,
742 hibernate_cryptvars_t * cryptvars)
743 {
744 IOReturn err = kIOReturnSuccess;
745 IOByteCount copy;
746 bool flush = false;
747
748 do
749 {
750 if (!bytes && !size)
751 {
752 // seek to end of block & flush
753 size = vars->position & (vars->blockSize - 1);
754 if (size)
755 size = vars->blockSize - size;
756 flush = true;
757 // use some garbage for the fill
758 bytes = vars->buffer + vars->bufferOffset;
759 }
760
761 copy = vars->bufferLimit - vars->bufferOffset;
762 if (copy > size)
763 copy = size;
764 else
765 flush = true;
766
767 if (bytes)
768 {
769 bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
770 bytes += copy;
771 }
772 else
773 bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
774
775 size -= copy;
776 vars->bufferOffset += copy;
777 vars->position += copy;
778
779 if (flush && vars->bufferOffset)
780 {
781 uint64_t offset = (vars->position - vars->bufferOffset
782 - vars->extentPosition + vars->currentExtent->start);
783 uint32_t length = (vars->bufferOffset);
784
785 if (cryptvars && vars->encryptStart && (vars->position > vars->encryptStart))
786 {
787 uint32_t encryptLen, encryptStart;
788 encryptLen = vars->position - vars->encryptStart;
789 if (encryptLen > length)
790 encryptLen = length;
791 encryptStart = length - encryptLen;
792
793 // encrypt the buffer
794 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
795 &cryptvars->aes_iv[0],
796 encryptLen / AES_BLOCK_SIZE,
797 vars->buffer + vars->bufferHalf + encryptStart,
798 &cryptvars->ctx.encrypt);
799 // save initial vector for following encrypts
800 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
801 &cryptvars->aes_iv[0],
802 AES_BLOCK_SIZE);
803 }
804
805 if (vars->io)
806 {
807 err = IOHibernatePollerIODone(vars);
808 if (kIOReturnSuccess != err)
809 break;
810 }
811
812 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
813 //if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
814
815 err = IOHibernatePollerIO(vars, kIOPolledWrite, vars->bufferHalf, offset, length);
816 if (kIOReturnSuccess != err)
817 break;
818 vars->io = true;
819
820 vars->extentRemaining -= vars->bufferOffset;
821 if (!vars->extentRemaining)
822 {
823 vars->currentExtent++;
824 vars->extentRemaining = vars->currentExtent->length;
825 vars->extentPosition = vars->position;
826 if (!vars->extentRemaining)
827 {
828 err = kIOReturnOverrun;
829 break;
830 }
831 }
832
833 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
834 vars->bufferOffset = 0;
835 if (vars->bufferSize <= vars->extentRemaining)
836 vars->bufferLimit = vars->bufferSize;
837 else
838 vars->bufferLimit = vars->extentRemaining;
839
840 flush = false;
841 }
842 }
843 while (size);
844
845 return (err);
846 }
847
848 static IOReturn
849 IOPolledFileRead(IOPolledFileIOVars * vars,
850 uint8_t * bytes, IOByteCount size,
851 hibernate_cryptvars_t * cryptvars)
852 {
853 IOReturn err = kIOReturnSuccess;
854 IOByteCount copy;
855
856 // bytesWritten += size;
857
858 do
859 {
860 copy = vars->bufferLimit - vars->bufferOffset;
861 if (copy > size)
862 copy = size;
863
864 if (bytes)
865 {
866 bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
867 bytes += copy;
868 }
869 size -= copy;
870 vars->bufferOffset += copy;
871 // vars->position += copy;
872
873 if (vars->bufferOffset == vars->bufferLimit)
874 {
875 if (vars->io)
876 {
877 err = IOHibernatePollerIODone(vars);
878 if (kIOReturnSuccess != err)
879 break;
880 }
881 else
882 cryptvars = 0;
883
884 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
885
886 vars->position += vars->lastRead;
887 vars->extentRemaining -= vars->lastRead;
888 vars->bufferLimit = vars->lastRead;
889
890 if (!vars->extentRemaining)
891 {
892 vars->currentExtent++;
893 vars->extentRemaining = vars->currentExtent->length;
894 vars->extentPosition = vars->position;
895 if (!vars->extentRemaining)
896 {
897 err = kIOReturnOverrun;
898 break;
899 }
900 }
901
902 if (vars->extentRemaining <= vars->bufferSize)
903 vars->lastRead = vars->extentRemaining;
904 else
905 vars->lastRead = vars->bufferSize;
906
907 uint64_t offset = (vars->position
908 - vars->extentPosition + vars->currentExtent->start);
909 uint64_t length = (vars->lastRead);
910
911 //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
912
913 err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
914 if (kIOReturnSuccess != err)
915 break;
916 vars->io = true;
917
918 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
919 vars->bufferOffset = 0;
920
921 if (cryptvars)
922 {
923 uint8_t thisVector[AES_BLOCK_SIZE];
924 // save initial vector for following decrypts
925 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
926 bcopy(vars->buffer + vars->bufferHalf + vars->lastRead - AES_BLOCK_SIZE,
927 &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
928 // decrypt the buffer
929 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
930 &thisVector[0],
931 vars->lastRead / AES_BLOCK_SIZE,
932 vars->buffer + vars->bufferHalf,
933 &cryptvars->ctx.decrypt);
934 }
935 }
936 }
937 while (size);
938
939 return (err);
940 }
941
942 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
943
944 IOReturn
945 IOHibernateSystemSleep(void)
946 {
947 IOReturn err;
948 OSData * data;
949 OSObject * obj;
950 OSString * str;
951 OSNumber * num;
952
953 IOHibernateVars * vars = &gIOHibernateVars;
954
955 if (vars->fileVars && vars->fileVars->fileRef)
956 // already on the way down
957 return (kIOReturnSuccess);
958
959 gIOHibernateState = kIOHibernateStateInactive;
960
961 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateModeKey)))
962 {
963 if ((num = OSDynamicCast(OSNumber, obj)))
964 gIOHibernateMode = num->unsigned32BitValue();
965 if (kIOHibernateModeSleep & gIOHibernateMode)
966 // default to discard clean for safe sleep
967 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
968 | kIOHibernateModeDiscardCleanActive);
969
970 obj->release();
971 }
972 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeRatioKey)))
973 {
974 if ((num = OSDynamicCast(OSNumber, obj)))
975 gIOHibernateFreeRatio = num->unsigned32BitValue();
976 obj->release();
977 }
978 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeTimeKey)))
979 {
980 if ((num = OSDynamicCast(OSNumber, obj)))
981 gIOHibernateFreeTime = num->unsigned32BitValue();
982 obj->release();
983 }
984 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
985 {
986 if ((str = OSDynamicCast(OSString, obj)))
987 strcpy(gIOHibernateFilename, str->getCStringNoCopy());
988 obj->release();
989 }
990
991 if (!gIOHibernateMode || !gIOHibernateFilename[0])
992 return (kIOReturnUnsupported);
993
994 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
995
996 do
997 {
998 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(0, 4 * page_size, page_size);
999 vars->ioBuffer = IOBufferMemoryDescriptor::withOptions(0, 2 * kDefaultIOSize, page_size);
1000
1001 if (!vars->srcBuffer || !vars->ioBuffer)
1002 {
1003 err = kIOReturnNoMemory;
1004 break;
1005 }
1006
1007 err = IOPolledFileOpen(gIOHibernateFilename, vars->ioBuffer,
1008 &vars->fileVars, &vars->fileExtents, &data);
1009 if (KERN_SUCCESS != err)
1010 {
1011 HIBLOG("IOPolledFileOpen(%x)\n", err);
1012 break;
1013 }
1014 if (vars->fileVars->fileRef)
1015 {
1016 // invalidate the image file
1017 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1018 int err = kern_write_file(vars->fileVars->fileRef, 0,
1019 (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1020 if (KERN_SUCCESS != err)
1021 HIBLOG("kern_write_file(%d)\n", err);
1022 }
1023
1024 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1025
1026 boolean_t encryptedswap;
1027 err = hibernate_setup(gIOHibernateCurrentHeader,
1028 gIOHibernateFreeRatio, gIOHibernateFreeTime,
1029 &vars->page_list, &vars->page_list_wired, &encryptedswap);
1030 if (KERN_SUCCESS != err)
1031 {
1032 HIBLOG("hibernate_setup(%d)\n", err);
1033 break;
1034 }
1035
1036 if (encryptedswap)
1037 gIOHibernateMode ^= kIOHibernateModeEncrypt;
1038
1039 vars->videoAllocSize = kVideoMapSize;
1040 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
1041 vars->videoMapping = 0;
1042
1043 // generate crypt keys
1044 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
1045 vars->wiredCryptKey[i] = random();
1046 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
1047 vars->cryptKey[i] = random();
1048
1049 // set nvram
1050
1051 IORegistryEntry * regEntry;
1052 if (!gIOOptionsEntry)
1053 {
1054 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
1055 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
1056 if (regEntry && !gIOOptionsEntry)
1057 regEntry->release();
1058 }
1059 if (!gIOChosenEntry)
1060 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1061
1062 if (gIOOptionsEntry)
1063 {
1064 const OSSymbol * sym;
1065 size_t len;
1066 char valueString[16];
1067
1068 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1069 if (sym)
1070 {
1071 gIOOptionsEntry->setProperty(sym, data);
1072 sym->release();
1073 }
1074 data->release();
1075
1076 vars->saveBootDevice = gIOOptionsEntry->copyProperty(kIOSelectedBootDeviceKey);
1077 if (gIOChosenEntry)
1078 {
1079 OSData * bootDevice = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOBootPathKey));
1080 if (bootDevice)
1081 {
1082 sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
1083 OSString * str2 = OSString::withCStringNoCopy((const char *) bootDevice->getBytesNoCopy());
1084 if (sym && str2)
1085 gIOOptionsEntry->setProperty(sym, str2);
1086 if (sym)
1087 sym->release();
1088 if (str2)
1089 str2->release();
1090 }
1091
1092 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMemorySignatureKey));
1093 if (data)
1094 {
1095 vars->haveFastBoot = true;
1096
1097 len = sprintf(valueString, "0x%lx", *((UInt32 *)data->getBytesNoCopy()));
1098 data = OSData::withBytes(valueString, len + 1);
1099 sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
1100 if (sym && data)
1101 gIOOptionsEntry->setProperty(sym, data);
1102 if (sym)
1103 sym->release();
1104 if (data)
1105 data->release();
1106 }
1107 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
1108 if (data)
1109 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
1110 }
1111
1112 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1113 {
1114 data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1115 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
1116 if (sym && data)
1117 gIOOptionsEntry->setProperty(sym, data);
1118 if (sym)
1119 sym->release();
1120 if (data)
1121 data->release();
1122 if (gIOHibernateBootSignature[0])
1123 {
1124 data = OSData::withCapacity(16);
1125 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
1126 if (sym && data)
1127 {
1128 char c;
1129 uint8_t value;
1130 for (uint32_t i = 0; (c = gIOHibernateBootSignature[i]); i++)
1131 {
1132 if (c >= 'a')
1133 c -= 'a' - 10;
1134 else if (c >= 'A')
1135 c -= 'A' - 10;
1136 else if (c >= '0')
1137 c -= '0';
1138 else
1139 continue;
1140 value = (value << 4) | c;
1141 if (i & 1)
1142 data->appendBytes(&value, sizeof(value));
1143 }
1144 gIOOptionsEntry->setProperty(sym, data);
1145 }
1146 if (sym)
1147 sym->release();
1148 if (data)
1149 data->release();
1150 }
1151 }
1152
1153 if (!vars->haveFastBoot)
1154 {
1155 // set boot volume to zero
1156 IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
1157 if (platform && (kIOReturnSuccess == platform->readXPRAM(kXPRamAudioVolume,
1158 &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume))))
1159 {
1160 uint8_t newVolume;
1161 newVolume = vars->saveBootAudioVolume & 0xf8;
1162 platform->writeXPRAM(kXPRamAudioVolume,
1163 &newVolume, sizeof(newVolume));
1164 }
1165 }
1166 }
1167 // --
1168
1169 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
1170 gIOHibernateState = kIOHibernateStateHibernating;
1171 }
1172 while (false);
1173
1174 return (err);
1175 }
1176
1177 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1178
1179 IOReturn
1180 IOHibernateSystemHasSlept(void)
1181 {
1182 IOHibernateVars * vars = &gIOHibernateVars;
1183
1184 if ((vars->previewData = OSDynamicCast(OSData,
1185 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewBufferKey))))
1186 {
1187 vars->previewBuffer = IOMemoryDescriptor::withAddress(
1188 (void *) vars->previewData->getBytesNoCopy(),
1189 vars->previewData->getLength(),
1190 kIODirectionInOut);
1191
1192 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
1193 {
1194 vars->previewBuffer->release();
1195 vars->previewBuffer = 0;
1196 }
1197 if (!vars->previewBuffer)
1198 vars->previewData = 0;
1199 }
1200 if (gIOOptionsEntry)
1201 gIOOptionsEntry->sync();
1202
1203 return (kIOReturnSuccess);
1204 }
1205
1206 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1207
1208 IOReturn
1209 IOHibernateSystemWake(void)
1210 {
1211 IOHibernateVars * vars = &gIOHibernateVars;
1212
1213 hibernate_teardown(vars->page_list, vars->page_list_wired);
1214
1215 if (vars->videoMapping)
1216 {
1217 if (vars->videoMapSize)
1218 // remove mappings
1219 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1220 if (vars->videoAllocSize)
1221 // dealloc range
1222 kmem_free(kernel_map, trunc_page_32(vars->videoMapping), vars->videoAllocSize);
1223 }
1224
1225 if (vars->previewBuffer)
1226 {
1227 vars->previewBuffer->release();
1228 vars->previewBuffer = 0;
1229 }
1230
1231 if (vars->fileVars)
1232 {
1233 IOPolledFileClose(vars->fileVars);
1234 }
1235
1236 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1237
1238 OSData * data = OSData::withCapacity(4);
1239 if (gIOOptionsEntry && data)
1240 {
1241 const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1242 if (sym)
1243 {
1244 gIOOptionsEntry->setProperty(sym, data);
1245 sym->release();
1246 }
1247 sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
1248 if (sym)
1249 {
1250 if (vars->saveBootDevice)
1251 {
1252 gIOOptionsEntry->setProperty(sym, vars->saveBootDevice);
1253 vars->saveBootDevice->release();
1254 }
1255 sym->release();
1256 }
1257 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
1258 if (sym)
1259 {
1260 gIOOptionsEntry->setProperty(sym, data);
1261 sym->release();
1262 }
1263 sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
1264 if (sym)
1265 {
1266 gIOOptionsEntry->removeProperty(sym);
1267 sym->release();
1268 }
1269 }
1270 if (data)
1271 data->release();
1272
1273 if (gIOOptionsEntry)
1274 {
1275 if (!vars->haveFastBoot)
1276 {
1277 // reset boot audio volume
1278 IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
1279 if (platform)
1280 platform->writeXPRAM(kXPRamAudioVolume,
1281 &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume));
1282 }
1283
1284 // sync now to hardware if the booter has not
1285 if (kIOHibernateStateInactive == gIOHibernateState)
1286 gIOOptionsEntry->sync();
1287 else
1288 // just sync the variables in case a later panic syncs nvram (it won't sync variables)
1289 gIOOptionsEntry->syncOFVariables();
1290 }
1291
1292 if (vars->srcBuffer)
1293 vars->srcBuffer->release();
1294 if (vars->ioBuffer)
1295 vars->ioBuffer->release();
1296 if (vars->fileExtents)
1297 vars->fileExtents->release();
1298
1299 bzero(vars, sizeof(*vars));
1300
1301 // gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1302
1303 return (kIOReturnSuccess);
1304 }
1305
1306 IOReturn
1307 IOHibernateSystemPostWake(void)
1308 {
1309 if (gIOHibernateFileRef)
1310 {
1311 kern_close_file_for_direct_io(gIOHibernateFileRef);
1312 gIOHibernateFileRef = 0;
1313 }
1314 return (kIOReturnSuccess);
1315 }
1316
1317 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1318
1319 void
1320 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1321 {
1322 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
1323 if (data)
1324 {
1325 rootDomain->setProperty(kIOHibernateStateKey, data);
1326 data->release();
1327 }
1328
1329 if (PE_parse_boot_arg("hfile", gIOHibernateFilename))
1330 gIOHibernateMode = kIOHibernateModeOn;
1331 else
1332 gIOHibernateFilename[0] = 0;
1333
1334 static SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
1335 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
1336 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
1337 sysctl_register_oid(&sysctl__kern_hibernatefile);
1338
1339 static SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
1340 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
1341 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
1342 sysctl_register_oid(&sysctl__kern_bootsignature);
1343
1344 static SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
1345 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
1346 &gIOHibernateMode, 0, "");
1347 sysctl_register_oid(&sysctl__kern_hibernatemode);
1348 }
1349
1350 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1351
1352 static void
1353 hibernate_setup_for_wake(void)
1354 {
1355 #if __ppc__
1356 // go slow (state needed for wake)
1357 ml_set_processor_speed(1);
1358 #endif
1359 }
1360
1361 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1362
1363 extern "C" boolean_t
1364 hibernate_write_image(void)
1365 {
1366 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
1367 IOHibernateVars * vars = &gIOHibernateVars;
1368 IOPolledFileExtent * fileExtents;
1369
1370 uint32_t pageCount, pagesDone;
1371 IOReturn err;
1372 vm_offset_t ppnum;
1373 IOItemCount page, count;
1374 uint8_t * src;
1375 uint8_t * data;
1376 IOByteCount pageCompressedSize;
1377 uint64_t compressedSize, uncompressedSize;
1378 uint64_t image1Size = 0;
1379 uint32_t bitmap_size;
1380 bool iterDone, pollerOpen, needEncryptStart;
1381 uint32_t restore1Sum, sum, sum1, sum2;
1382 uint32_t tag;
1383 uint32_t pageType;
1384 uint32_t pageAndCount[2];
1385
1386 AbsoluteTime startTime, endTime;
1387 AbsoluteTime allTime, compTime, decoTime;
1388 uint64_t nsec;
1389 uint32_t lastProgressStamp = 0;
1390 uint32_t progressStamp;
1391
1392 hibernate_cryptvars_t _cryptvars;
1393 hibernate_cryptvars_t * cryptvars = 0;
1394
1395 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
1396 return (false /* sleep */ );
1397
1398 restore1Sum = sum1 = sum2 = 0;
1399
1400 // encryption data. "iv" is the "initial vector".
1401 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1402 {
1403 static const unsigned char first_iv[AES_BLOCK_SIZE]
1404 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
1405 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
1406
1407 cryptvars = &gIOHibernateCryptWakeContext;
1408 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
1409 aes_encrypt_key(vars->cryptKey,
1410 kIOHibernateAESKeySize,
1411 &cryptvars->ctx.encrypt);
1412 aes_decrypt_key(vars->cryptKey,
1413 kIOHibernateAESKeySize,
1414 &cryptvars->ctx.decrypt);
1415
1416 cryptvars = &_cryptvars;
1417 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
1418 aes_encrypt_key(vars->wiredCryptKey,
1419 kIOHibernateAESKeySize,
1420 &cryptvars->ctx.encrypt);
1421
1422 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1423 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1424 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
1425 bzero(gIOHibernateCryptWakeVars, sizeof(hibernate_cryptwakevars_t));
1426 }
1427
1428 hibernate_setup_for_wake();
1429
1430 hibernate_page_list_setall(vars->page_list,
1431 vars->page_list_wired,
1432 &pageCount);
1433
1434 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
1435
1436 fileExtents = (IOPolledFileExtent *) vars->fileExtents->getBytesNoCopy();
1437
1438 #if 0
1439 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
1440 for (page = 0; page < count; page++)
1441 {
1442 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
1443 fileExtents[page].start, fileExtents[page].length,
1444 fileExtents[page].start + fileExtents[page].length);
1445 }
1446 #endif
1447
1448 needEncryptStart = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
1449
1450 AbsoluteTime_to_scalar(&compTime) = 0;
1451 AbsoluteTime_to_scalar(&decoTime) = 0;
1452
1453 clock_get_uptime(&allTime);
1454
1455 do
1456 {
1457 compressedSize = 0;
1458 uncompressedSize = 0;
1459 iterDone = false;
1460 pageType = 0; // wired pages first
1461
1462 IOPolledFileSeek(vars->fileVars, sizeof(IOHibernateImageHeader));
1463
1464 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
1465 ml_get_interrupts_enabled());
1466 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledBeforeSleepState, vars->ioBuffer);
1467 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
1468 pollerOpen = (kIOReturnSuccess == err);
1469 if (!pollerOpen)
1470 break;
1471
1472 // copy file block extent list if larger than header
1473
1474 count = vars->fileExtents->getLength();
1475 if (count > sizeof(header->fileExtentMap))
1476 {
1477 count -= sizeof(header->fileExtentMap);
1478 err = IOPolledFileWrite(vars->fileVars,
1479 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
1480 if (kIOReturnSuccess != err)
1481 break;
1482 }
1483
1484 // copy out restore1 code
1485
1486 page = atop_32(sectHIBB);
1487 count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
1488 header->restore1CodePage = page;
1489 header->restore1PageCount = count;
1490 header->restore1CodeOffset = ((uint32_t) &hibernate_machine_entrypoint) - sectHIBB;
1491 header->restore1StackOffset = ((uint32_t) &gIOHibernateRestoreStackEnd[0]) - 64 - sectHIBB;
1492
1493 // sum __HIB sect, with zeros for the stack
1494 src = (uint8_t *) trunc_page(sectHIBB);
1495 for (page = 0; page < count; page++)
1496 {
1497 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
1498 restore1Sum += hibernate_sum(src, page_size);
1499 else
1500 restore1Sum += 0x10000001;
1501 src += page_size;
1502 }
1503 sum1 = restore1Sum;
1504
1505 // write the __HIB sect, with zeros for the stack
1506
1507 src = (uint8_t *) trunc_page(sectHIBB);
1508 count = ((uint32_t) &gIOHibernateRestoreStack[0]) - trunc_page(sectHIBB);
1509 if (count)
1510 {
1511 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
1512 if (kIOReturnSuccess != err)
1513 break;
1514 }
1515 err = IOPolledFileWrite(vars->fileVars,
1516 (uint8_t *) 0,
1517 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
1518 cryptvars);
1519 if (kIOReturnSuccess != err)
1520 break;
1521 src = &gIOHibernateRestoreStackEnd[0];
1522 count = round_page(sectHIBB + sectSizeHIB) - ((uint32_t) src);
1523 if (count)
1524 {
1525 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
1526 if (kIOReturnSuccess != err)
1527 break;
1528 }
1529
1530 // write the preview buffer
1531
1532 addr64_t phys64;
1533 IOByteCount segLen;
1534
1535 if (vars->previewData)
1536 {
1537 ppnum = 0;
1538 count = 0;
1539 do
1540 {
1541 phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen);
1542 pageAndCount[0] = atop_64(phys64);
1543 pageAndCount[1] = atop_32(segLen);
1544 err = IOPolledFileWrite(vars->fileVars,
1545 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1546 cryptvars);
1547 if (kIOReturnSuccess != err)
1548 break;
1549 count += segLen;
1550 ppnum += sizeof(pageAndCount);
1551 }
1552 while (phys64);
1553 if (kIOReturnSuccess != err)
1554 break;
1555
1556 src = (uint8_t *) vars->previewData->getBytesNoCopy();
1557 count = vars->previewData->getLength();
1558
1559 header->previewPageListSize = ppnum;
1560 header->previewSize = count + ppnum;
1561
1562 for (page = 0; page < count; page += page_size)
1563 sum1 += hibernate_sum(src + page, page_size);
1564
1565 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
1566 if (kIOReturnSuccess != err)
1567 break;
1568 }
1569
1570 // mark areas for no save
1571
1572 for (count = 0;
1573 (phys64 = vars->ioBuffer->getPhysicalSegment64(count, &segLen));
1574 count += segLen)
1575 {
1576 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1577 atop_64(phys64), atop_32(segLen),
1578 kIOHibernatePageStateFree);
1579 pageCount -= atop_32(segLen);
1580 }
1581
1582 for (count = 0;
1583 (phys64 = vars->srcBuffer->getPhysicalSegment64(count, &segLen));
1584 count += segLen)
1585 {
1586 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1587 atop_64(phys64), atop_32(segLen),
1588 kIOHibernatePageStateFree);
1589 pageCount -= atop_32(segLen);
1590 }
1591
1592 // copy out bitmap of pages available for trashing during restore
1593
1594 bitmap_size = vars->page_list_wired->list_size;
1595 src = (uint8_t *) vars->page_list_wired;
1596 err = IOPolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
1597 if (kIOReturnSuccess != err)
1598 break;
1599
1600 // mark more areas for no save, but these are not available
1601 // for trashing during restore
1602
1603 #if !__i386__
1604 page = atop_32(sectHIBB);
1605 count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
1606 #else
1607 // XXX
1608 page = atop_32(sectHIBB & 0x3FFFFFFF);
1609 count = atop_32(round_page((sectHIBB + sectSizeHIB) & 0x3FFFFFFF)) - page;
1610 #endif
1611 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1612 page, count,
1613 kIOHibernatePageStateFree);
1614 pageCount -= count;
1615
1616
1617
1618 if (vars->previewBuffer) for (count = 0;
1619 (phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen));
1620 count += segLen)
1621 {
1622 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1623 atop_64(phys64), atop_32(segLen),
1624 kIOHibernatePageStateFree);
1625 pageCount -= atop_32(segLen);
1626 }
1627
1628 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
1629
1630 void * iter = 0;
1631 pagesDone = 0;
1632
1633 HIBLOG("writing %d pages\n", pageCount);
1634
1635 do
1636 {
1637 count = hibernate_page_list_iterate(pageType ? vars->page_list : vars->page_list_wired,
1638 &iter, &ppnum);
1639 // kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
1640
1641 iterDone = !count;
1642
1643 pageAndCount[0] = ppnum;
1644 pageAndCount[1] = count;
1645 err = IOPolledFileWrite(vars->fileVars,
1646 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1647 cryptvars);
1648 if (kIOReturnSuccess != err)
1649 break;
1650
1651 for (page = 0; page < count; page++)
1652 {
1653 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(ppnum), page_size);
1654 if (err)
1655 {
1656 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%d] %x\n", __LINE__, ppnum, err);
1657 break;
1658 }
1659
1660 sum = hibernate_sum(src, page_size);
1661
1662 clock_get_uptime(&startTime);
1663
1664 pageCompressedSize = WKdm_compress ((WK_word*) src, (WK_word*) (src + page_size), PAGE_SIZE_IN_WORDS);
1665
1666 clock_get_uptime(&endTime);
1667 ADD_ABSOLUTETIME(&compTime, &endTime);
1668 SUB_ABSOLUTETIME(&compTime, &startTime);
1669
1670 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1671 pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
1672
1673 if (pageCompressedSize > page_size)
1674 {
1675 // HIBLOG("------------lose: %d\n", pageCompressedSize);
1676 pageCompressedSize = page_size;
1677 }
1678
1679 if (pageCompressedSize != page_size)
1680 data = (src + page_size);
1681 else
1682 data = src;
1683
1684 tag = pageCompressedSize | kIOHibernateTagSignature;
1685
1686 if (pageType)
1687 sum2 += sum;
1688 else
1689 sum1 += sum;
1690
1691 if (needEncryptStart && (ppnum >= atop_32(sectDATAB)))
1692 {
1693 // start encrypting partway into the data about to be written
1694 vars->fileVars->encryptStart = (vars->fileVars->position + AES_BLOCK_SIZE - 1)
1695 & ~(AES_BLOCK_SIZE - 1);
1696 needEncryptStart = false;
1697 }
1698
1699 err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
1700 if (kIOReturnSuccess != err)
1701 break;
1702
1703 err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
1704 if (kIOReturnSuccess != err)
1705 break;
1706
1707 compressedSize += pageCompressedSize;
1708 if (pageCompressedSize)
1709 uncompressedSize += page_size;
1710 ppnum++;
1711 pagesDone++;
1712
1713 if (0 == (8191 & pagesDone))
1714 {
1715 clock_get_uptime(&endTime);
1716 SUB_ABSOLUTETIME(&endTime, &allTime);
1717 absolutetime_to_nanoseconds(endTime, &nsec);
1718 progressStamp = nsec / 750000000ULL;
1719 if (progressStamp != lastProgressStamp)
1720 {
1721 lastProgressStamp = progressStamp;
1722 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
1723 }
1724 }
1725 }
1726 if (kIOReturnSuccess != err)
1727 break;
1728 if (iterDone && !pageType)
1729 {
1730 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
1731 if (kIOReturnSuccess != err)
1732 break;
1733
1734 iterDone = false;
1735 pageType = 1;
1736 iter = 0;
1737 image1Size = vars->fileVars->position;
1738 if (cryptvars)
1739 {
1740 bcopy(&cryptvars->aes_iv[0],
1741 &gIOHibernateCryptWakeContext.aes_iv[0],
1742 sizeof(cryptvars->aes_iv));
1743 cryptvars = &gIOHibernateCryptWakeContext;
1744 }
1745 HIBLOG("image1Size %qd\n", image1Size);
1746 }
1747 }
1748 while (!iterDone);
1749 if (kIOReturnSuccess != err)
1750 break;
1751 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
1752 if (kIOReturnSuccess != err)
1753 break;
1754
1755 // Header:
1756
1757 header->imageSize = vars->fileVars->position;
1758 header->image1Size = image1Size;
1759 header->bitmapSize = bitmap_size;
1760 header->pageCount = pageCount;
1761 header->encryptStart = vars->fileVars->encryptStart;
1762
1763 header->restore1Sum = restore1Sum;
1764 header->image1Sum = sum1;
1765 header->image2Sum = sum2;
1766
1767 count = vars->fileExtents->getLength();
1768 if (count > sizeof(header->fileExtentMap))
1769 {
1770 header->fileExtentMapSize = count;
1771 count = sizeof(header->fileExtentMap);
1772 }
1773 else
1774 header->fileExtentMapSize = sizeof(header->fileExtentMap);
1775 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
1776
1777 IOPolledFileSeek(vars->fileVars, 0);
1778 err = IOPolledFileWrite(vars->fileVars,
1779 (uint8_t *) header, sizeof(IOHibernateImageHeader),
1780 cryptvars);
1781 if (kIOReturnSuccess != err)
1782 break;
1783 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
1784 if (kIOReturnSuccess != err)
1785 break;
1786 err = IOHibernatePollerIODone(vars->fileVars);
1787 if (kIOReturnSuccess != err)
1788 break;
1789 }
1790 while (false);
1791
1792 clock_get_uptime(&endTime);
1793 SUB_ABSOLUTETIME(&endTime, &allTime);
1794 absolutetime_to_nanoseconds(endTime, &nsec);
1795 HIBLOG("all time: %qd ms, ",
1796 nsec / 1000000ULL);
1797
1798 absolutetime_to_nanoseconds(compTime, &nsec);
1799 HIBLOG("comp time: %qd ms, ",
1800 nsec / 1000000ULL);
1801
1802 absolutetime_to_nanoseconds(decoTime, &nsec);
1803 HIBLOG("deco time: %qd ms, ",
1804 nsec / 1000000ULL);
1805
1806 HIBLOG("\nimage %qd, uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
1807 header->imageSize,
1808 uncompressedSize, atop_32(uncompressedSize), compressedSize,
1809 (int) ((compressedSize * 100ULL) / uncompressedSize),
1810 sum1, sum2);
1811
1812 if (pollerOpen)
1813 IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
1814
1815 HIBLOG("hibernate_write_image done(%x)\n", err);
1816
1817 // should we come back via regular wake, set the state in memory.
1818 gIOHibernateState = kIOHibernateStateInactive;
1819
1820 if ((kIOReturnSuccess == err) && !(kIOHibernateModeSleep & gIOHibernateMode))
1821 return (true /* power down */ );
1822 else
1823 return (false /* sleep */ );
1824 }
1825
1826 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1827
1828 DECLARE_IOHIBERNATEPROGRESSALPHA
1829
1830 static void
1831 ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
1832 {
1833 uint32_t rowBytes, pixelShift;
1834 uint32_t x, y;
1835 int32_t blob, lastBlob;
1836 uint32_t alpha, in, color, result;
1837 uint8_t * out;
1838 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1839
1840 pixelShift = display->depth >> 4;
1841 if (pixelShift < 1)
1842 return;
1843
1844 rowBytes = display->rowBytes;
1845
1846 screen += ((display->width
1847 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1848 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1849
1850 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
1851
1852 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
1853
1854 for (y = 0; y < kIOHibernateProgressHeight; y++)
1855 {
1856 out = screen + y * rowBytes;
1857 for (blob = firstBlob; blob <= lastBlob; blob++)
1858 {
1859 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
1860 for (x = 0; x < kIOHibernateProgressWidth; x++)
1861 {
1862 alpha = gIOHibernateProgressAlpha[y][x];
1863 result = color;
1864 if (alpha)
1865 {
1866 if (0xff != alpha)
1867 {
1868 in = display->progressSaveUnder[blob][saveindex[blob]++];
1869 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
1870 }
1871 if (1 == pixelShift)
1872 {
1873 result >>= 3;
1874 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1875 }
1876 else
1877 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1878 }
1879 out += (1 << pixelShift);
1880 }
1881 out += (kIOHibernateProgressSpacing << pixelShift);
1882 }
1883 }
1884 }
1885
1886 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1887
1888 extern "C" void
1889 hibernate_machine_init(void)
1890 {
1891 IOReturn err;
1892 uint32_t sum;
1893 uint32_t pagesDone;
1894 AbsoluteTime allTime, endTime;
1895 uint64_t nsec;
1896 uint32_t lastProgressStamp = 0;
1897 uint32_t progressStamp;
1898 uint64_t progressZeroPosition = 0;
1899 uint32_t blob, lastBlob = (uint32_t) -1L;
1900 hibernate_cryptvars_t * cryptvars = 0;
1901
1902 IOHibernateVars * vars = &gIOHibernateVars;
1903
1904 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
1905 return;
1906
1907 if ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)
1908 hibernate_page_list_discard(vars->page_list);
1909
1910
1911 sum = gIOHibernateCurrentHeader->actualImage1Sum;
1912 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
1913
1914 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, image1Size %qx, conflictCount %d, nextFree %x\n",
1915 gIOHibernateState, pagesDone, sum, gIOHibernateCurrentHeader->image1Size,
1916 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
1917
1918 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
1919 {
1920 HIBLOG("regular wake\n");
1921 return;
1922 }
1923
1924 HIBPRINT("diag %x %x %x %x\n",
1925 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
1926 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
1927
1928 HIBPRINT("video %x %d %d %d\n",
1929 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
1930 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height);
1931
1932 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)
1933 {
1934 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
1935 * gIOHibernateGraphicsInfo->rowBytes);
1936 IOMapPages(kernel_map,
1937 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
1938 vars->videoMapSize, kIOMapInhibitCache );
1939 }
1940
1941 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();;
1942 uint32_t decoOffset;
1943
1944 clock_get_uptime(&allTime);
1945
1946 HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
1947 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0);
1948 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
1949
1950 if (gIOHibernateCurrentHeader->previewSize)
1951 progressZeroPosition = gIOHibernateCurrentHeader->previewSize
1952 + gIOHibernateCurrentHeader->fileExtentMapSize
1953 - sizeof(gIOHibernateCurrentHeader->fileExtentMap)
1954 + ptoa_64(gIOHibernateCurrentHeader->restore1PageCount);
1955
1956 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
1957
1958 if (vars->videoMapping)
1959 {
1960 lastBlob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
1961 / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
1962 ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, 0, lastBlob);
1963 }
1964
1965 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
1966 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1967 {
1968 cryptvars = &gIOHibernateCryptWakeContext;
1969 bcopy(&gIOHibernateCryptWakeVars->aes_iv[0],
1970 &cryptvars->aes_iv[0],
1971 sizeof(cryptvars->aes_iv));
1972 }
1973
1974 // kick off the read ahead
1975 vars->fileVars->io = false;
1976 vars->fileVars->bufferHalf = 0;
1977 vars->fileVars->bufferLimit = 0;
1978 vars->fileVars->lastRead = 0;
1979 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
1980
1981 IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
1982 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
1983 // --
1984
1985 HIBLOG("hibernate_machine_init reading\n");
1986
1987 uint32_t * header = (uint32_t *) src;
1988 sum = 0;
1989
1990 do
1991 {
1992 unsigned int count;
1993 unsigned int page;
1994 uint32_t tag;
1995 vm_offset_t ppnum, compressedSize;
1996
1997 IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
1998
1999 ppnum = header[0];
2000 count = header[1];
2001
2002 // HIBPRINT("(%x, %x)\n", ppnum, count);
2003
2004 if (!count)
2005 break;
2006
2007 for (page = 0; page < count; page++)
2008 {
2009 IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2010
2011 compressedSize = kIOHibernateTagLength & tag;
2012 if (!compressedSize)
2013 {
2014 ppnum++;
2015 pagesDone++;
2016 continue;
2017 }
2018
2019 IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2020
2021 if (compressedSize != page_size)
2022 {
2023 decoOffset = page_size;
2024 WKdm_decompress((WK_word*) src, (WK_word*) (src + decoOffset), PAGE_SIZE_IN_WORDS);
2025 }
2026 else
2027 decoOffset = 0;
2028
2029 sum += hibernate_sum((src + decoOffset), page_size);
2030
2031 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2032 if (err)
2033 HIBLOG("IOMemoryDescriptorReadToPhysical [%d] %x\n", ppnum, err);
2034
2035 ppnum++;
2036 pagesDone++;
2037
2038 if (vars->videoMapping && (0 == (255 & pagesDone)))
2039 {
2040 blob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
2041 / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
2042 if (blob != lastBlob)
2043 {
2044 ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, lastBlob, blob);
2045 lastBlob = blob;
2046 }
2047 }
2048
2049 if (0 == (8191 & pagesDone))
2050 {
2051 clock_get_uptime(&endTime);
2052 SUB_ABSOLUTETIME(&endTime, &allTime);
2053 absolutetime_to_nanoseconds(endTime, &nsec);
2054 progressStamp = nsec / 750000000ULL;
2055 if (progressStamp != lastProgressStamp)
2056 {
2057 lastProgressStamp = progressStamp;
2058 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2059 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2060 }
2061 }
2062 }
2063 }
2064 while (true);
2065
2066 gIOHibernateCurrentHeader->actualImage2Sum = sum;
2067
2068 if (vars->fileVars->io)
2069 (void) IOHibernatePollerIODone(vars->fileVars);
2070
2071 err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
2072
2073 if (vars->videoMapping)
2074 ProgressUpdate(gIOHibernateGraphicsInfo,
2075 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2076
2077 clock_get_uptime(&endTime);
2078 SUB_ABSOLUTETIME(&endTime, &allTime);
2079 absolutetime_to_nanoseconds(endTime, &nsec);
2080
2081 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %qd ms\n",
2082 pagesDone, sum, nsec / 1000000ULL);
2083 }
2084
2085 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
mirror of XNU