]> git.saurik.com Git - apple/xnu.git/blob - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blob
? search:


09c22816deda23b04acdc495e2b314b30e65e4aa
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
1 /*
2 * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
11 * file.
12 *
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
20 *
21 * @APPLE_LICENSE_HEADER_END@
22 */
23
24
25 /*
26
27 Sleep:
28
29 - PMRootDomain calls IOHibernateSystemSleep() before system sleep
30 (devices awake, normal execution context)
31 - IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
32 grabs its extents and searches for a polling driver willing to work with that IOMedia.
33 The BSD code makes an ioctl to the storage driver to get the partition base offset to
34 the disk, and other ioctls to get the transfer constraints
35 If successful, the file is written to make sure its initially not bootable (in case of
36 later failure) and nvram set to point to the first block of the file. (Has to be done
37 here so blocking is possible in nvram support).
38 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
39 page out any pages it wants to (currently zero, but probably some percentage of memory).
40 Its assumed just allocating pages will cause the VM system to naturally select the best
41 pages for eviction. It also copies processor flags needed for the restore path and sets
42 a flag in the boot processor proc info.
43 gIOHibernateState = kIOHibernateStateHibernating.
44 - Regular sleep progresses - some drivers may inspect the root domain property
45 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
46 as usual but leaves motherboard I/O on.
47 - Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
48 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
49 all ppc RC bits out of the hash table and caches into the mapping structures.
50 - hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
51 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
52 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
53 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
54 The image header and block list are written. The header includes the second file extent so
55 only the header block is needed to read the file, regardless of filesystem.
56 The kernel section "__HIB" is written uncompressed to the image. This section of code and data
57 (only) is used to decompress the image during wake/boot.
58 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
59 The bitmaps are written to the image.
60 More areas are removed from the bitmaps (after they have been written to the image) - the
61 section "__HIB" pages and interrupt stack.
62 Each wired page is compressed and written and then each non-wired page. Compression and
63 disk writes are in parallel.
64 The image header is written to the start of the file and the polling driver closed.
65 The machine powers down (or sleeps).
66
67 Boot/Wake:
68
69 - BootX sees the boot-image nvram variable containing the device and block number of the image,
70 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
71 - BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
72 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
73 that is in the kernel's __HIB section.
74 - BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
75 hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
76 only code & data in that section is safe to call since all the other wired pages are still
77 compressed in the image.
78 - hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
79 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
80 location directly, and copies those that can't to interim free pages. When the image has been
81 completed, the copies are uncompressed, overwriting the wired image pages.
82 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
83 is used to get pages into place for 64bit.
84 - the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
85 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
86 are removed from the software strutures, and the hash table is reinitialized.
87 - After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
88 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
89 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
90 for the remaining non wired pages.
91 - The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
92 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
93 is closed via bsd.
94
95 Polled Mode I/O:
96
97 IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
98 registry, specifying an object of calls IOPolledInterface.
99
100 Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
101 partition) that the image is going to live, looking for polled interface properties. If it finds
102 one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
103 interfaces found are kept in an ordered list.
104
105 There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
106 few different contexts things happen in:
107
108 - there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
109 up and running) and after wake - this is safe to allocate memory and do anything. The device
110 ignores sleep requests from that point since its a waste of time if it goes to sleep and
111 immediately wakes back up for the image write.
112
113 - there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
114 immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
115 from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
116 used to flush and set the disk to sleep.
117
118 - there is an Open/Close (AfterSleep) pair made around the image read operations that happen
119 immediately after sleep. These can't block or allocate memory. This is happening after the platform
120 expert has woken the low level bits of the system, but most of the I/O system has not. There is only
121 one thread running.
122
123 For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
124 (prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
125 the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
126 Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
127 that is called for the hardware to check for events, and complete the I/O via the callback.
128 The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
129 to restrict I/O ops.
130 */
131
132 #include <sys/systm.h>
133
134 #include <IOKit/IOWorkLoop.h>
135 #include <IOKit/IOCommandGate.h>
136 #include <IOKit/IOTimerEventSource.h>
137 #include <IOKit/IOPlatformExpert.h>
138 #include <IOKit/IOKitDebug.h>
139 #include <IOKit/IOTimeStamp.h>
140 #include <IOKit/pwr_mgt/RootDomain.h>
141 #include <IOKit/pwr_mgt/IOPMPrivate.h>
142 #include <IOKit/IOMessage.h>
143 #include <IOKit/IODeviceTreeSupport.h>
144 #include <IOKit/IOBSD.h>
145 #include "RootDomainUserClient.h"
146 #include <IOKit/pwr_mgt/IOPowerConnection.h>
147 #include "IOPMPowerStateQueue.h"
148 #include <IOKit/IOBufferMemoryDescriptor.h>
149 #include <crypto/aes.h>
150
151 #include <sys/uio.h>
152 #include <sys/conf.h>
153 #include <sys/stat.h>
154 #include <sys/fcntl.h> // (FWRITE, ...)
155 extern "C" {
156 #include <sys/sysctl.h>
157 }
158
159 #include <IOKit/IOHibernatePrivate.h>
160 #include <IOKit/IOPolledInterface.h>
161 #include <IOKit/IONVRAM.h>
162 #include "IOHibernateInternal.h"
163 #include "WKdm.h"
164 #include "IOKitKernelInternal.h"
165
166 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
167
168 OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
169
170 OSMetaClassDefineReservedUnused(IOPolledInterface, 0);
171 OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
172 OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
173 OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
174 OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
175 OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
176 OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
177 OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
178 OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
179 OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
180 OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
181 OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
182 OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
183 OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
184 OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
185 OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
186
187 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
188
189 extern uint32_t gIOHibernateState;
190 uint32_t gIOHibernateMode;
191 static char gIOHibernateBootSignature[256+1];
192 static char gIOHibernateFilename[MAXPATHLEN+1];
193 static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
194 uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
195
196 static IODTNVRAM * gIOOptionsEntry;
197 static IORegistryEntry * gIOChosenEntry;
198
199 static IOPolledFileIOVars gFileVars;
200 static IOHibernateVars gIOHibernateVars;
201 static struct kern_direct_file_io_ref_t * gIOHibernateFileRef;
202 static hibernate_cryptvars_t gIOHibernateCryptWakeContext;
203
204 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
205
206 enum { kXPRamAudioVolume = 8 };
207 enum { kDefaultIOSize = 128 * 1024 };
208 enum { kVideoMapSize = 32 * 1024 * 1024 };
209
210 #ifndef kIOMediaPreferredBlockSizeKey
211 #define kIOMediaPreferredBlockSizeKey "Preferred Block Size"
212 #endif
213
214 #ifndef kIOBootPathKey
215 #define kIOBootPathKey "bootpath"
216 #endif
217 #ifndef kIOSelectedBootDeviceKey
218 #define kIOSelectedBootDeviceKey "boot-device"
219 #endif
220
221
222 enum { kIOHibernateMinPollersNeeded = 2 };
223
224 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
225
226 // copy from phys addr to MD
227
228 static IOReturn
229 IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
230 IOByteCount offset, addr64_t bytes, IOByteCount length)
231 {
232 addr64_t srcAddr = bytes;
233 IOByteCount remaining;
234
235 remaining = length = min(length, md->getLength() - offset);
236 while (remaining) { // (process another target segment?)
237 addr64_t dstAddr64;
238 IOByteCount dstLen;
239
240 dstAddr64 = md->getPhysicalSegment64(offset, &dstLen);
241 if (!dstAddr64)
242 break;
243
244 // Clip segment length to remaining
245 if (dstLen > remaining)
246 dstLen = remaining;
247
248 #if 1
249 bcopy_phys(srcAddr, dstAddr64, dstLen);
250 #else
251 copypv(srcAddr, dstAddr64, dstLen,
252 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
253 #endif
254 srcAddr += dstLen;
255 offset += dstLen;
256 remaining -= dstLen;
257 }
258
259 assert(!remaining);
260
261 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
262 }
263
264 // copy from MD to phys addr
265
266 static IOReturn
267 IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
268 IOByteCount offset, addr64_t bytes, IOByteCount length)
269 {
270 addr64_t dstAddr = bytes;
271 IOByteCount remaining;
272
273 remaining = length = min(length, md->getLength() - offset);
274 while (remaining) { // (process another target segment?)
275 addr64_t srcAddr64;
276 IOByteCount dstLen;
277
278 srcAddr64 = md->getPhysicalSegment64(offset, &dstLen);
279 if (!srcAddr64)
280 break;
281
282 // Clip segment length to remaining
283 if (dstLen > remaining)
284 dstLen = remaining;
285
286 #if 1
287 bcopy_phys(srcAddr64, dstAddr, dstLen);
288 #else
289 copypv(srcAddr, dstAddr64, dstLen,
290 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
291 #endif
292 dstAddr += dstLen;
293 offset += dstLen;
294 remaining -= dstLen;
295 }
296
297 assert(!remaining);
298
299 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
300 }
301
302 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
303
304 void
305 hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
306 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
307 {
308 count += ppnum;
309 switch (kind)
310 {
311 case kIOHibernatePageStateUnwiredSave:
312 // unwired save
313 for (; ppnum < count; ppnum++)
314 {
315 hibernate_page_bitset(page_list, FALSE, ppnum);
316 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
317 }
318 break;
319 case kIOHibernatePageStateWiredSave:
320 // wired save
321 for (; ppnum < count; ppnum++)
322 {
323 hibernate_page_bitset(page_list, FALSE, ppnum);
324 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
325 }
326 break;
327 case kIOHibernatePageStateFree:
328 // free page
329 for (; ppnum < count; ppnum++)
330 {
331 hibernate_page_bitset(page_list, TRUE, ppnum);
332 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
333 }
334 break;
335 default:
336 panic("hibernate_set_page_state");
337 }
338 }
339
340 static vm_offset_t
341 hibernate_page_list_iterate(hibernate_page_list_t * list,
342 void ** iterator, vm_offset_t * ppnum)
343 {
344 uint32_t count, idx;
345
346 idx = (uint32_t) *iterator;
347
348 if (!idx)
349 idx = hibernate_page_list_count(list, TRUE, idx);
350
351 *ppnum = idx;
352 count = hibernate_page_list_count(list, FALSE, idx);
353 idx += count;
354 idx += hibernate_page_list_count(list, TRUE, idx);
355 *iterator = (void *) idx;
356
357 return (count);
358 }
359
360 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
361
362 static IOReturn
363 IOHibernatePollerProbe(IOPolledFileIOVars * vars, IOService * target)
364 {
365 IOReturn err = kIOReturnError;
366 int32_t idx;
367 IOPolledInterface * poller;
368
369 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
370 {
371 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
372 err = poller->probe(target);
373 if (err)
374 {
375 HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
376 break;
377 }
378 }
379
380 return (err);
381 }
382
383 static IOReturn
384 IOHibernatePollerOpen(IOPolledFileIOVars * vars, uint32_t state, IOMemoryDescriptor * md)
385 {
386 IOReturn err = kIOReturnError;
387 int32_t idx;
388 IOPolledInterface * poller;
389
390 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
391 {
392 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
393 err = poller->open(state, md);
394 if (err)
395 {
396 HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
397 break;
398 }
399 }
400
401 return (err);
402 }
403
404 static IOReturn
405 IOHibernatePollerClose(IOPolledFileIOVars * vars, uint32_t state)
406 {
407 IOReturn err = kIOReturnError;
408 int32_t idx;
409 IOPolledInterface * poller;
410
411 for (idx = 0;
412 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
413 idx++)
414 {
415 err = poller->close(state);
416 if (err)
417 HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
418 }
419
420 return (err);
421 }
422
423 static void
424 IOHibernatePollerIOComplete(void * target,
425 void * parameter,
426 IOReturn status,
427 UInt64 actualByteCount)
428 {
429 IOPolledFileIOVars * vars = (IOPolledFileIOVars *) parameter;
430
431 vars->ioStatus = status;
432 }
433
434 static IOReturn
435 IOHibernatePollerIO(IOPolledFileIOVars * vars,
436 uint32_t operation, uint32_t bufferOffset,
437 uint64_t deviceOffset, uint64_t length)
438 {
439
440 IOReturn err = kIOReturnError;
441 IOPolledInterface * poller;
442 IOPolledCompletion completion;
443
444 completion.target = 0;
445 completion.action = &IOHibernatePollerIOComplete;
446 completion.parameter = vars;
447
448 vars->ioStatus = -1;
449
450 poller = (IOPolledInterface *) vars->pollers->getObject(0);
451 err = poller->startIO(operation, bufferOffset, deviceOffset + vars->block0, length, completion);
452 if (err)
453 HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
454
455 return (err);
456 }
457
458 static IOReturn
459 IOHibernatePollerIODone(IOPolledFileIOVars * vars)
460 {
461 IOReturn err = kIOReturnError;
462 int32_t idx;
463 IOPolledInterface * poller;
464
465 while (-1 == vars->ioStatus)
466 {
467 for (idx = 0;
468 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
469 idx++)
470 {
471 err = poller->checkForWork();
472 if (err)
473 HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
474 }
475 }
476
477 if (kIOReturnSuccess != vars->ioStatus)
478 HIBLOG("IOPolledInterface::ioStatus 0x%x\n", vars->ioStatus);
479
480 return (vars->ioStatus);
481 }
482
483 IOReturn
484 IOPolledInterface::checkAllForWork(void)
485 {
486 IOReturn err = kIOReturnNotReady;
487 int32_t idx;
488 IOPolledInterface * poller;
489
490 IOHibernateVars * vars = &gIOHibernateVars;
491
492 if (!vars->fileVars || !vars->fileVars->pollers)
493 return (err);
494
495 for (idx = 0;
496 (poller = (IOPolledInterface *) vars->fileVars->pollers->getObject(idx));
497 idx++)
498 {
499 err = poller->checkForWork();
500 if (err)
501 HIBLOG("IOPolledInterface::checkAllForWork[%d] 0x%x\n", idx, err);
502 }
503
504 return (err);
505 }
506
507 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
508
509 struct _OpenFileContext
510 {
511 OSData * extents;
512 uint64_t size;
513 };
514
515 static void
516 file_extent_callback(void * ref, uint64_t start, uint64_t length)
517 {
518 _OpenFileContext * ctx = (_OpenFileContext *) ref;
519 IOPolledFileExtent extent;
520
521 extent.start = start;
522 extent.length = length;
523
524 ctx->extents->appendBytes(&extent, sizeof(extent));
525 ctx->size += length;
526 }
527
528 IOReturn
529 IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
530 IOPolledFileIOVars ** fileVars, OSData ** fileExtents,
531 OSData ** imagePath)
532 {
533 IOReturn err = kIOReturnError;
534 IOPolledFileIOVars * vars;
535 _OpenFileContext ctx;
536 OSData * extentsData;
537 OSNumber * num;
538 IORegistryEntry * part = 0;
539 OSDictionary * matching;
540 OSIterator * iter;
541 dev_t hibernate_image_dev;
542 uint64_t maxiobytes;
543
544 vars = &gFileVars;
545 do
546 {
547 HIBLOG("sizeof(IOHibernateImageHeader) == %ld\n", sizeof(IOHibernateImageHeader));
548 if (sizeof(IOHibernateImageHeader) != 512)
549 continue;
550
551 vars->io = false;
552 vars->buffer = (uint8_t *) ioBuffer->getBytesNoCopy();
553 vars->bufferHalf = 0;
554 vars->bufferOffset = 0;
555 vars->bufferSize = ioBuffer->getLength() >> 1;
556
557 extentsData = OSData::withCapacity(32);
558
559 ctx.extents = extentsData;
560 ctx.size = 0;
561 vars->fileRef = kern_open_file_for_direct_io(filename,
562 &file_extent_callback, &ctx,
563 &hibernate_image_dev,
564 &vars->block0,
565 &maxiobytes);
566 if (!vars->fileRef)
567 {
568 err = kIOReturnNoSpace;
569 break;
570 }
571 HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx\n", filename, ctx.size,
572 vars->block0, maxiobytes);
573 if (ctx.size < 1*1024*1024) // check against image size estimate!
574 {
575 err = kIOReturnNoSpace;
576 break;
577 }
578
579 if (maxiobytes < vars->bufferSize)
580 vars->bufferSize = maxiobytes;
581
582 vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
583
584 matching = IOService::serviceMatching("IOMedia");
585 num = OSNumber::withNumber(major(hibernate_image_dev), 32);
586 matching->setObject(kIOBSDMajorKey, num);
587 num->release();
588 num = OSNumber::withNumber(minor(hibernate_image_dev), 32);
589 matching->setObject(kIOBSDMinorKey, num);
590 num->release();
591 iter = IOService::getMatchingServices(matching);
592 matching->release();
593 if (iter)
594 {
595 part = (IORegistryEntry *) iter->getNextObject();
596 part->retain();
597 iter->release();
598 }
599
600 int minor, major;
601 IORegistryEntry * next;
602 IORegistryEntry * child;
603 OSData * data;
604
605 num = (OSNumber *) part->getProperty(kIOBSDMajorKey);
606 if (!num)
607 break;
608 major = num->unsigned32BitValue();
609 num = (OSNumber *) part->getProperty(kIOBSDMinorKey);
610 if (!num)
611 break;
612 minor = num->unsigned32BitValue();
613
614 hibernate_image_dev = makedev(major, minor);
615
616 vars->pollers = OSArray::withCapacity(4);
617 if (!vars->pollers)
618 break;
619
620 vars->blockSize = 512;
621 next = part;
622 do
623 {
624 IOPolledInterface * poller;
625 if ((poller = OSDynamicCast(IOPolledInterface, next->getProperty(kIOPolledInterfaceSupportKey))))
626 vars->pollers->setObject(poller);
627 if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey))))
628 vars->blockSize = num->unsigned32BitValue();
629 child = next;
630 }
631 while ((next = child->getParentEntry(gIOServicePlane))
632 && child->isParent(next, gIOServicePlane, true));
633
634 HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
635 major, minor, vars->blockSize, vars->pollers->getCount());
636 if (vars->pollers->getCount() < kIOHibernateMinPollersNeeded)
637 continue;
638
639 err = IOHibernatePollerProbe(vars, (IOService *) part);
640 if (kIOReturnSuccess != err)
641 break;
642
643 err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer);
644 if (kIOReturnSuccess != err)
645 break;
646
647 *fileVars = vars;
648 *fileExtents = extentsData;
649
650 // make imagePath
651 char str1[256];
652 char str2[24];
653 int len = sizeof(str1);
654
655 if ((extentsData->getLength() >= sizeof(IOPolledFileExtent))
656 && part->getPath(str1, &len, gIODTPlane))
657 {
658 // (strip the plane name)
659 char * tail = strchr(str1, ':');
660 if (!tail)
661 tail = str1 - 1;
662 data = OSData::withBytes(tail + 1, strlen(tail + 1));
663 sprintf(str2, ",%qx", vars->extentMap[0]);
664 data->appendBytes(str2, strlen(str2));
665 *imagePath = data;
666 }
667 }
668 while (false);
669
670 if (kIOReturnSuccess != err)
671 {
672 HIBLOG("error 0x%x opening hibernation file\n", err);
673 if (vars->fileRef)
674 kern_close_file_for_direct_io(vars->fileRef);
675 }
676
677 if (part)
678 part->release();
679
680 return (err);
681 }
682
683 IOReturn
684 IOPolledFileClose( IOPolledFileIOVars * vars )
685 {
686 if (vars->pollers)
687 {
688 IOHibernatePollerClose(vars, kIOPolledPostflightState);
689 vars->pollers->release();
690 }
691
692 gIOHibernateFileRef = vars->fileRef;
693
694 bzero(vars, sizeof(IOPolledFileIOVars));
695
696 return (kIOReturnSuccess);
697 }
698
699 static IOReturn
700 IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
701 {
702 IOPolledFileExtent * extentMap;
703
704 extentMap = vars->extentMap;
705
706 vars->position = position;
707
708 while (position >= extentMap->length)
709 {
710 position -= extentMap->length;
711 extentMap++;
712 }
713
714 vars->currentExtent = extentMap;
715 vars->extentRemaining = extentMap->length - position;
716 vars->extentPosition = vars->position - position;
717
718 if (vars->bufferSize <= vars->extentRemaining)
719 vars->bufferLimit = vars->bufferSize;
720 else
721 vars->bufferLimit = vars->extentRemaining;
722
723 return (kIOReturnSuccess);
724 }
725
726 static IOReturn
727 IOPolledFileWrite(IOPolledFileIOVars * vars,
728 const uint8_t * bytes, IOByteCount size,
729 hibernate_cryptvars_t * cryptvars)
730 {
731 IOReturn err = kIOReturnSuccess;
732 IOByteCount copy;
733 bool flush = false;
734
735 do
736 {
737 if (!bytes && !size)
738 {
739 // seek to end of block & flush
740 size = vars->position & (vars->blockSize - 1);
741 if (size)
742 size = vars->blockSize - size;
743 flush = true;
744 // use some garbage for the fill
745 bytes = vars->buffer + vars->bufferOffset;
746 }
747
748 copy = vars->bufferLimit - vars->bufferOffset;
749 if (copy > size)
750 copy = size;
751 else
752 flush = true;
753
754 if (bytes)
755 {
756 bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
757 bytes += copy;
758 }
759 else
760 bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
761
762 size -= copy;
763 vars->bufferOffset += copy;
764 vars->position += copy;
765
766 if (flush && vars->bufferOffset)
767 {
768 uint64_t offset = (vars->position - vars->bufferOffset
769 - vars->extentPosition + vars->currentExtent->start);
770 uint32_t length = (vars->bufferOffset);
771
772 if (cryptvars && vars->encryptStart && (vars->position > vars->encryptStart))
773 {
774 uint32_t encryptLen, encryptStart;
775 encryptLen = vars->position - vars->encryptStart;
776 if (encryptLen > length)
777 encryptLen = length;
778 encryptStart = length - encryptLen;
779
780 // encrypt the buffer
781 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
782 &cryptvars->aes_iv[0],
783 encryptLen / AES_BLOCK_SIZE,
784 vars->buffer + vars->bufferHalf + encryptStart,
785 &cryptvars->ctx.encrypt);
786 // save initial vector for following encrypts
787 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
788 &cryptvars->aes_iv[0],
789 AES_BLOCK_SIZE);
790 }
791
792 if (vars->io)
793 {
794 err = IOHibernatePollerIODone(vars);
795 if (kIOReturnSuccess != err)
796 break;
797 }
798
799 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
800 //if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
801
802 err = IOHibernatePollerIO(vars, kIOPolledWrite, vars->bufferHalf, offset, length);
803 if (kIOReturnSuccess != err)
804 break;
805 vars->io = true;
806
807 vars->extentRemaining -= vars->bufferOffset;
808 if (!vars->extentRemaining)
809 {
810 vars->currentExtent++;
811 vars->extentRemaining = vars->currentExtent->length;
812 vars->extentPosition = vars->position;
813 if (!vars->extentRemaining)
814 {
815 err = kIOReturnOverrun;
816 break;
817 }
818 }
819
820 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
821 vars->bufferOffset = 0;
822 if (vars->bufferSize <= vars->extentRemaining)
823 vars->bufferLimit = vars->bufferSize;
824 else
825 vars->bufferLimit = vars->extentRemaining;
826
827 flush = false;
828 }
829 }
830 while (size);
831
832 return (err);
833 }
834
835 static IOReturn
836 IOPolledFileRead(IOPolledFileIOVars * vars,
837 uint8_t * bytes, IOByteCount size,
838 hibernate_cryptvars_t * cryptvars)
839 {
840 IOReturn err = kIOReturnSuccess;
841 IOByteCount copy;
842
843 // bytesWritten += size;
844
845 do
846 {
847 copy = vars->bufferLimit - vars->bufferOffset;
848 if (copy > size)
849 copy = size;
850
851 if (bytes)
852 {
853 bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
854 bytes += copy;
855 }
856 size -= copy;
857 vars->bufferOffset += copy;
858 // vars->position += copy;
859
860 if (vars->bufferOffset == vars->bufferLimit)
861 {
862 if (vars->io)
863 {
864 err = IOHibernatePollerIODone(vars);
865 if (kIOReturnSuccess != err)
866 break;
867 }
868 else
869 cryptvars = 0;
870
871 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
872
873 vars->position += vars->lastRead;
874 vars->extentRemaining -= vars->lastRead;
875 vars->bufferLimit = vars->lastRead;
876
877 if (!vars->extentRemaining)
878 {
879 vars->currentExtent++;
880 vars->extentRemaining = vars->currentExtent->length;
881 vars->extentPosition = vars->position;
882 if (!vars->extentRemaining)
883 {
884 err = kIOReturnOverrun;
885 break;
886 }
887 }
888
889 if (vars->extentRemaining <= vars->bufferSize)
890 vars->lastRead = vars->extentRemaining;
891 else
892 vars->lastRead = vars->bufferSize;
893
894 uint64_t offset = (vars->position
895 - vars->extentPosition + vars->currentExtent->start);
896 uint64_t length = (vars->lastRead);
897
898 //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
899
900 err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
901 if (kIOReturnSuccess != err)
902 break;
903 vars->io = true;
904
905 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
906 vars->bufferOffset = 0;
907
908 if (cryptvars)
909 {
910 uint8_t thisVector[AES_BLOCK_SIZE];
911 // save initial vector for following decrypts
912 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
913 bcopy(vars->buffer + vars->bufferHalf + vars->lastRead - AES_BLOCK_SIZE,
914 &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
915 // decrypt the buffer
916 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
917 &thisVector[0],
918 vars->lastRead / AES_BLOCK_SIZE,
919 vars->buffer + vars->bufferHalf,
920 &cryptvars->ctx.decrypt);
921 }
922 }
923 }
924 while (size);
925
926 return (err);
927 }
928
929 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
930
931 IOReturn
932 IOHibernateSystemSleep(void)
933 {
934 IOReturn err;
935 OSData * data;
936 OSObject * obj;
937 OSString * str;
938 OSNumber * num;
939
940 IOHibernateVars * vars = &gIOHibernateVars;
941
942 if (vars->fileVars && vars->fileVars->fileRef)
943 // already on the way down
944 return (kIOReturnSuccess);
945
946 gIOHibernateState = kIOHibernateStateInactive;
947
948 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateModeKey)))
949 {
950 if ((num = OSDynamicCast(OSNumber, obj)))
951 gIOHibernateMode = num->unsigned32BitValue();
952 if (kIOHibernateModeSleep & gIOHibernateMode)
953 // default to discard clean for safe sleep
954 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
955 | kIOHibernateModeDiscardCleanActive);
956
957 obj->release();
958 }
959 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeRatioKey)))
960 {
961 if ((num = OSDynamicCast(OSNumber, obj)))
962 gIOHibernateFreeRatio = num->unsigned32BitValue();
963 obj->release();
964 }
965 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeTimeKey)))
966 {
967 if ((num = OSDynamicCast(OSNumber, obj)))
968 gIOHibernateFreeTime = num->unsigned32BitValue();
969 obj->release();
970 }
971 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
972 {
973 if ((str = OSDynamicCast(OSString, obj)))
974 strcpy(gIOHibernateFilename, str->getCStringNoCopy());
975 obj->release();
976 }
977
978 if (!gIOHibernateMode || !gIOHibernateFilename[0])
979 return (kIOReturnUnsupported);
980
981 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
982
983 do
984 {
985 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(0, 4 * page_size, page_size);
986 vars->ioBuffer = IOBufferMemoryDescriptor::withOptions(0, 2 * kDefaultIOSize, page_size);
987
988 if (!vars->srcBuffer || !vars->ioBuffer)
989 {
990 err = kIOReturnNoMemory;
991 break;
992 }
993
994 err = IOPolledFileOpen(gIOHibernateFilename, vars->ioBuffer,
995 &vars->fileVars, &vars->fileExtents, &data);
996 if (KERN_SUCCESS != err)
997 {
998 HIBLOG("IOPolledFileOpen(%x)\n", err);
999 break;
1000 }
1001 if (vars->fileVars->fileRef)
1002 {
1003 // invalidate the image file
1004 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1005 int err = kern_write_file(vars->fileVars->fileRef, 0,
1006 (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1007 if (KERN_SUCCESS != err)
1008 HIBLOG("kern_write_file(%d)\n", err);
1009 }
1010
1011 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1012
1013 boolean_t encryptedswap;
1014 err = hibernate_setup(gIOHibernateCurrentHeader,
1015 gIOHibernateFreeRatio, gIOHibernateFreeTime,
1016 &vars->page_list, &vars->page_list_wired, &encryptedswap);
1017 if (KERN_SUCCESS != err)
1018 {
1019 HIBLOG("hibernate_setup(%d)\n", err);
1020 break;
1021 }
1022
1023 if (encryptedswap)
1024 gIOHibernateMode ^= kIOHibernateModeEncrypt;
1025
1026 vars->videoAllocSize = kVideoMapSize;
1027 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
1028 vars->videoMapping = 0;
1029
1030 // generate crypt keys
1031 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
1032 vars->wiredCryptKey[i] = random();
1033 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
1034 vars->cryptKey[i] = random();
1035
1036 // set nvram
1037
1038 IORegistryEntry * regEntry;
1039 if (!gIOOptionsEntry)
1040 {
1041 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
1042 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
1043 if (regEntry && !gIOOptionsEntry)
1044 regEntry->release();
1045 }
1046 if (!gIOChosenEntry)
1047 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1048
1049 if (gIOOptionsEntry)
1050 {
1051 const OSSymbol * sym;
1052 size_t len;
1053 char valueString[16];
1054
1055 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1056 if (sym)
1057 {
1058 gIOOptionsEntry->setProperty(sym, data);
1059 sym->release();
1060 }
1061 data->release();
1062
1063 vars->saveBootDevice = gIOOptionsEntry->copyProperty(kIOSelectedBootDeviceKey);
1064 if (gIOChosenEntry)
1065 {
1066 OSData * bootDevice = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOBootPathKey));
1067 if (bootDevice)
1068 {
1069 sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
1070 OSString * str2 = OSString::withCStringNoCopy((const char *) bootDevice->getBytesNoCopy());
1071 if (sym && str2)
1072 gIOOptionsEntry->setProperty(sym, str2);
1073 if (sym)
1074 sym->release();
1075 if (str2)
1076 str2->release();
1077 }
1078
1079 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMemorySignatureKey));
1080 if (data)
1081 {
1082 vars->haveFastBoot = true;
1083
1084 len = sprintf(valueString, "0x%lx", *((UInt32 *)data->getBytesNoCopy()));
1085 data = OSData::withBytes(valueString, len + 1);
1086 sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
1087 if (sym && data)
1088 gIOOptionsEntry->setProperty(sym, data);
1089 if (sym)
1090 sym->release();
1091 if (data)
1092 data->release();
1093 }
1094 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
1095 if (data)
1096 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
1097 }
1098
1099 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1100 {
1101 data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1102 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
1103 if (sym && data)
1104 gIOOptionsEntry->setProperty(sym, data);
1105 if (sym)
1106 sym->release();
1107 if (data)
1108 data->release();
1109 if (gIOHibernateBootSignature[0])
1110 {
1111 data = OSData::withCapacity(16);
1112 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
1113 if (sym && data)
1114 {
1115 char c;
1116 uint8_t value;
1117 for (uint32_t i = 0; (c = gIOHibernateBootSignature[i]); i++)
1118 {
1119 if (c >= 'a')
1120 c -= 'a' - 10;
1121 else if (c >= 'A')
1122 c -= 'A' - 10;
1123 else if (c >= '0')
1124 c -= '0';
1125 else
1126 continue;
1127 value = (value << 4) | c;
1128 if (i & 1)
1129 data->appendBytes(&value, sizeof(value));
1130 }
1131 gIOOptionsEntry->setProperty(sym, data);
1132 }
1133 if (sym)
1134 sym->release();
1135 if (data)
1136 data->release();
1137 }
1138 }
1139
1140 if (!vars->haveFastBoot)
1141 {
1142 // set boot volume to zero
1143 IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
1144 if (platform && (kIOReturnSuccess == platform->readXPRAM(kXPRamAudioVolume,
1145 &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume))))
1146 {
1147 uint8_t newVolume;
1148 newVolume = vars->saveBootAudioVolume & 0xf8;
1149 platform->writeXPRAM(kXPRamAudioVolume,
1150 &newVolume, sizeof(newVolume));
1151 }
1152 }
1153 }
1154 // --
1155
1156 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
1157 gIOHibernateState = kIOHibernateStateHibernating;
1158 }
1159 while (false);
1160
1161 return (err);
1162 }
1163
1164 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1165
1166 IOReturn
1167 IOHibernateSystemHasSlept(void)
1168 {
1169 IOHibernateVars * vars = &gIOHibernateVars;
1170
1171 if ((vars->previewData = OSDynamicCast(OSData,
1172 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewBufferKey))))
1173 {
1174 vars->previewBuffer = IOMemoryDescriptor::withAddress(
1175 (void *) vars->previewData->getBytesNoCopy(),
1176 vars->previewData->getLength(),
1177 kIODirectionInOut);
1178
1179 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
1180 {
1181 vars->previewBuffer->release();
1182 vars->previewBuffer = 0;
1183 }
1184 if (!vars->previewBuffer)
1185 vars->previewData = 0;
1186 }
1187 if (gIOOptionsEntry)
1188 gIOOptionsEntry->sync();
1189
1190 return (kIOReturnSuccess);
1191 }
1192
1193 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1194
1195 IOReturn
1196 IOHibernateSystemWake(void)
1197 {
1198 IOHibernateVars * vars = &gIOHibernateVars;
1199
1200 hibernate_teardown(vars->page_list, vars->page_list_wired);
1201
1202 if (vars->videoMapping)
1203 {
1204 if (vars->videoMapSize)
1205 // remove mappings
1206 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1207 if (vars->videoAllocSize)
1208 // dealloc range
1209 kmem_free(kernel_map, trunc_page_32(vars->videoMapping), vars->videoAllocSize);
1210 }
1211
1212 if (vars->previewBuffer)
1213 {
1214 vars->previewBuffer->release();
1215 vars->previewBuffer = 0;
1216 }
1217
1218 if (vars->fileVars)
1219 {
1220 IOPolledFileClose(vars->fileVars);
1221 }
1222
1223 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1224
1225 OSData * data = OSData::withCapacity(4);
1226 if (gIOOptionsEntry && data)
1227 {
1228 const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1229 if (sym)
1230 {
1231 gIOOptionsEntry->setProperty(sym, data);
1232 sym->release();
1233 }
1234 sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
1235 if (sym)
1236 {
1237 if (vars->saveBootDevice)
1238 {
1239 gIOOptionsEntry->setProperty(sym, vars->saveBootDevice);
1240 vars->saveBootDevice->release();
1241 }
1242 sym->release();
1243 }
1244 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
1245 if (sym)
1246 {
1247 gIOOptionsEntry->setProperty(sym, data);
1248 sym->release();
1249 }
1250 sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
1251 if (sym)
1252 {
1253 gIOOptionsEntry->removeProperty(sym);
1254 sym->release();
1255 }
1256 }
1257 if (data)
1258 data->release();
1259
1260 if (gIOOptionsEntry)
1261 {
1262 if (!vars->haveFastBoot)
1263 {
1264 // reset boot audio volume
1265 IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
1266 if (platform)
1267 platform->writeXPRAM(kXPRamAudioVolume,
1268 &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume));
1269 }
1270
1271 // sync now to hardware if the booter has not
1272 if (kIOHibernateStateInactive == gIOHibernateState)
1273 gIOOptionsEntry->sync();
1274 else
1275 // just sync the variables in case a later panic syncs nvram (it won't sync variables)
1276 gIOOptionsEntry->syncOFVariables();
1277 }
1278
1279 if (vars->srcBuffer)
1280 vars->srcBuffer->release();
1281 if (vars->ioBuffer)
1282 vars->ioBuffer->release();
1283 if (vars->fileExtents)
1284 vars->fileExtents->release();
1285
1286 bzero(vars, sizeof(*vars));
1287
1288 // gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1289
1290 return (kIOReturnSuccess);
1291 }
1292
1293 IOReturn
1294 IOHibernateSystemPostWake(void)
1295 {
1296 if (gIOHibernateFileRef)
1297 {
1298 kern_close_file_for_direct_io(gIOHibernateFileRef);
1299 gIOHibernateFileRef = 0;
1300 }
1301 return (kIOReturnSuccess);
1302 }
1303
1304 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1305
1306 void
1307 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1308 {
1309 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
1310 if (data)
1311 {
1312 rootDomain->setProperty(kIOHibernateStateKey, data);
1313 data->release();
1314 }
1315
1316 if (PE_parse_boot_arg("hfile", gIOHibernateFilename))
1317 gIOHibernateMode = kIOHibernateModeOn;
1318 else
1319 gIOHibernateFilename[0] = 0;
1320
1321 static SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
1322 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
1323 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
1324 sysctl_register_oid(&sysctl__kern_hibernatefile);
1325
1326 static SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
1327 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
1328 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
1329 sysctl_register_oid(&sysctl__kern_bootsignature);
1330
1331 static SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
1332 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
1333 &gIOHibernateMode, 0, "");
1334 sysctl_register_oid(&sysctl__kern_hibernatemode);
1335 }
1336
1337 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1338
1339 static void
1340 hibernate_setup_for_wake(void)
1341 {
1342 #if __ppc__
1343 // go slow (state needed for wake)
1344 ml_set_processor_speed(1);
1345 #endif
1346 }
1347
1348 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1349
1350 extern "C" boolean_t
1351 hibernate_write_image(void)
1352 {
1353 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
1354 IOHibernateVars * vars = &gIOHibernateVars;
1355 IOPolledFileExtent * fileExtents;
1356
1357 uint32_t pageCount, pagesDone;
1358 IOReturn err;
1359 vm_offset_t ppnum;
1360 IOItemCount page, count;
1361 uint8_t * src;
1362 uint8_t * data;
1363 IOByteCount pageCompressedSize;
1364 uint64_t compressedSize, uncompressedSize;
1365 uint64_t image1Size = 0;
1366 uint32_t bitmap_size;
1367 bool iterDone, pollerOpen, needEncryptStart;
1368 uint32_t restore1Sum, sum, sum1, sum2;
1369 uint32_t tag;
1370 uint32_t pageType;
1371 uint32_t pageAndCount[2];
1372
1373 AbsoluteTime startTime, endTime;
1374 AbsoluteTime allTime, compTime, decoTime;
1375 uint64_t nsec;
1376 uint32_t lastProgressStamp = 0;
1377 uint32_t progressStamp;
1378
1379 hibernate_cryptvars_t _cryptvars;
1380 hibernate_cryptvars_t * cryptvars = 0;
1381
1382 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
1383 return (false /* sleep */ );
1384
1385 restore1Sum = sum1 = sum2 = 0;
1386
1387 // encryption data. "iv" is the "initial vector".
1388 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1389 {
1390 static const unsigned char first_iv[AES_BLOCK_SIZE]
1391 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
1392 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
1393
1394 cryptvars = &gIOHibernateCryptWakeContext;
1395 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
1396 aes_encrypt_key(vars->cryptKey,
1397 kIOHibernateAESKeySize,
1398 &cryptvars->ctx.encrypt);
1399 aes_decrypt_key(vars->cryptKey,
1400 kIOHibernateAESKeySize,
1401 &cryptvars->ctx.decrypt);
1402
1403 cryptvars = &_cryptvars;
1404 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
1405 aes_encrypt_key(vars->wiredCryptKey,
1406 kIOHibernateAESKeySize,
1407 &cryptvars->ctx.encrypt);
1408
1409 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1410 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1411 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
1412 bzero(gIOHibernateCryptWakeVars, sizeof(hibernate_cryptwakevars_t));
1413 }
1414
1415 hibernate_setup_for_wake();
1416
1417 hibernate_page_list_setall(vars->page_list,
1418 vars->page_list_wired,
1419 &pageCount);
1420
1421 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
1422
1423 fileExtents = (IOPolledFileExtent *) vars->fileExtents->getBytesNoCopy();
1424
1425 #if 0
1426 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
1427 for (page = 0; page < count; page++)
1428 {
1429 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
1430 fileExtents[page].start, fileExtents[page].length,
1431 fileExtents[page].start + fileExtents[page].length);
1432 }
1433 #endif
1434
1435 needEncryptStart = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
1436
1437 AbsoluteTime_to_scalar(&compTime) = 0;
1438 AbsoluteTime_to_scalar(&decoTime) = 0;
1439
1440 clock_get_uptime(&allTime);
1441
1442 do
1443 {
1444 compressedSize = 0;
1445 uncompressedSize = 0;
1446 iterDone = false;
1447 pageType = 0; // wired pages first
1448
1449 IOPolledFileSeek(vars->fileVars, sizeof(IOHibernateImageHeader));
1450
1451 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
1452 ml_get_interrupts_enabled());
1453 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledBeforeSleepState, vars->ioBuffer);
1454 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
1455 pollerOpen = (kIOReturnSuccess == err);
1456 if (!pollerOpen)
1457 break;
1458
1459 // copy file block extent list if larger than header
1460
1461 count = vars->fileExtents->getLength();
1462 if (count > sizeof(header->fileExtentMap))
1463 {
1464 count -= sizeof(header->fileExtentMap);
1465 err = IOPolledFileWrite(vars->fileVars,
1466 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
1467 if (kIOReturnSuccess != err)
1468 break;
1469 }
1470
1471 // copy out restore1 code
1472
1473 page = atop_32(sectHIBB);
1474 count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
1475 header->restore1CodePage = page;
1476 header->restore1PageCount = count;
1477 header->restore1CodeOffset = ((uint32_t) &hibernate_machine_entrypoint) - sectHIBB;
1478 header->restore1StackOffset = ((uint32_t) &gIOHibernateRestoreStackEnd[0]) - 64 - sectHIBB;
1479
1480 // sum __HIB sect, with zeros for the stack
1481 src = (uint8_t *) trunc_page(sectHIBB);
1482 for (page = 0; page < count; page++)
1483 {
1484 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
1485 restore1Sum += hibernate_sum(src, page_size);
1486 else
1487 restore1Sum += 0x10000001;
1488 src += page_size;
1489 }
1490 sum1 = restore1Sum;
1491
1492 // write the __HIB sect, with zeros for the stack
1493
1494 src = (uint8_t *) trunc_page(sectHIBB);
1495 count = ((uint32_t) &gIOHibernateRestoreStack[0]) - trunc_page(sectHIBB);
1496 if (count)
1497 {
1498 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
1499 if (kIOReturnSuccess != err)
1500 break;
1501 }
1502 err = IOPolledFileWrite(vars->fileVars,
1503 (uint8_t *) 0,
1504 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
1505 cryptvars);
1506 if (kIOReturnSuccess != err)
1507 break;
1508 src = &gIOHibernateRestoreStackEnd[0];
1509 count = round_page(sectHIBB + sectSizeHIB) - ((uint32_t) src);
1510 if (count)
1511 {
1512 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
1513 if (kIOReturnSuccess != err)
1514 break;
1515 }
1516
1517 // write the preview buffer
1518
1519 addr64_t phys64;
1520 IOByteCount segLen;
1521
1522 if (vars->previewData)
1523 {
1524 ppnum = 0;
1525 count = 0;
1526 do
1527 {
1528 phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen);
1529 pageAndCount[0] = atop_64(phys64);
1530 pageAndCount[1] = atop_32(segLen);
1531 err = IOPolledFileWrite(vars->fileVars,
1532 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1533 cryptvars);
1534 if (kIOReturnSuccess != err)
1535 break;
1536 count += segLen;
1537 ppnum += sizeof(pageAndCount);
1538 }
1539 while (phys64);
1540 if (kIOReturnSuccess != err)
1541 break;
1542
1543 src = (uint8_t *) vars->previewData->getBytesNoCopy();
1544 count = vars->previewData->getLength();
1545
1546 header->previewPageListSize = ppnum;
1547 header->previewSize = count + ppnum;
1548
1549 for (page = 0; page < count; page += page_size)
1550 sum1 += hibernate_sum(src + page, page_size);
1551
1552 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
1553 if (kIOReturnSuccess != err)
1554 break;
1555 }
1556
1557 // mark areas for no save
1558
1559 for (count = 0;
1560 (phys64 = vars->ioBuffer->getPhysicalSegment64(count, &segLen));
1561 count += segLen)
1562 {
1563 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1564 atop_64(phys64), atop_32(segLen),
1565 kIOHibernatePageStateFree);
1566 pageCount -= atop_32(segLen);
1567 }
1568
1569 for (count = 0;
1570 (phys64 = vars->srcBuffer->getPhysicalSegment64(count, &segLen));
1571 count += segLen)
1572 {
1573 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1574 atop_64(phys64), atop_32(segLen),
1575 kIOHibernatePageStateFree);
1576 pageCount -= atop_32(segLen);
1577 }
1578
1579 // copy out bitmap of pages available for trashing during restore
1580
1581 bitmap_size = vars->page_list_wired->list_size;
1582 src = (uint8_t *) vars->page_list_wired;
1583 err = IOPolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
1584 if (kIOReturnSuccess != err)
1585 break;
1586
1587 // mark more areas for no save, but these are not available
1588 // for trashing during restore
1589
1590 #if !__i386__
1591 page = atop_32(sectHIBB);
1592 count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
1593 #else
1594 // XXX
1595 page = atop_32(sectHIBB & 0x3FFFFFFF);
1596 count = atop_32(round_page((sectHIBB + sectSizeHIB) & 0x3FFFFFFF)) - page;
1597 #endif
1598 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1599 page, count,
1600 kIOHibernatePageStateFree);
1601 pageCount -= count;
1602
1603
1604
1605 if (vars->previewBuffer) for (count = 0;
1606 (phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen));
1607 count += segLen)
1608 {
1609 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1610 atop_64(phys64), atop_32(segLen),
1611 kIOHibernatePageStateFree);
1612 pageCount -= atop_32(segLen);
1613 }
1614
1615 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
1616
1617 void * iter = 0;
1618 pagesDone = 0;
1619
1620 HIBLOG("writing %d pages\n", pageCount);
1621
1622 do
1623 {
1624 count = hibernate_page_list_iterate(pageType ? vars->page_list : vars->page_list_wired,
1625 &iter, &ppnum);
1626 // kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
1627
1628 iterDone = !count;
1629
1630 pageAndCount[0] = ppnum;
1631 pageAndCount[1] = count;
1632 err = IOPolledFileWrite(vars->fileVars,
1633 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1634 cryptvars);
1635 if (kIOReturnSuccess != err)
1636 break;
1637
1638 for (page = 0; page < count; page++)
1639 {
1640 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(ppnum), page_size);
1641 if (err)
1642 {
1643 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%d] %x\n", __LINE__, ppnum, err);
1644 break;
1645 }
1646
1647 sum = hibernate_sum(src, page_size);
1648
1649 clock_get_uptime(&startTime);
1650
1651 pageCompressedSize = WKdm_compress ((WK_word*) src, (WK_word*) (src + page_size), PAGE_SIZE_IN_WORDS);
1652
1653 clock_get_uptime(&endTime);
1654 ADD_ABSOLUTETIME(&compTime, &endTime);
1655 SUB_ABSOLUTETIME(&compTime, &startTime);
1656
1657 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1658 pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
1659
1660 if (pageCompressedSize > page_size)
1661 {
1662 // HIBLOG("------------lose: %d\n", pageCompressedSize);
1663 pageCompressedSize = page_size;
1664 }
1665
1666 if (pageCompressedSize != page_size)
1667 data = (src + page_size);
1668 else
1669 data = src;
1670
1671 tag = pageCompressedSize | kIOHibernateTagSignature;
1672
1673 if (pageType)
1674 sum2 += sum;
1675 else
1676 sum1 += sum;
1677
1678 if (needEncryptStart && (ppnum >= atop_32(sectDATAB)))
1679 {
1680 // start encrypting partway into the data about to be written
1681 vars->fileVars->encryptStart = (vars->fileVars->position + AES_BLOCK_SIZE - 1)
1682 & ~(AES_BLOCK_SIZE - 1);
1683 needEncryptStart = false;
1684 }
1685
1686 err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
1687 if (kIOReturnSuccess != err)
1688 break;
1689
1690 err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
1691 if (kIOReturnSuccess != err)
1692 break;
1693
1694 compressedSize += pageCompressedSize;
1695 if (pageCompressedSize)
1696 uncompressedSize += page_size;
1697 ppnum++;
1698 pagesDone++;
1699
1700 if (0 == (8191 & pagesDone))
1701 {
1702 clock_get_uptime(&endTime);
1703 SUB_ABSOLUTETIME(&endTime, &allTime);
1704 absolutetime_to_nanoseconds(endTime, &nsec);
1705 progressStamp = nsec / 750000000ULL;
1706 if (progressStamp != lastProgressStamp)
1707 {
1708 lastProgressStamp = progressStamp;
1709 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
1710 }
1711 }
1712 }
1713 if (kIOReturnSuccess != err)
1714 break;
1715 if (iterDone && !pageType)
1716 {
1717 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
1718 if (kIOReturnSuccess != err)
1719 break;
1720
1721 iterDone = false;
1722 pageType = 1;
1723 iter = 0;
1724 image1Size = vars->fileVars->position;
1725 if (cryptvars)
1726 {
1727 bcopy(&cryptvars->aes_iv[0],
1728 &gIOHibernateCryptWakeContext.aes_iv[0],
1729 sizeof(cryptvars->aes_iv));
1730 cryptvars = &gIOHibernateCryptWakeContext;
1731 }
1732 HIBLOG("image1Size %qd\n", image1Size);
1733 }
1734 }
1735 while (!iterDone);
1736 if (kIOReturnSuccess != err)
1737 break;
1738 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
1739 if (kIOReturnSuccess != err)
1740 break;
1741
1742 // Header:
1743
1744 header->imageSize = vars->fileVars->position;
1745 header->image1Size = image1Size;
1746 header->bitmapSize = bitmap_size;
1747 header->pageCount = pageCount;
1748 header->encryptStart = vars->fileVars->encryptStart;
1749
1750 header->restore1Sum = restore1Sum;
1751 header->image1Sum = sum1;
1752 header->image2Sum = sum2;
1753
1754 count = vars->fileExtents->getLength();
1755 if (count > sizeof(header->fileExtentMap))
1756 {
1757 header->fileExtentMapSize = count;
1758 count = sizeof(header->fileExtentMap);
1759 }
1760 else
1761 header->fileExtentMapSize = sizeof(header->fileExtentMap);
1762 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
1763
1764 IOPolledFileSeek(vars->fileVars, 0);
1765 err = IOPolledFileWrite(vars->fileVars,
1766 (uint8_t *) header, sizeof(IOHibernateImageHeader),
1767 cryptvars);
1768 if (kIOReturnSuccess != err)
1769 break;
1770 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
1771 if (kIOReturnSuccess != err)
1772 break;
1773 err = IOHibernatePollerIODone(vars->fileVars);
1774 if (kIOReturnSuccess != err)
1775 break;
1776 }
1777 while (false);
1778
1779 clock_get_uptime(&endTime);
1780 SUB_ABSOLUTETIME(&endTime, &allTime);
1781 absolutetime_to_nanoseconds(endTime, &nsec);
1782 HIBLOG("all time: %qd ms, ",
1783 nsec / 1000000ULL);
1784
1785 absolutetime_to_nanoseconds(compTime, &nsec);
1786 HIBLOG("comp time: %qd ms, ",
1787 nsec / 1000000ULL);
1788
1789 absolutetime_to_nanoseconds(decoTime, &nsec);
1790 HIBLOG("deco time: %qd ms, ",
1791 nsec / 1000000ULL);
1792
1793 HIBLOG("\nimage %qd, uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
1794 header->imageSize,
1795 uncompressedSize, atop_32(uncompressedSize), compressedSize,
1796 (int) ((compressedSize * 100ULL) / uncompressedSize),
1797 sum1, sum2);
1798
1799 if (pollerOpen)
1800 IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
1801
1802 HIBLOG("hibernate_write_image done(%x)\n", err);
1803
1804 // should we come back via regular wake, set the state in memory.
1805 gIOHibernateState = kIOHibernateStateInactive;
1806
1807 if ((kIOReturnSuccess == err) && !(kIOHibernateModeSleep & gIOHibernateMode))
1808 return (true /* power down */ );
1809 else
1810 return (false /* sleep */ );
1811 }
1812
1813 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1814
1815 DECLARE_IOHIBERNATEPROGRESSALPHA
1816
1817 static void
1818 ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
1819 {
1820 uint32_t rowBytes, pixelShift;
1821 uint32_t x, y;
1822 int32_t blob, lastBlob;
1823 uint32_t alpha, in, color, result;
1824 uint8_t * out;
1825 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1826
1827 pixelShift = display->depth >> 4;
1828 if (pixelShift < 1)
1829 return;
1830
1831 rowBytes = display->rowBytes;
1832
1833 screen += ((display->width
1834 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1835 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1836
1837 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
1838
1839 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
1840
1841 for (y = 0; y < kIOHibernateProgressHeight; y++)
1842 {
1843 out = screen + y * rowBytes;
1844 for (blob = firstBlob; blob <= lastBlob; blob++)
1845 {
1846 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
1847 for (x = 0; x < kIOHibernateProgressWidth; x++)
1848 {
1849 alpha = gIOHibernateProgressAlpha[y][x];
1850 result = color;
1851 if (alpha)
1852 {
1853 if (0xff != alpha)
1854 {
1855 in = display->progressSaveUnder[blob][saveindex[blob]++];
1856 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
1857 }
1858 if (1 == pixelShift)
1859 {
1860 result >>= 3;
1861 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1862 }
1863 else
1864 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1865 }
1866 out += (1 << pixelShift);
1867 }
1868 out += (kIOHibernateProgressSpacing << pixelShift);
1869 }
1870 }
1871 }
1872
1873 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1874
1875 extern "C" void
1876 hibernate_machine_init(void)
1877 {
1878 IOReturn err;
1879 uint32_t sum;
1880 uint32_t pagesDone;
1881 AbsoluteTime allTime, endTime;
1882 uint64_t nsec;
1883 uint32_t lastProgressStamp = 0;
1884 uint32_t progressStamp;
1885 uint64_t progressZeroPosition = 0;
1886 uint32_t blob, lastBlob = (uint32_t) -1L;
1887 hibernate_cryptvars_t * cryptvars = 0;
1888
1889 IOHibernateVars * vars = &gIOHibernateVars;
1890
1891 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
1892 return;
1893
1894 if ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)
1895 hibernate_page_list_discard(vars->page_list);
1896
1897
1898 sum = gIOHibernateCurrentHeader->actualImage1Sum;
1899 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
1900
1901 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, image1Size %qx, conflictCount %d, nextFree %x\n",
1902 gIOHibernateState, pagesDone, sum, gIOHibernateCurrentHeader->image1Size,
1903 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
1904
1905 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
1906 {
1907 HIBLOG("regular wake\n");
1908 return;
1909 }
1910
1911 HIBPRINT("diag %x %x %x %x\n",
1912 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
1913 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
1914
1915 HIBPRINT("video %x %d %d %d\n",
1916 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
1917 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height);
1918
1919 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)
1920 {
1921 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
1922 * gIOHibernateGraphicsInfo->rowBytes);
1923 IOMapPages(kernel_map,
1924 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
1925 vars->videoMapSize, kIOMapInhibitCache );
1926 }
1927
1928 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();;
1929 uint32_t decoOffset;
1930
1931 clock_get_uptime(&allTime);
1932
1933 HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
1934 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0);
1935 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
1936
1937 if (gIOHibernateCurrentHeader->previewSize)
1938 progressZeroPosition = gIOHibernateCurrentHeader->previewSize
1939 + gIOHibernateCurrentHeader->fileExtentMapSize
1940 - sizeof(gIOHibernateCurrentHeader->fileExtentMap)
1941 + ptoa_64(gIOHibernateCurrentHeader->restore1PageCount);
1942
1943 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
1944
1945 if (vars->videoMapping)
1946 {
1947 lastBlob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
1948 / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
1949 ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, 0, lastBlob);
1950 }
1951
1952 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
1953 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1954 {
1955 cryptvars = &gIOHibernateCryptWakeContext;
1956 bcopy(&gIOHibernateCryptWakeVars->aes_iv[0],
1957 &cryptvars->aes_iv[0],
1958 sizeof(cryptvars->aes_iv));
1959 }
1960
1961 // kick off the read ahead
1962 vars->fileVars->io = false;
1963 vars->fileVars->bufferHalf = 0;
1964 vars->fileVars->bufferLimit = 0;
1965 vars->fileVars->lastRead = 0;
1966 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
1967
1968 IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
1969 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
1970 // --
1971
1972 HIBLOG("hibernate_machine_init reading\n");
1973
1974 uint32_t * header = (uint32_t *) src;
1975 sum = 0;
1976
1977 do
1978 {
1979 unsigned int count;
1980 unsigned int page;
1981 uint32_t tag;
1982 vm_offset_t ppnum, compressedSize;
1983
1984 IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
1985
1986 ppnum = header[0];
1987 count = header[1];
1988
1989 // HIBPRINT("(%x, %x)\n", ppnum, count);
1990
1991 if (!count)
1992 break;
1993
1994 for (page = 0; page < count; page++)
1995 {
1996 IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
1997
1998 compressedSize = kIOHibernateTagLength & tag;
1999 if (!compressedSize)
2000 {
2001 ppnum++;
2002 pagesDone++;
2003 continue;
2004 }
2005
2006 IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2007
2008 if (compressedSize != page_size)
2009 {
2010 decoOffset = page_size;
2011 WKdm_decompress((WK_word*) src, (WK_word*) (src + decoOffset), PAGE_SIZE_IN_WORDS);
2012 }
2013 else
2014 decoOffset = 0;
2015
2016 sum += hibernate_sum((src + decoOffset), page_size);
2017
2018 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2019 if (err)
2020 HIBLOG("IOMemoryDescriptorReadToPhysical [%d] %x\n", ppnum, err);
2021
2022 ppnum++;
2023 pagesDone++;
2024
2025 if (vars->videoMapping && (0 == (255 & pagesDone)))
2026 {
2027 blob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
2028 / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
2029 if (blob != lastBlob)
2030 {
2031 ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, lastBlob, blob);
2032 lastBlob = blob;
2033 }
2034 }
2035
2036 if (0 == (8191 & pagesDone))
2037 {
2038 clock_get_uptime(&endTime);
2039 SUB_ABSOLUTETIME(&endTime, &allTime);
2040 absolutetime_to_nanoseconds(endTime, &nsec);
2041 progressStamp = nsec / 750000000ULL;
2042 if (progressStamp != lastProgressStamp)
2043 {
2044 lastProgressStamp = progressStamp;
2045 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2046 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2047 }
2048 }
2049 }
2050 }
2051 while (true);
2052
2053 gIOHibernateCurrentHeader->actualImage2Sum = sum;
2054
2055 if (vars->fileVars->io)
2056 (void) IOHibernatePollerIODone(vars->fileVars);
2057
2058 err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
2059
2060 if (vars->videoMapping)
2061 ProgressUpdate(gIOHibernateGraphicsInfo,
2062 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2063
2064 clock_get_uptime(&endTime);
2065 SUB_ABSOLUTETIME(&endTime, &allTime);
2066 absolutetime_to_nanoseconds(endTime, &nsec);
2067
2068 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %qd ms\n",
2069 pagesDone, sum, nsec / 1000000ULL);
2070 }
2071
2072 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
mirror of XNU