]> git.saurik.com Git - apple/xnu.git/blob - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blob
? search:


4beb2606cf00c9d27a48af4514296a44c7f79ded
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
1 /*
2 * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
11 *
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
18 * under the License.
19 *
20 * @APPLE_LICENSE_HEADER_END@
21 */
22
23
24 /*
25
26 Sleep:
27
28 - PMRootDomain calls IOHibernateSystemSleep() before system sleep
29 (devices awake, normal execution context)
30 - IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
31 grabs its extents and searches for a polling driver willing to work with that IOMedia.
32 The BSD code makes an ioctl to the storage driver to get the partition base offset to
33 the disk, and other ioctls to get the transfer constraints
34 If successful, the file is written to make sure its initially not bootable (in case of
35 later failure) and nvram set to point to the first block of the file. (Has to be done
36 here so blocking is possible in nvram support).
37 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
38 page out any pages it wants to (currently zero, but probably some percentage of memory).
39 Its assumed just allocating pages will cause the VM system to naturally select the best
40 pages for eviction. It also copies processor flags needed for the restore path and sets
41 a flag in the boot processor proc info.
42 gIOHibernateState = kIOHibernateStateHibernating.
43 - Regular sleep progresses - some drivers may inspect the root domain property
44 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
45 as usual but leaves motherboard I/O on.
46 - Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
47 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
48 all ppc RC bits out of the hash table and caches into the mapping structures.
49 - hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
50 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
51 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
52 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
53 The image header and block list are written. The header includes the second file extent so
54 only the header block is needed to read the file, regardless of filesystem.
55 The kernel section "__HIB" is written uncompressed to the image. This section of code and data
56 (only) is used to decompress the image during wake/boot.
57 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
58 The bitmaps are written to the image.
59 More areas are removed from the bitmaps (after they have been written to the image) - the
60 section "__HIB" pages and interrupt stack.
61 Each wired page is compressed and written and then each non-wired page. Compression and
62 disk writes are in parallel.
63 The image header is written to the start of the file and the polling driver closed.
64 The machine powers down (or sleeps).
65
66 Boot/Wake:
67
68 - BootX sees the boot-image nvram variable containing the device and block number of the image,
69 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
70 - BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
71 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
72 that is in the kernel's __HIB section.
73 - BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
74 hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
75 only code & data in that section is safe to call since all the other wired pages are still
76 compressed in the image.
77 - hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
78 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
79 location directly, and copies those that can't to interim free pages. When the image has been
80 completed, the copies are uncompressed, overwriting the wired image pages.
81 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
82 is used to get pages into place for 64bit.
83 - the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
84 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
85 are removed from the software strutures, and the hash table is reinitialized.
86 - After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
87 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
88 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
89 for the remaining non wired pages.
90 - The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
91 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
92 is closed via bsd.
93
94 Polled Mode I/O:
95
96 IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
97 registry, specifying an object of calls IOPolledInterface.
98
99 Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
100 partition) that the image is going to live, looking for polled interface properties. If it finds
101 one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
102 interfaces found are kept in an ordered list.
103
104 There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
105 few different contexts things happen in:
106
107 - there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
108 up and running) and after wake - this is safe to allocate memory and do anything. The device
109 ignores sleep requests from that point since its a waste of time if it goes to sleep and
110 immediately wakes back up for the image write.
111
112 - there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
113 immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
114 from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
115 used to flush and set the disk to sleep.
116
117 - there is an Open/Close (AfterSleep) pair made around the image read operations that happen
118 immediately after sleep. These can't block or allocate memory. This is happening after the platform
119 expert has woken the low level bits of the system, but most of the I/O system has not. There is only
120 one thread running.
121
122 For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
123 (prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
124 the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
125 Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
126 that is called for the hardware to check for events, and complete the I/O via the callback.
127 The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
128 to restrict I/O ops.
129 */
130
131 #include <sys/systm.h>
132
133 #include <IOKit/IOWorkLoop.h>
134 #include <IOKit/IOCommandGate.h>
135 #include <IOKit/IOTimerEventSource.h>
136 #include <IOKit/IOPlatformExpert.h>
137 #include <IOKit/IOKitDebug.h>
138 #include <IOKit/IOTimeStamp.h>
139 #include <IOKit/pwr_mgt/RootDomain.h>
140 #include <IOKit/pwr_mgt/IOPMPrivate.h>
141 #include <IOKit/IOMessage.h>
142 #include <IOKit/IODeviceTreeSupport.h>
143 #include <IOKit/IOBSD.h>
144 #include "RootDomainUserClient.h"
145 #include <IOKit/pwr_mgt/IOPowerConnection.h>
146 #include "IOPMPowerStateQueue.h"
147 #include <IOKit/IOBufferMemoryDescriptor.h>
148 #include <crypto/aes.h>
149
150 #include <sys/uio.h>
151 #include <sys/conf.h>
152 #include <sys/stat.h>
153 #include <sys/fcntl.h> // (FWRITE, ...)
154 #include <sys/sysctl.h>
155
156 #include <IOKit/IOHibernatePrivate.h>
157 #include <IOKit/IOPolledInterface.h>
158 #include <IOKit/IONVRAM.h>
159 #include "IOHibernateInternal.h"
160 #include "WKdm.h"
161 #include "IOKitKernelInternal.h"
162
163 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
164
165 OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
166
167 OSMetaClassDefineReservedUnused(IOPolledInterface, 0);
168 OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
169 OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
170 OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
171 OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
172 OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
173 OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
174 OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
175 OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
176 OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
177 OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
178 OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
179 OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
180 OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
181 OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
182 OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
183
184 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
185
186 extern uint32_t gIOHibernateState;
187 uint32_t gIOHibernateMode;
188 static char gIOHibernateBootSignature[256+1];
189 static char gIOHibernateFilename[MAXPATHLEN+1];
190 static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
191 uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
192
193 static IODTNVRAM * gIOOptionsEntry;
194 static IORegistryEntry * gIOChosenEntry;
195 #ifdef __i386__
196 static const OSSymbol * gIOCreateEFIDevicePathSymbol;
197 #endif
198
199 static IOPolledFileIOVars gFileVars;
200 static IOHibernateVars gIOHibernateVars;
201 static struct kern_direct_file_io_ref_t * gIOHibernateFileRef;
202 static hibernate_cryptvars_t gIOHibernateCryptWakeContext;
203
204 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
205
206 enum { kXPRamAudioVolume = 8 };
207 enum { kDefaultIOSize = 128 * 1024 };
208 enum { kVideoMapSize = 32 * 1024 * 1024 };
209
210 #ifndef kIOMediaPreferredBlockSizeKey
211 #define kIOMediaPreferredBlockSizeKey "Preferred Block Size"
212 #endif
213
214 #ifndef kIOBootPathKey
215 #define kIOBootPathKey "bootpath"
216 #endif
217 #ifndef kIOSelectedBootDeviceKey
218 #define kIOSelectedBootDeviceKey "boot-device"
219 #endif
220
221
222 enum { kIOHibernateMinPollersNeeded = 2 };
223
224 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
225
226 // copy from phys addr to MD
227
228 static IOReturn
229 IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
230 IOByteCount offset, addr64_t bytes, IOByteCount length)
231 {
232 addr64_t srcAddr = bytes;
233 IOByteCount remaining;
234
235 remaining = length = min(length, md->getLength() - offset);
236 while (remaining) { // (process another target segment?)
237 addr64_t dstAddr64;
238 IOByteCount dstLen;
239
240 dstAddr64 = md->getPhysicalSegment64(offset, &dstLen);
241 if (!dstAddr64)
242 break;
243
244 // Clip segment length to remaining
245 if (dstLen > remaining)
246 dstLen = remaining;
247
248 #if 1
249 bcopy_phys(srcAddr, dstAddr64, dstLen);
250 #else
251 copypv(srcAddr, dstAddr64, dstLen,
252 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
253 #endif
254 srcAddr += dstLen;
255 offset += dstLen;
256 remaining -= dstLen;
257 }
258
259 assert(!remaining);
260
261 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
262 }
263
264 // copy from MD to phys addr
265
266 static IOReturn
267 IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
268 IOByteCount offset, addr64_t bytes, IOByteCount length)
269 {
270 addr64_t dstAddr = bytes;
271 IOByteCount remaining;
272
273 remaining = length = min(length, md->getLength() - offset);
274 while (remaining) { // (process another target segment?)
275 addr64_t srcAddr64;
276 IOByteCount dstLen;
277
278 srcAddr64 = md->getPhysicalSegment64(offset, &dstLen);
279 if (!srcAddr64)
280 break;
281
282 // Clip segment length to remaining
283 if (dstLen > remaining)
284 dstLen = remaining;
285
286 #if 1
287 bcopy_phys(srcAddr64, dstAddr, dstLen);
288 #else
289 copypv(srcAddr, dstAddr64, dstLen,
290 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
291 #endif
292 dstAddr += dstLen;
293 offset += dstLen;
294 remaining -= dstLen;
295 }
296
297 assert(!remaining);
298
299 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
300 }
301
302 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
303
304 void
305 hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
306 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
307 {
308 count += ppnum;
309 switch (kind)
310 {
311 case kIOHibernatePageStateUnwiredSave:
312 // unwired save
313 for (; ppnum < count; ppnum++)
314 {
315 hibernate_page_bitset(page_list, FALSE, ppnum);
316 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
317 }
318 break;
319 case kIOHibernatePageStateWiredSave:
320 // wired save
321 for (; ppnum < count; ppnum++)
322 {
323 hibernate_page_bitset(page_list, FALSE, ppnum);
324 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
325 }
326 break;
327 case kIOHibernatePageStateFree:
328 // free page
329 for (; ppnum < count; ppnum++)
330 {
331 hibernate_page_bitset(page_list, TRUE, ppnum);
332 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
333 }
334 break;
335 default:
336 panic("hibernate_set_page_state");
337 }
338 }
339
340 static vm_offset_t
341 hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
342 {
343 uint32_t page = *pPage;
344 uint32_t count;
345 hibernate_bitmap_t * bitmap;
346
347 while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
348 {
349 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
350 if (!count)
351 break;
352 page += count;
353 if (page <= bitmap->last_page)
354 break;
355 }
356
357 *pPage = page;
358 if (bitmap)
359 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
360 else
361 count = 0;
362
363 return (count);
364 }
365
366 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
367
368 static IOReturn
369 IOHibernatePollerProbe(IOPolledFileIOVars * vars, IOService * target)
370 {
371 IOReturn err = kIOReturnError;
372 int32_t idx;
373 IOPolledInterface * poller;
374
375 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
376 {
377 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
378 err = poller->probe(target);
379 if (err)
380 {
381 HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
382 break;
383 }
384 }
385
386 return (err);
387 }
388
389 static IOReturn
390 IOHibernatePollerOpen(IOPolledFileIOVars * vars, uint32_t state, IOMemoryDescriptor * md)
391 {
392 IOReturn err = kIOReturnError;
393 int32_t idx;
394 IOPolledInterface * poller;
395
396 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
397 {
398 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
399 err = poller->open(state, md);
400 if (err)
401 {
402 HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
403 break;
404 }
405 }
406
407 return (err);
408 }
409
410 static IOReturn
411 IOHibernatePollerClose(IOPolledFileIOVars * vars, uint32_t state)
412 {
413 IOReturn err = kIOReturnError;
414 int32_t idx;
415 IOPolledInterface * poller;
416
417 for (idx = 0;
418 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
419 idx++)
420 {
421 err = poller->close(state);
422 if (err)
423 HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
424 }
425
426 return (err);
427 }
428
429 static void
430 IOHibernatePollerIOComplete(void * target,
431 void * parameter,
432 IOReturn status,
433 UInt64 actualByteCount)
434 {
435 IOPolledFileIOVars * vars = (IOPolledFileIOVars *) parameter;
436
437 vars->ioStatus = status;
438 }
439
440 static IOReturn
441 IOHibernatePollerIO(IOPolledFileIOVars * vars,
442 uint32_t operation, uint32_t bufferOffset,
443 uint64_t deviceOffset, uint64_t length)
444 {
445
446 IOReturn err = kIOReturnError;
447 IOPolledInterface * poller;
448 IOPolledCompletion completion;
449
450 completion.target = 0;
451 completion.action = &IOHibernatePollerIOComplete;
452 completion.parameter = vars;
453
454 vars->ioStatus = -1;
455
456 poller = (IOPolledInterface *) vars->pollers->getObject(0);
457 err = poller->startIO(operation, bufferOffset, deviceOffset + vars->block0, length, completion);
458 if (err)
459 HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
460
461 return (err);
462 }
463
464 static IOReturn
465 IOHibernatePollerIODone(IOPolledFileIOVars * vars)
466 {
467 IOReturn err = kIOReturnError;
468 int32_t idx;
469 IOPolledInterface * poller;
470
471 while (-1 == vars->ioStatus)
472 {
473 for (idx = 0;
474 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
475 idx++)
476 {
477 err = poller->checkForWork();
478 if (err)
479 HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
480 }
481 }
482
483 if (kIOReturnSuccess != vars->ioStatus)
484 HIBLOG("IOPolledInterface::ioStatus 0x%x\n", vars->ioStatus);
485
486 return (vars->ioStatus);
487 }
488
489 IOReturn
490 IOPolledInterface::checkAllForWork(void)
491 {
492 IOReturn err = kIOReturnNotReady;
493 int32_t idx;
494 IOPolledInterface * poller;
495
496 IOHibernateVars * vars = &gIOHibernateVars;
497
498 if (!vars->fileVars || !vars->fileVars->pollers)
499 return (err);
500
501 for (idx = 0;
502 (poller = (IOPolledInterface *) vars->fileVars->pollers->getObject(idx));
503 idx++)
504 {
505 err = poller->checkForWork();
506 if (err)
507 HIBLOG("IOPolledInterface::checkAllForWork[%d] 0x%x\n", idx, err);
508 }
509
510 return (err);
511 }
512
513 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
514
515 struct _OpenFileContext
516 {
517 OSData * extents;
518 uint64_t size;
519 };
520
521 static void
522 file_extent_callback(void * ref, uint64_t start, uint64_t length)
523 {
524 _OpenFileContext * ctx = (_OpenFileContext *) ref;
525 IOPolledFileExtent extent;
526
527 extent.start = start;
528 extent.length = length;
529
530 ctx->extents->appendBytes(&extent, sizeof(extent));
531 ctx->size += length;
532 }
533
534 IOReturn
535 IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
536 IOPolledFileIOVars ** fileVars, OSData ** fileExtents,
537 OSData ** imagePath)
538 {
539 IOReturn err = kIOReturnError;
540 IOPolledFileIOVars * vars;
541 _OpenFileContext ctx;
542 OSData * extentsData;
543 OSNumber * num;
544 IORegistryEntry * part = 0;
545 OSDictionary * matching;
546 OSIterator * iter;
547 dev_t hibernate_image_dev;
548 uint64_t maxiobytes;
549
550 vars = &gFileVars;
551 do
552 {
553 HIBLOG("sizeof(IOHibernateImageHeader) == %ld\n", sizeof(IOHibernateImageHeader));
554 if (sizeof(IOHibernateImageHeader) != 512)
555 continue;
556
557 vars->io = false;
558 vars->buffer = (uint8_t *) ioBuffer->getBytesNoCopy();
559 vars->bufferHalf = 0;
560 vars->bufferOffset = 0;
561 vars->bufferSize = ioBuffer->getLength() >> 1;
562
563 extentsData = OSData::withCapacity(32);
564
565 ctx.extents = extentsData;
566 ctx.size = 0;
567 vars->fileRef = kern_open_file_for_direct_io(filename,
568 &file_extent_callback, &ctx,
569 &hibernate_image_dev,
570 &vars->block0,
571 &maxiobytes);
572 if (!vars->fileRef)
573 {
574 err = kIOReturnNoSpace;
575 break;
576 }
577 HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx\n", filename, ctx.size,
578 vars->block0, maxiobytes);
579 if (ctx.size < 1*1024*1024) // check against image size estimate!
580 {
581 err = kIOReturnNoSpace;
582 break;
583 }
584
585 if (maxiobytes < vars->bufferSize)
586 vars->bufferSize = maxiobytes;
587
588 vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
589
590 matching = IOService::serviceMatching("IOMedia");
591 num = OSNumber::withNumber(major(hibernate_image_dev), 32);
592 matching->setObject(kIOBSDMajorKey, num);
593 num->release();
594 num = OSNumber::withNumber(minor(hibernate_image_dev), 32);
595 matching->setObject(kIOBSDMinorKey, num);
596 num->release();
597 iter = IOService::getMatchingServices(matching);
598 matching->release();
599 if (iter)
600 {
601 part = (IORegistryEntry *) iter->getNextObject();
602 part->retain();
603 iter->release();
604 }
605
606 int minor, major;
607 IORegistryEntry * next;
608 IORegistryEntry * child;
609 OSData * data;
610
611 num = (OSNumber *) part->getProperty(kIOBSDMajorKey);
612 if (!num)
613 break;
614 major = num->unsigned32BitValue();
615 num = (OSNumber *) part->getProperty(kIOBSDMinorKey);
616 if (!num)
617 break;
618 minor = num->unsigned32BitValue();
619
620 hibernate_image_dev = makedev(major, minor);
621
622 vars->pollers = OSArray::withCapacity(4);
623 if (!vars->pollers)
624 break;
625
626 vars->blockSize = 512;
627 next = part;
628 do
629 {
630 IOPolledInterface * poller;
631 OSObject * obj;
632
633 obj = next->getProperty(kIOPolledInterfaceSupportKey);
634 if (kOSBooleanFalse == obj)
635 {
636 vars->pollers->flushCollection();
637 break;
638 }
639 else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
640 vars->pollers->setObject(poller);
641 if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey))))
642 vars->blockSize = num->unsigned32BitValue();
643 child = next;
644 }
645 while ((next = child->getParentEntry(gIOServicePlane))
646 && child->isParent(next, gIOServicePlane, true));
647
648 HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
649 major, minor, vars->blockSize, vars->pollers->getCount());
650 if (vars->pollers->getCount() < kIOHibernateMinPollersNeeded)
651 continue;
652
653 err = IOHibernatePollerProbe(vars, (IOService *) part);
654 if (kIOReturnSuccess != err)
655 break;
656
657 err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer);
658 if (kIOReturnSuccess != err)
659 break;
660
661 *fileVars = vars;
662 *fileExtents = extentsData;
663
664 // make imagePath
665
666 if ((extentsData->getLength() >= sizeof(IOPolledFileExtent)))
667 {
668 char str2[24];
669
670 #if __i386__
671 if (!gIOCreateEFIDevicePathSymbol)
672 gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
673
674 sprintf(str2, "%qx", vars->extentMap[0]);
675
676 err = IOService::getPlatform()->callPlatformFunction(
677 gIOCreateEFIDevicePathSymbol, false,
678 (void *) part, (void *) str2, (void *) true,
679 (void *) &data);
680 #else
681 char str1[256];
682 int len = sizeof(str1);
683
684 if (!part->getPath(str1, &len, gIODTPlane))
685 err = kIOReturnNotFound;
686 else
687 {
688 sprintf(str2, ",%qx", vars->extentMap[0]);
689 // (strip the plane name)
690 char * tail = strchr(str1, ':');
691 if (!tail)
692 tail = str1 - 1;
693 data = OSData::withBytes(tail + 1, strlen(tail + 1));
694 data->appendBytes(str2, strlen(str2));
695 }
696 #endif
697 if (kIOReturnSuccess == err)
698 *imagePath = data;
699 else
700 HIBLOG("error 0x%x getting path\n", err);
701 }
702 }
703 while (false);
704
705 if (kIOReturnSuccess != err)
706 {
707 HIBLOG("error 0x%x opening hibernation file\n", err);
708 if (vars->fileRef)
709 kern_close_file_for_direct_io(vars->fileRef);
710 }
711
712 if (part)
713 part->release();
714
715 return (err);
716 }
717
718 IOReturn
719 IOPolledFileClose( IOPolledFileIOVars * vars )
720 {
721 if (vars->pollers)
722 {
723 IOHibernatePollerClose(vars, kIOPolledPostflightState);
724 vars->pollers->release();
725 }
726
727 gIOHibernateFileRef = vars->fileRef;
728
729 bzero(vars, sizeof(IOPolledFileIOVars));
730
731 return (kIOReturnSuccess);
732 }
733
734 static IOReturn
735 IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
736 {
737 IOPolledFileExtent * extentMap;
738
739 extentMap = vars->extentMap;
740
741 vars->position = position;
742
743 while (position >= extentMap->length)
744 {
745 position -= extentMap->length;
746 extentMap++;
747 }
748
749 vars->currentExtent = extentMap;
750 vars->extentRemaining = extentMap->length - position;
751 vars->extentPosition = vars->position - position;
752
753 if (vars->bufferSize <= vars->extentRemaining)
754 vars->bufferLimit = vars->bufferSize;
755 else
756 vars->bufferLimit = vars->extentRemaining;
757
758 return (kIOReturnSuccess);
759 }
760
761 static IOReturn
762 IOPolledFileWrite(IOPolledFileIOVars * vars,
763 const uint8_t * bytes, IOByteCount size,
764 hibernate_cryptvars_t * cryptvars)
765 {
766 IOReturn err = kIOReturnSuccess;
767 IOByteCount copy;
768 bool flush = false;
769
770 do
771 {
772 if (!bytes && !size)
773 {
774 // seek to end of block & flush
775 size = vars->position & (vars->blockSize - 1);
776 if (size)
777 size = vars->blockSize - size;
778 flush = true;
779 // use some garbage for the fill
780 bytes = vars->buffer + vars->bufferOffset;
781 }
782
783 copy = vars->bufferLimit - vars->bufferOffset;
784 if (copy > size)
785 copy = size;
786 else
787 flush = true;
788
789 if (bytes)
790 {
791 bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
792 bytes += copy;
793 }
794 else
795 bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
796
797 size -= copy;
798 vars->bufferOffset += copy;
799 vars->position += copy;
800
801 if (flush && vars->bufferOffset)
802 {
803 uint64_t offset = (vars->position - vars->bufferOffset
804 - vars->extentPosition + vars->currentExtent->start);
805 uint32_t length = (vars->bufferOffset);
806
807 if (cryptvars && vars->encryptStart && (vars->position > vars->encryptStart))
808 {
809 uint32_t encryptLen, encryptStart;
810 encryptLen = vars->position - vars->encryptStart;
811 if (encryptLen > length)
812 encryptLen = length;
813 encryptStart = length - encryptLen;
814
815 // encrypt the buffer
816 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
817 &cryptvars->aes_iv[0],
818 encryptLen / AES_BLOCK_SIZE,
819 vars->buffer + vars->bufferHalf + encryptStart,
820 &cryptvars->ctx.encrypt);
821 // save initial vector for following encrypts
822 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
823 &cryptvars->aes_iv[0],
824 AES_BLOCK_SIZE);
825 }
826
827 if (vars->io)
828 {
829 err = IOHibernatePollerIODone(vars);
830 if (kIOReturnSuccess != err)
831 break;
832 }
833
834 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
835 //if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
836
837 err = IOHibernatePollerIO(vars, kIOPolledWrite, vars->bufferHalf, offset, length);
838 if (kIOReturnSuccess != err)
839 break;
840 vars->io = true;
841
842 vars->extentRemaining -= vars->bufferOffset;
843 if (!vars->extentRemaining)
844 {
845 vars->currentExtent++;
846 vars->extentRemaining = vars->currentExtent->length;
847 vars->extentPosition = vars->position;
848 if (!vars->extentRemaining)
849 {
850 err = kIOReturnOverrun;
851 break;
852 }
853 }
854
855 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
856 vars->bufferOffset = 0;
857 if (vars->bufferSize <= vars->extentRemaining)
858 vars->bufferLimit = vars->bufferSize;
859 else
860 vars->bufferLimit = vars->extentRemaining;
861
862 flush = false;
863 }
864 }
865 while (size);
866
867 return (err);
868 }
869
870 static IOReturn
871 IOPolledFileRead(IOPolledFileIOVars * vars,
872 uint8_t * bytes, IOByteCount size,
873 hibernate_cryptvars_t * cryptvars)
874 {
875 IOReturn err = kIOReturnSuccess;
876 IOByteCount copy;
877
878 // bytesWritten += size;
879
880 do
881 {
882 copy = vars->bufferLimit - vars->bufferOffset;
883 if (copy > size)
884 copy = size;
885
886 if (bytes)
887 {
888 bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
889 bytes += copy;
890 }
891 size -= copy;
892 vars->bufferOffset += copy;
893 // vars->position += copy;
894
895 if (vars->bufferOffset == vars->bufferLimit)
896 {
897 if (vars->io)
898 {
899 err = IOHibernatePollerIODone(vars);
900 if (kIOReturnSuccess != err)
901 break;
902 }
903 else
904 cryptvars = 0;
905
906 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
907
908 vars->position += vars->lastRead;
909 vars->extentRemaining -= vars->lastRead;
910 vars->bufferLimit = vars->lastRead;
911
912 if (!vars->extentRemaining)
913 {
914 vars->currentExtent++;
915 vars->extentRemaining = vars->currentExtent->length;
916 vars->extentPosition = vars->position;
917 if (!vars->extentRemaining)
918 {
919 err = kIOReturnOverrun;
920 break;
921 }
922 }
923
924 if (vars->extentRemaining <= vars->bufferSize)
925 vars->lastRead = vars->extentRemaining;
926 else
927 vars->lastRead = vars->bufferSize;
928
929 uint64_t offset = (vars->position
930 - vars->extentPosition + vars->currentExtent->start);
931 uint64_t length = (vars->lastRead);
932
933 //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
934
935 err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
936 if (kIOReturnSuccess != err)
937 break;
938 vars->io = true;
939
940 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
941 vars->bufferOffset = 0;
942
943 if (cryptvars)
944 {
945 uint8_t thisVector[AES_BLOCK_SIZE];
946 // save initial vector for following decrypts
947 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
948 bcopy(vars->buffer + vars->bufferHalf + vars->lastRead - AES_BLOCK_SIZE,
949 &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
950 // decrypt the buffer
951 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
952 &thisVector[0],
953 vars->lastRead / AES_BLOCK_SIZE,
954 vars->buffer + vars->bufferHalf,
955 &cryptvars->ctx.decrypt);
956 }
957 }
958 }
959 while (size);
960
961 return (err);
962 }
963
964 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
965
966 IOReturn
967 IOHibernateSystemSleep(void)
968 {
969 IOReturn err;
970 OSData * data;
971 OSObject * obj;
972 OSString * str;
973 OSNumber * num;
974
975 IOHibernateVars * vars = &gIOHibernateVars;
976
977 if (vars->fileVars && vars->fileVars->fileRef)
978 // already on the way down
979 return (kIOReturnSuccess);
980
981 gIOHibernateState = kIOHibernateStateInactive;
982
983 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateModeKey)))
984 {
985 if ((num = OSDynamicCast(OSNumber, obj)))
986 gIOHibernateMode = num->unsigned32BitValue();
987 if (kIOHibernateModeSleep & gIOHibernateMode)
988 // default to discard clean for safe sleep
989 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
990 | kIOHibernateModeDiscardCleanActive);
991
992 obj->release();
993 }
994 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeRatioKey)))
995 {
996 if ((num = OSDynamicCast(OSNumber, obj)))
997 gIOHibernateFreeRatio = num->unsigned32BitValue();
998 obj->release();
999 }
1000 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeTimeKey)))
1001 {
1002 if ((num = OSDynamicCast(OSNumber, obj)))
1003 gIOHibernateFreeTime = num->unsigned32BitValue();
1004 obj->release();
1005 }
1006 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
1007 {
1008 if ((str = OSDynamicCast(OSString, obj)))
1009 strcpy(gIOHibernateFilename, str->getCStringNoCopy());
1010 obj->release();
1011 }
1012
1013 if (!gIOHibernateMode || !gIOHibernateFilename[0])
1014 return (kIOReturnUnsupported);
1015
1016 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
1017
1018 do
1019 {
1020 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1021 4 * page_size, page_size);
1022 vars->ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1023 2 * kDefaultIOSize, page_size);
1024
1025 if (!vars->srcBuffer || !vars->ioBuffer)
1026 {
1027 err = kIOReturnNoMemory;
1028 break;
1029 }
1030
1031 err = IOPolledFileOpen(gIOHibernateFilename, vars->ioBuffer,
1032 &vars->fileVars, &vars->fileExtents, &data);
1033 if (KERN_SUCCESS != err)
1034 {
1035 HIBLOG("IOPolledFileOpen(%x)\n", err);
1036 break;
1037 }
1038 if (vars->fileVars->fileRef)
1039 {
1040 // invalidate the image file
1041 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1042 int err = kern_write_file(vars->fileVars->fileRef, 0,
1043 (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1044 if (KERN_SUCCESS != err)
1045 HIBLOG("kern_write_file(%d)\n", err);
1046 }
1047
1048 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1049
1050 boolean_t encryptedswap;
1051 err = hibernate_setup(gIOHibernateCurrentHeader,
1052 gIOHibernateFreeRatio, gIOHibernateFreeTime,
1053 &vars->page_list, &vars->page_list_wired, &encryptedswap);
1054 if (KERN_SUCCESS != err)
1055 {
1056 HIBLOG("hibernate_setup(%d)\n", err);
1057 break;
1058 }
1059
1060 if (encryptedswap)
1061 gIOHibernateMode ^= kIOHibernateModeEncrypt;
1062
1063 vars->videoAllocSize = kVideoMapSize;
1064 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
1065 vars->videoMapping = 0;
1066
1067 // generate crypt keys
1068 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
1069 vars->wiredCryptKey[i] = random();
1070 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
1071 vars->cryptKey[i] = random();
1072
1073 // set nvram
1074
1075 IORegistryEntry * regEntry;
1076 if (!gIOOptionsEntry)
1077 {
1078 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
1079 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
1080 if (regEntry && !gIOOptionsEntry)
1081 regEntry->release();
1082 }
1083 if (!gIOChosenEntry)
1084 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1085
1086 if (gIOOptionsEntry)
1087 {
1088 const OSSymbol * sym;
1089
1090 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1091 if (sym)
1092 {
1093 gIOOptionsEntry->setProperty(sym, data);
1094 sym->release();
1095 }
1096 data->release();
1097
1098 #ifdef __ppc__
1099 size_t len;
1100 char valueString[16];
1101
1102 vars->saveBootDevice = gIOOptionsEntry->copyProperty(kIOSelectedBootDeviceKey);
1103 if (gIOChosenEntry)
1104 {
1105 OSData * bootDevice = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOBootPathKey));
1106 if (bootDevice)
1107 {
1108 sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
1109 OSString * str2 = OSString::withCStringNoCopy((const char *) bootDevice->getBytesNoCopy());
1110 if (sym && str2)
1111 gIOOptionsEntry->setProperty(sym, str2);
1112 if (sym)
1113 sym->release();
1114 if (str2)
1115 str2->release();
1116 }
1117 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMemorySignatureKey));
1118 if (data)
1119 {
1120 vars->haveFastBoot = true;
1121
1122 len = sprintf(valueString, "0x%lx", *((UInt32 *)data->getBytesNoCopy()));
1123 data = OSData::withBytes(valueString, len + 1);
1124 sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
1125 if (sym && data)
1126 gIOOptionsEntry->setProperty(sym, data);
1127 if (sym)
1128 sym->release();
1129 if (data)
1130 data->release();
1131 }
1132 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
1133 if (data)
1134 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
1135 }
1136 #endif /* __ppc__ */
1137 #ifdef __i386__
1138 struct AppleRTCHibernateVars
1139 {
1140 uint8_t signature[4];
1141 uint32_t revision;
1142 uint8_t booterSignature[20];
1143 uint8_t wiredCryptKey[16];
1144 };
1145 AppleRTCHibernateVars rtcVars;
1146
1147 rtcVars.signature[0] = 'A';
1148 rtcVars.signature[1] = 'A';
1149 rtcVars.signature[2] = 'P';
1150 rtcVars.signature[3] = 'L';
1151 rtcVars.revision = 1;
1152 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
1153 if (gIOHibernateBootSignature[0])
1154 {
1155 char c;
1156 uint8_t value = 0;
1157 for (uint32_t i = 0;
1158 (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
1159 i++)
1160 {
1161 if (c >= 'a')
1162 c -= 'a' - 10;
1163 else if (c >= 'A')
1164 c -= 'A' - 10;
1165 else if (c >= '0')
1166 c -= '0';
1167 else
1168 continue;
1169 value = (value << 4) | c;
1170 if (i & 1)
1171 rtcVars.booterSignature[i >> 1] = value;
1172 }
1173 }
1174 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
1175 if (data)
1176 {
1177 IOService::getPMRootDomain()->setProperty(kIOHibernateRTCVariablesKey, data);
1178 data->release();
1179 }
1180 if (gIOChosenEntry)
1181 {
1182 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
1183 if (data)
1184 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
1185 }
1186 #else /* !__i386__ */
1187 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1188 {
1189 data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1190 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
1191 if (sym && data)
1192 gIOOptionsEntry->setProperty(sym, data);
1193 if (sym)
1194 sym->release();
1195 if (data)
1196 data->release();
1197 if (gIOHibernateBootSignature[0])
1198 {
1199 data = OSData::withCapacity(16);
1200 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
1201 if (sym && data)
1202 {
1203 char c;
1204 uint8_t value = 0;
1205 for (uint32_t i = 0; (c = gIOHibernateBootSignature[i]); i++)
1206 {
1207 if (c >= 'a')
1208 c -= 'a' - 10;
1209 else if (c >= 'A')
1210 c -= 'A' - 10;
1211 else if (c >= '0')
1212 c -= '0';
1213 else
1214 continue;
1215 value = (value << 4) | c;
1216 if (i & 1)
1217 data->appendBytes(&value, sizeof(value));
1218 }
1219 gIOOptionsEntry->setProperty(sym, data);
1220 }
1221 if (sym)
1222 sym->release();
1223 if (data)
1224 data->release();
1225 }
1226 }
1227 if (!vars->haveFastBoot)
1228 {
1229 // set boot volume to zero
1230 IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
1231 if (platform && (kIOReturnSuccess == platform->readXPRAM(kXPRamAudioVolume,
1232 &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume))))
1233 {
1234 uint8_t newVolume;
1235 newVolume = vars->saveBootAudioVolume & 0xf8;
1236 platform->writeXPRAM(kXPRamAudioVolume,
1237 &newVolume, sizeof(newVolume));
1238 }
1239 }
1240 #endif /* !__i386__ */
1241 }
1242 // --
1243
1244 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
1245 gIOHibernateState = kIOHibernateStateHibernating;
1246 }
1247 while (false);
1248
1249 return (err);
1250 }
1251
1252 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1253
1254 IOReturn
1255 IOHibernateSystemHasSlept(void)
1256 {
1257 IOHibernateVars * vars = &gIOHibernateVars;
1258
1259 if ((vars->previewData = OSDynamicCast(OSData,
1260 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewBufferKey))))
1261 {
1262 vars->previewBuffer = IOMemoryDescriptor::withAddress(
1263 (void *) vars->previewData->getBytesNoCopy(),
1264 vars->previewData->getLength(),
1265 kIODirectionInOut);
1266
1267 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
1268 {
1269 vars->previewBuffer->release();
1270 vars->previewBuffer = 0;
1271 }
1272 if (!vars->previewBuffer)
1273 vars->previewData = 0;
1274 }
1275 if (gIOOptionsEntry)
1276 gIOOptionsEntry->sync();
1277
1278 return (kIOReturnSuccess);
1279 }
1280
1281 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1282
1283 IOReturn
1284 IOHibernateSystemWake(void)
1285 {
1286 IOHibernateVars * vars = &gIOHibernateVars;
1287
1288 hibernate_teardown(vars->page_list, vars->page_list_wired);
1289
1290 if (vars->videoMapping)
1291 {
1292 if (vars->videoMapSize)
1293 // remove mappings
1294 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1295 if (vars->videoAllocSize)
1296 // dealloc range
1297 kmem_free(kernel_map, trunc_page_32(vars->videoMapping), vars->videoAllocSize);
1298 }
1299
1300 if (vars->previewBuffer)
1301 {
1302 vars->previewBuffer->release();
1303 vars->previewBuffer = 0;
1304 }
1305
1306 if (vars->fileVars)
1307 {
1308 IOPolledFileClose(vars->fileVars);
1309 }
1310
1311 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1312
1313 #ifdef __ppc__
1314 OSData * data = OSData::withCapacity(4);
1315 if (gIOOptionsEntry && data)
1316 {
1317 const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1318 if (sym)
1319 {
1320 gIOOptionsEntry->setProperty(sym, data);
1321 sym->release();
1322 }
1323 sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
1324 if (sym)
1325 {
1326 if (vars->saveBootDevice)
1327 {
1328 gIOOptionsEntry->setProperty(sym, vars->saveBootDevice);
1329 vars->saveBootDevice->release();
1330 }
1331 sym->release();
1332 }
1333 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
1334 if (sym)
1335 {
1336 gIOOptionsEntry->setProperty(sym, data);
1337 sym->release();
1338 }
1339 sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
1340 if (sym)
1341 {
1342 gIOOptionsEntry->removeProperty(sym);
1343 sym->release();
1344 }
1345 }
1346 if (data)
1347 data->release();
1348
1349 if (gIOOptionsEntry)
1350 {
1351 if (!vars->haveFastBoot)
1352 {
1353 // reset boot audio volume
1354 IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
1355 if (platform)
1356 platform->writeXPRAM(kXPRamAudioVolume,
1357 &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume));
1358 }
1359
1360 // sync now to hardware if the booter has not
1361 if (kIOHibernateStateInactive == gIOHibernateState)
1362 gIOOptionsEntry->sync();
1363 else
1364 // just sync the variables in case a later panic syncs nvram (it won't sync variables)
1365 gIOOptionsEntry->syncOFVariables();
1366 }
1367 #endif
1368
1369 #ifdef __i386__
1370 IOService::getPMRootDomain()->removeProperty(kIOHibernateRTCVariablesKey);
1371 #endif
1372
1373 if (vars->srcBuffer)
1374 vars->srcBuffer->release();
1375 if (vars->ioBuffer)
1376 vars->ioBuffer->release();
1377 if (vars->fileExtents)
1378 vars->fileExtents->release();
1379
1380 bzero(vars, sizeof(*vars));
1381
1382 // gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1383
1384 return (kIOReturnSuccess);
1385 }
1386
1387 IOReturn
1388 IOHibernateSystemPostWake(void)
1389 {
1390 if (gIOHibernateFileRef)
1391 {
1392 // invalidate the image file
1393 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1394 int err = kern_write_file(gIOHibernateFileRef, 0,
1395 (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1396 if (KERN_SUCCESS != err)
1397 HIBLOG("kern_write_file(%d)\n", err);
1398
1399 kern_close_file_for_direct_io(gIOHibernateFileRef);
1400 gIOHibernateFileRef = 0;
1401 }
1402 return (kIOReturnSuccess);
1403 }
1404
1405 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1406
1407 void
1408 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1409 {
1410 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
1411 if (data)
1412 {
1413 rootDomain->setProperty(kIOHibernateStateKey, data);
1414 data->release();
1415 }
1416
1417 if (PE_parse_boot_arg("hfile", gIOHibernateFilename))
1418 gIOHibernateMode = kIOHibernateModeOn;
1419 else
1420 gIOHibernateFilename[0] = 0;
1421
1422 static SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
1423 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
1424 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
1425 sysctl_register_oid(&sysctl__kern_hibernatefile);
1426
1427 static SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
1428 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
1429 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
1430 sysctl_register_oid(&sysctl__kern_bootsignature);
1431
1432 static SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
1433 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
1434 &gIOHibernateMode, 0, "");
1435 sysctl_register_oid(&sysctl__kern_hibernatemode);
1436 }
1437
1438 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1439
1440 static void
1441 hibernate_setup_for_wake(void)
1442 {
1443 #if __ppc__
1444 // go slow (state needed for wake)
1445 ml_set_processor_speed(1);
1446 #endif
1447 }
1448
1449 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1450
1451 #define C_ASSERT(e) typedef char __C_ASSERT__[(e) ? 1 : -1]
1452
1453 extern "C" boolean_t
1454 hibernate_write_image(void)
1455 {
1456 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
1457 IOHibernateVars * vars = &gIOHibernateVars;
1458 IOPolledFileExtent * fileExtents;
1459
1460 C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
1461
1462 uint32_t pageCount, pagesDone;
1463 IOReturn err;
1464 vm_offset_t ppnum;
1465 IOItemCount page, count;
1466 uint8_t * src;
1467 uint8_t * data;
1468 IOByteCount pageCompressedSize;
1469 uint64_t compressedSize, uncompressedSize;
1470 uint64_t image1Size = 0;
1471 uint32_t bitmap_size;
1472 bool iterDone, pollerOpen, needEncryptStart;
1473 uint32_t restore1Sum, sum, sum1, sum2;
1474 uint32_t tag;
1475 uint32_t pageType;
1476 uint32_t pageAndCount[2];
1477
1478 AbsoluteTime startTime, endTime;
1479 AbsoluteTime allTime, compTime, decoTime;
1480 uint64_t nsec;
1481 uint32_t lastProgressStamp = 0;
1482 uint32_t progressStamp;
1483
1484 hibernate_cryptvars_t _cryptvars;
1485 hibernate_cryptvars_t * cryptvars = 0;
1486
1487 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
1488 return (false /* sleep */ );
1489
1490 restore1Sum = sum1 = sum2 = 0;
1491
1492 // encryption data. "iv" is the "initial vector".
1493 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1494 {
1495 static const unsigned char first_iv[AES_BLOCK_SIZE]
1496 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
1497 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
1498
1499 cryptvars = &gIOHibernateCryptWakeContext;
1500 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
1501 aes_encrypt_key(vars->cryptKey,
1502 kIOHibernateAESKeySize,
1503 &cryptvars->ctx.encrypt);
1504 aes_decrypt_key(vars->cryptKey,
1505 kIOHibernateAESKeySize,
1506 &cryptvars->ctx.decrypt);
1507
1508 cryptvars = &_cryptvars;
1509 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
1510 aes_encrypt_key(vars->wiredCryptKey,
1511 kIOHibernateAESKeySize,
1512 &cryptvars->ctx.encrypt);
1513
1514 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1515 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1516 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
1517 bzero(gIOHibernateCryptWakeVars, sizeof(hibernate_cryptwakevars_t));
1518 }
1519
1520 hibernate_setup_for_wake();
1521
1522 hibernate_page_list_setall(vars->page_list,
1523 vars->page_list_wired,
1524 &pageCount);
1525
1526 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
1527
1528 fileExtents = (IOPolledFileExtent *) vars->fileExtents->getBytesNoCopy();
1529
1530 #if 0
1531 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
1532 for (page = 0; page < count; page++)
1533 {
1534 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
1535 fileExtents[page].start, fileExtents[page].length,
1536 fileExtents[page].start + fileExtents[page].length);
1537 }
1538 #endif
1539
1540 needEncryptStart = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
1541
1542 AbsoluteTime_to_scalar(&compTime) = 0;
1543 AbsoluteTime_to_scalar(&decoTime) = 0;
1544
1545 clock_get_uptime(&allTime);
1546
1547 do
1548 {
1549 compressedSize = 0;
1550 uncompressedSize = 0;
1551 iterDone = false;
1552 pageType = 0; // wired pages first
1553
1554 IOPolledFileSeek(vars->fileVars, sizeof(IOHibernateImageHeader));
1555
1556 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
1557 ml_get_interrupts_enabled());
1558 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledBeforeSleepState, vars->ioBuffer);
1559 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
1560 pollerOpen = (kIOReturnSuccess == err);
1561 if (!pollerOpen)
1562 break;
1563
1564 // copy file block extent list if larger than header
1565
1566 count = vars->fileExtents->getLength();
1567 if (count > sizeof(header->fileExtentMap))
1568 {
1569 count -= sizeof(header->fileExtentMap);
1570 err = IOPolledFileWrite(vars->fileVars,
1571 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
1572 if (kIOReturnSuccess != err)
1573 break;
1574 }
1575
1576 // copy out restore1 code
1577
1578 page = atop_32(sectHIBB);
1579 count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
1580 header->restore1CodePage = page;
1581 header->restore1PageCount = count;
1582 header->restore1CodeOffset = ((uint32_t) &hibernate_machine_entrypoint) - sectHIBB;
1583 header->restore1StackOffset = ((uint32_t) &gIOHibernateRestoreStackEnd[0]) - 64 - sectHIBB;
1584
1585 // sum __HIB sect, with zeros for the stack
1586 src = (uint8_t *) trunc_page(sectHIBB);
1587 for (page = 0; page < count; page++)
1588 {
1589 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
1590 restore1Sum += hibernate_sum(src, page_size);
1591 else
1592 restore1Sum += 0x10000001;
1593 src += page_size;
1594 }
1595 sum1 = restore1Sum;
1596
1597 // write the __HIB sect, with zeros for the stack
1598
1599 src = (uint8_t *) trunc_page(sectHIBB);
1600 count = ((uint32_t) &gIOHibernateRestoreStack[0]) - trunc_page(sectHIBB);
1601 if (count)
1602 {
1603 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
1604 if (kIOReturnSuccess != err)
1605 break;
1606 }
1607 err = IOPolledFileWrite(vars->fileVars,
1608 (uint8_t *) 0,
1609 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
1610 cryptvars);
1611 if (kIOReturnSuccess != err)
1612 break;
1613 src = &gIOHibernateRestoreStackEnd[0];
1614 count = round_page(sectHIBB + sectSizeHIB) - ((uint32_t) src);
1615 if (count)
1616 {
1617 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
1618 if (kIOReturnSuccess != err)
1619 break;
1620 }
1621
1622 // write the preview buffer
1623
1624 addr64_t phys64;
1625 IOByteCount segLen;
1626
1627 if (vars->previewData)
1628 {
1629 ppnum = 0;
1630 count = 0;
1631 do
1632 {
1633 phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen);
1634 pageAndCount[0] = atop_64(phys64);
1635 pageAndCount[1] = atop_32(segLen);
1636 err = IOPolledFileWrite(vars->fileVars,
1637 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1638 cryptvars);
1639 if (kIOReturnSuccess != err)
1640 break;
1641 count += segLen;
1642 ppnum += sizeof(pageAndCount);
1643 }
1644 while (phys64);
1645 if (kIOReturnSuccess != err)
1646 break;
1647
1648 src = (uint8_t *) vars->previewData->getBytesNoCopy();
1649 count = vars->previewData->getLength();
1650
1651 header->previewPageListSize = ppnum;
1652 header->previewSize = count + ppnum;
1653
1654 for (page = 0; page < count; page += page_size)
1655 sum1 += hibernate_sum(src + page, page_size);
1656
1657 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
1658 if (kIOReturnSuccess != err)
1659 break;
1660 }
1661
1662 // mark areas for no save
1663
1664 for (count = 0;
1665 (phys64 = vars->ioBuffer->getPhysicalSegment64(count, &segLen));
1666 count += segLen)
1667 {
1668 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1669 atop_64(phys64), atop_32(segLen),
1670 kIOHibernatePageStateFree);
1671 pageCount -= atop_32(segLen);
1672 }
1673
1674 for (count = 0;
1675 (phys64 = vars->srcBuffer->getPhysicalSegment64(count, &segLen));
1676 count += segLen)
1677 {
1678 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1679 atop_64(phys64), atop_32(segLen),
1680 kIOHibernatePageStateFree);
1681 pageCount -= atop_32(segLen);
1682 }
1683
1684 // copy out bitmap of pages available for trashing during restore
1685
1686 bitmap_size = vars->page_list_wired->list_size;
1687 src = (uint8_t *) vars->page_list_wired;
1688 err = IOPolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
1689 if (kIOReturnSuccess != err)
1690 break;
1691
1692 // mark more areas for no save, but these are not available
1693 // for trashing during restore
1694
1695 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
1696
1697 page = atop_32(sectHIBB);
1698 count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
1699 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1700 page, count,
1701 kIOHibernatePageStateFree);
1702 pageCount -= count;
1703
1704 if (vars->previewBuffer) for (count = 0;
1705 (phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen));
1706 count += segLen)
1707 {
1708 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1709 atop_64(phys64), atop_32(segLen),
1710 kIOHibernatePageStateFree);
1711 pageCount -= atop_32(segLen);
1712 }
1713
1714 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
1715
1716 ppnum = 0;
1717 pagesDone = 0;
1718
1719 HIBLOG("writing %d pages\n", pageCount);
1720
1721 do
1722 {
1723 count = hibernate_page_list_iterate(pageType ? vars->page_list : vars->page_list_wired,
1724 &ppnum);
1725 // kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
1726
1727 iterDone = !count;
1728
1729 pageAndCount[0] = ppnum;
1730 pageAndCount[1] = count;
1731 err = IOPolledFileWrite(vars->fileVars,
1732 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1733 cryptvars);
1734 if (kIOReturnSuccess != err)
1735 break;
1736
1737 for (page = 0; page < count; page++)
1738 {
1739 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(ppnum), page_size);
1740 if (err)
1741 {
1742 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%d] %x\n", __LINE__, ppnum, err);
1743 break;
1744 }
1745
1746 sum = hibernate_sum(src, page_size);
1747
1748 clock_get_uptime(&startTime);
1749
1750 pageCompressedSize = WKdm_compress ((WK_word*) src, (WK_word*) (src + page_size), PAGE_SIZE_IN_WORDS);
1751
1752 clock_get_uptime(&endTime);
1753 ADD_ABSOLUTETIME(&compTime, &endTime);
1754 SUB_ABSOLUTETIME(&compTime, &startTime);
1755
1756 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1757 pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
1758
1759 if (pageCompressedSize > page_size)
1760 {
1761 // HIBLOG("------------lose: %d\n", pageCompressedSize);
1762 pageCompressedSize = page_size;
1763 }
1764
1765 if (pageCompressedSize != page_size)
1766 data = (src + page_size);
1767 else
1768 data = src;
1769
1770 tag = pageCompressedSize | kIOHibernateTagSignature;
1771
1772 if (pageType)
1773 sum2 += sum;
1774 else
1775 sum1 += sum;
1776
1777 if (needEncryptStart && (ppnum >= atop_32(sectDATAB)))
1778 {
1779 // start encrypting partway into the data about to be written
1780 vars->fileVars->encryptStart = (vars->fileVars->position + AES_BLOCK_SIZE - 1)
1781 & ~(AES_BLOCK_SIZE - 1);
1782 needEncryptStart = false;
1783 }
1784
1785 err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
1786 if (kIOReturnSuccess != err)
1787 break;
1788
1789 err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
1790 if (kIOReturnSuccess != err)
1791 break;
1792
1793 compressedSize += pageCompressedSize;
1794 if (pageCompressedSize)
1795 uncompressedSize += page_size;
1796 ppnum++;
1797 pagesDone++;
1798
1799 if (0 == (8191 & pagesDone))
1800 {
1801 clock_get_uptime(&endTime);
1802 SUB_ABSOLUTETIME(&endTime, &allTime);
1803 absolutetime_to_nanoseconds(endTime, &nsec);
1804 progressStamp = nsec / 750000000ULL;
1805 if (progressStamp != lastProgressStamp)
1806 {
1807 lastProgressStamp = progressStamp;
1808 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
1809 }
1810 }
1811 }
1812 if (kIOReturnSuccess != err)
1813 break;
1814 if (iterDone && !pageType)
1815 {
1816 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
1817 if (kIOReturnSuccess != err)
1818 break;
1819
1820 iterDone = false;
1821 pageType = 1;
1822 ppnum = 0;
1823 image1Size = vars->fileVars->position;
1824 if (cryptvars)
1825 {
1826 bcopy(&cryptvars->aes_iv[0],
1827 &gIOHibernateCryptWakeContext.aes_iv[0],
1828 sizeof(cryptvars->aes_iv));
1829 cryptvars = &gIOHibernateCryptWakeContext;
1830 }
1831 HIBLOG("image1Size %qd\n", image1Size);
1832 }
1833 }
1834 while (!iterDone);
1835 if (kIOReturnSuccess != err)
1836 break;
1837 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
1838 if (kIOReturnSuccess != err)
1839 break;
1840
1841 // Header:
1842
1843 header->imageSize = vars->fileVars->position;
1844 header->image1Size = image1Size;
1845 header->bitmapSize = bitmap_size;
1846 header->pageCount = pageCount;
1847 header->encryptStart = vars->fileVars->encryptStart;
1848
1849 header->restore1Sum = restore1Sum;
1850 header->image1Sum = sum1;
1851 header->image2Sum = sum2;
1852
1853 count = vars->fileExtents->getLength();
1854 if (count > sizeof(header->fileExtentMap))
1855 {
1856 header->fileExtentMapSize = count;
1857 count = sizeof(header->fileExtentMap);
1858 }
1859 else
1860 header->fileExtentMapSize = sizeof(header->fileExtentMap);
1861 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
1862
1863 IOPolledFileSeek(vars->fileVars, 0);
1864 err = IOPolledFileWrite(vars->fileVars,
1865 (uint8_t *) header, sizeof(IOHibernateImageHeader),
1866 cryptvars);
1867 if (kIOReturnSuccess != err)
1868 break;
1869 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
1870 if (kIOReturnSuccess != err)
1871 break;
1872 err = IOHibernatePollerIODone(vars->fileVars);
1873 if (kIOReturnSuccess != err)
1874 break;
1875 }
1876 while (false);
1877
1878 clock_get_uptime(&endTime);
1879 SUB_ABSOLUTETIME(&endTime, &allTime);
1880 absolutetime_to_nanoseconds(endTime, &nsec);
1881 HIBLOG("all time: %qd ms, ",
1882 nsec / 1000000ULL);
1883
1884 absolutetime_to_nanoseconds(compTime, &nsec);
1885 HIBLOG("comp time: %qd ms, ",
1886 nsec / 1000000ULL);
1887
1888 absolutetime_to_nanoseconds(decoTime, &nsec);
1889 HIBLOG("deco time: %qd ms, ",
1890 nsec / 1000000ULL);
1891
1892 HIBLOG("\nimage %qd, uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
1893 header->imageSize,
1894 uncompressedSize, atop_32(uncompressedSize), compressedSize,
1895 (int) ((compressedSize * 100ULL) / uncompressedSize),
1896 sum1, sum2);
1897
1898 if (pollerOpen)
1899 IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
1900
1901 HIBLOG("hibernate_write_image done(%x)\n", err);
1902
1903 // should we come back via regular wake, set the state in memory.
1904 gIOHibernateState = kIOHibernateStateInactive;
1905
1906 if ((kIOReturnSuccess == err) && !(kIOHibernateModeSleep & gIOHibernateMode))
1907 return (true /* power down */ );
1908 else
1909 return (false /* sleep */ );
1910 }
1911
1912 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1913
1914 DECLARE_IOHIBERNATEPROGRESSALPHA
1915
1916 static void
1917 ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
1918 {
1919 uint32_t rowBytes, pixelShift;
1920 uint32_t x, y;
1921 int32_t blob, lastBlob;
1922 uint32_t alpha, in, color, result;
1923 uint8_t * out;
1924 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1925
1926 pixelShift = display->depth >> 4;
1927 if (pixelShift < 1)
1928 return;
1929
1930 rowBytes = display->rowBytes;
1931
1932 screen += ((display->width
1933 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1934 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1935
1936 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
1937
1938 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
1939
1940 for (y = 0; y < kIOHibernateProgressHeight; y++)
1941 {
1942 out = screen + y * rowBytes;
1943 for (blob = firstBlob; blob <= lastBlob; blob++)
1944 {
1945 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
1946 for (x = 0; x < kIOHibernateProgressWidth; x++)
1947 {
1948 alpha = gIOHibernateProgressAlpha[y][x];
1949 result = color;
1950 if (alpha)
1951 {
1952 if (0xff != alpha)
1953 {
1954 in = display->progressSaveUnder[blob][saveindex[blob]++];
1955 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
1956 }
1957 if (1 == pixelShift)
1958 {
1959 result >>= 3;
1960 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1961 }
1962 else
1963 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1964 }
1965 out += (1 << pixelShift);
1966 }
1967 out += (kIOHibernateProgressSpacing << pixelShift);
1968 }
1969 }
1970 }
1971
1972 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1973
1974 extern "C" void
1975 hibernate_machine_init(void)
1976 {
1977 IOReturn err;
1978 uint32_t sum;
1979 uint32_t pagesDone;
1980 AbsoluteTime allTime, endTime;
1981 uint64_t nsec;
1982 uint32_t lastProgressStamp = 0;
1983 uint32_t progressStamp;
1984 uint64_t progressZeroPosition = 0;
1985 uint32_t blob, lastBlob = (uint32_t) -1L;
1986 hibernate_cryptvars_t * cryptvars = 0;
1987
1988 IOHibernateVars * vars = &gIOHibernateVars;
1989
1990 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
1991 return;
1992
1993 sum = gIOHibernateCurrentHeader->actualImage1Sum;
1994 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
1995
1996 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, image1Size %qx, conflictCount %d, nextFree %x\n",
1997 gIOHibernateState, pagesDone, sum, gIOHibernateCurrentHeader->image1Size,
1998 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
1999
2000 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
2001 {
2002 HIBLOG("regular wake\n");
2003 return;
2004 }
2005
2006 HIBPRINT("diag %x %x %x %x\n",
2007 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
2008 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
2009
2010 HIBPRINT("video %x %d %d %d\n",
2011 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2012 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height);
2013
2014 if ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)
2015 hibernate_page_list_discard(vars->page_list);
2016
2017 boot_args *args = (boot_args *) PE_state.bootArgs;
2018
2019 if (vars->videoMapping
2020 && gIOHibernateGraphicsInfo->physicalAddress
2021 && (args->Video.v_baseAddr == gIOHibernateGraphicsInfo->physicalAddress))
2022 {
2023 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2024 * gIOHibernateGraphicsInfo->rowBytes);
2025 IOMapPages(kernel_map,
2026 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2027 vars->videoMapSize, kIOMapInhibitCache );
2028 }
2029
2030 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2031
2032 if (gIOHibernateWakeMapSize)
2033 {
2034 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(gIOHibernateWakeMap),
2035 gIOHibernateWakeMapSize);
2036 if (kIOReturnSuccess == err)
2037 hibernate_newruntime_map(src, gIOHibernateWakeMapSize,
2038 gIOHibernateCurrentHeader->systemTableOffset);
2039 gIOHibernateWakeMap = 0;
2040 gIOHibernateWakeMapSize = 0;
2041 }
2042
2043 uint32_t decoOffset;
2044
2045 clock_get_uptime(&allTime);
2046
2047 HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2048 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0);
2049 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
2050
2051 if (gIOHibernateCurrentHeader->previewSize)
2052 progressZeroPosition = gIOHibernateCurrentHeader->previewSize
2053 + gIOHibernateCurrentHeader->fileExtentMapSize
2054 - sizeof(gIOHibernateCurrentHeader->fileExtentMap)
2055 + ptoa_64(gIOHibernateCurrentHeader->restore1PageCount);
2056
2057 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2058
2059 if (vars->videoMapping)
2060 {
2061 lastBlob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
2062 / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
2063 ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, 0, lastBlob);
2064 }
2065
2066 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
2067 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2068 {
2069 cryptvars = &gIOHibernateCryptWakeContext;
2070 bcopy(&gIOHibernateCryptWakeVars->aes_iv[0],
2071 &cryptvars->aes_iv[0],
2072 sizeof(cryptvars->aes_iv));
2073 }
2074
2075 // kick off the read ahead
2076 vars->fileVars->io = false;
2077 vars->fileVars->bufferHalf = 0;
2078 vars->fileVars->bufferLimit = 0;
2079 vars->fileVars->lastRead = 0;
2080 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2081
2082 IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
2083 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2084 // --
2085
2086 HIBLOG("hibernate_machine_init reading\n");
2087
2088 uint32_t * header = (uint32_t *) src;
2089 sum = 0;
2090
2091 do
2092 {
2093 unsigned int count;
2094 unsigned int page;
2095 uint32_t tag;
2096 vm_offset_t ppnum, compressedSize;
2097
2098 IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2099
2100 ppnum = header[0];
2101 count = header[1];
2102
2103 // HIBPRINT("(%x, %x)\n", ppnum, count);
2104
2105 if (!count)
2106 break;
2107
2108 for (page = 0; page < count; page++)
2109 {
2110 IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2111
2112 compressedSize = kIOHibernateTagLength & tag;
2113 if (!compressedSize)
2114 {
2115 ppnum++;
2116 pagesDone++;
2117 continue;
2118 }
2119
2120 IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2121
2122 if (compressedSize != page_size)
2123 {
2124 decoOffset = page_size;
2125 WKdm_decompress((WK_word*) src, (WK_word*) (src + decoOffset), PAGE_SIZE_IN_WORDS);
2126 }
2127 else
2128 decoOffset = 0;
2129
2130 sum += hibernate_sum((src + decoOffset), page_size);
2131
2132 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2133 if (err)
2134 HIBLOG("IOMemoryDescriptorReadToPhysical [%d] %x\n", ppnum, err);
2135
2136 ppnum++;
2137 pagesDone++;
2138
2139 if (vars->videoMapping && (0 == (255 & pagesDone)))
2140 {
2141 blob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
2142 / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
2143 if (blob != lastBlob)
2144 {
2145 ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, lastBlob, blob);
2146 lastBlob = blob;
2147 }
2148 }
2149
2150 if (0 == (8191 & pagesDone))
2151 {
2152 clock_get_uptime(&endTime);
2153 SUB_ABSOLUTETIME(&endTime, &allTime);
2154 absolutetime_to_nanoseconds(endTime, &nsec);
2155 progressStamp = nsec / 750000000ULL;
2156 if (progressStamp != lastProgressStamp)
2157 {
2158 lastProgressStamp = progressStamp;
2159 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2160 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2161 }
2162 }
2163 }
2164 }
2165 while (true);
2166
2167 gIOHibernateCurrentHeader->actualImage2Sum = sum;
2168
2169 if (vars->fileVars->io)
2170 (void) IOHibernatePollerIODone(vars->fileVars);
2171
2172 err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
2173
2174 if (vars->videoMapping)
2175 ProgressUpdate(gIOHibernateGraphicsInfo,
2176 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2177
2178 clock_get_uptime(&endTime);
2179 SUB_ABSOLUTETIME(&endTime, &allTime);
2180 absolutetime_to_nanoseconds(endTime, &nsec);
2181
2182 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %qd ms\n",
2183 pagesDone, sum, nsec / 1000000ULL);
2184 }
2185
2186 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
mirror of XNU