]> git.saurik.com Git - apple/xnu.git/blob - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blob
? search:


030368a72f658f89d554758773eb209f515db31a
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
1 /*
2 * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29
30 /*
31
32 Sleep:
33
34 - PMRootDomain calls IOHibernateSystemSleep() before system sleep
35 (devices awake, normal execution context)
36 - IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
37 grabs its extents and searches for a polling driver willing to work with that IOMedia.
38 The BSD code makes an ioctl to the storage driver to get the partition base offset to
39 the disk, and other ioctls to get the transfer constraints
40 If successful, the file is written to make sure its initially not bootable (in case of
41 later failure) and nvram set to point to the first block of the file. (Has to be done
42 here so blocking is possible in nvram support).
43 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
44 page out any pages it wants to (currently zero, but probably some percentage of memory).
45 Its assumed just allocating pages will cause the VM system to naturally select the best
46 pages for eviction. It also copies processor flags needed for the restore path and sets
47 a flag in the boot processor proc info.
48 gIOHibernateState = kIOHibernateStateHibernating.
49 - Regular sleep progresses - some drivers may inspect the root domain property
50 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
51 as usual but leaves motherboard I/O on.
52 - Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
53 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
54 all ppc RC bits out of the hash table and caches into the mapping structures.
55 - hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
56 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
57 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
58 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
59 The image header and block list are written. The header includes the second file extent so
60 only the header block is needed to read the file, regardless of filesystem.
61 The kernel section "__HIB" is written uncompressed to the image. This section of code and data
62 (only) is used to decompress the image during wake/boot.
63 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
64 The bitmaps are written to the image.
65 More areas are removed from the bitmaps (after they have been written to the image) - the
66 section "__HIB" pages and interrupt stack.
67 Each wired page is compressed and written and then each non-wired page. Compression and
68 disk writes are in parallel.
69 The image header is written to the start of the file and the polling driver closed.
70 The machine powers down (or sleeps).
71
72 Boot/Wake:
73
74 - BootX sees the boot-image nvram variable containing the device and block number of the image,
75 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
76 - BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
77 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
78 that is in the kernel's __HIB section.
79 - BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
80 hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
81 only code & data in that section is safe to call since all the other wired pages are still
82 compressed in the image.
83 - hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
84 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
85 location directly, and copies those that can't to interim free pages. When the image has been
86 completed, the copies are uncompressed, overwriting the wired image pages.
87 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
88 is used to get pages into place for 64bit.
89 - the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
90 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
91 are removed from the software strutures, and the hash table is reinitialized.
92 - After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
93 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
94 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
95 for the remaining non wired pages.
96 - The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
97 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
98 is closed via bsd.
99
100 Polled Mode I/O:
101
102 IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
103 registry, specifying an object of calls IOPolledInterface.
104
105 Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
106 partition) that the image is going to live, looking for polled interface properties. If it finds
107 one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
108 interfaces found are kept in an ordered list.
109
110 There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
111 few different contexts things happen in:
112
113 - there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
114 up and running) and after wake - this is safe to allocate memory and do anything. The device
115 ignores sleep requests from that point since its a waste of time if it goes to sleep and
116 immediately wakes back up for the image write.
117
118 - there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
119 immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
120 from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
121 used to flush and set the disk to sleep.
122
123 - there is an Open/Close (AfterSleep) pair made around the image read operations that happen
124 immediately after sleep. These can't block or allocate memory. This is happening after the platform
125 expert has woken the low level bits of the system, but most of the I/O system has not. There is only
126 one thread running.
127
128 For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
129 (prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
130 the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
131 Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
132 that is called for the hardware to check for events, and complete the I/O via the callback.
133 The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
134 to restrict I/O ops.
135 */
136
137 #include <sys/systm.h>
138
139 #include <IOKit/IOWorkLoop.h>
140 #include <IOKit/IOCommandGate.h>
141 #include <IOKit/IOTimerEventSource.h>
142 #include <IOKit/IOPlatformExpert.h>
143 #include <IOKit/IOKitDebug.h>
144 #include <IOKit/IOTimeStamp.h>
145 #include <IOKit/pwr_mgt/RootDomain.h>
146 #include <IOKit/pwr_mgt/IOPMPrivate.h>
147 #include <IOKit/IOMessage.h>
148 #include <IOKit/IODeviceTreeSupport.h>
149 #include <IOKit/IOBSD.h>
150 #include "RootDomainUserClient.h"
151 #include <IOKit/pwr_mgt/IOPowerConnection.h>
152 #include "IOPMPowerStateQueue.h"
153 #include <IOKit/IOBufferMemoryDescriptor.h>
154 #include <crypto/aes.h>
155
156 #include <sys/uio.h>
157 #include <sys/conf.h>
158 #include <sys/stat.h>
159 #include <sys/fcntl.h> // (FWRITE, ...)
160 #include <sys/sysctl.h>
161
162 #include <IOKit/IOHibernatePrivate.h>
163 #include <IOKit/IOPolledInterface.h>
164 #include <IOKit/IONVRAM.h>
165 #include "IOHibernateInternal.h"
166 #include "WKdm.h"
167 #include "IOKitKernelInternal.h"
168
169 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
170
171 OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
172
173 OSMetaClassDefineReservedUnused(IOPolledInterface, 0);
174 OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
175 OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
176 OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
177 OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
178 OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
179 OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
180 OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
181 OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
182 OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
183 OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
184 OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
185 OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
186 OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
187 OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
188 OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
189
190 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
191
192 extern uint32_t gIOHibernateState;
193 uint32_t gIOHibernateMode;
194 static char gIOHibernateBootSignature[256+1];
195 static char gIOHibernateFilename[MAXPATHLEN+1];
196 static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
197 uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
198
199 static IODTNVRAM * gIOOptionsEntry;
200 static IORegistryEntry * gIOChosenEntry;
201 #ifdef __i386__
202 static const OSSymbol * gIOCreateEFIDevicePathSymbol;
203 #endif
204
205 static IOPolledFileIOVars gFileVars;
206 static IOHibernateVars gIOHibernateVars;
207 static struct kern_direct_file_io_ref_t * gIOHibernateFileRef;
208 static hibernate_cryptvars_t gIOHibernateCryptWakeContext;
209
210 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
211
212 enum { kXPRamAudioVolume = 8 };
213 enum { kDefaultIOSize = 128 * 1024 };
214 enum { kVideoMapSize = 32 * 1024 * 1024 };
215
216 #ifndef kIOMediaPreferredBlockSizeKey
217 #define kIOMediaPreferredBlockSizeKey "Preferred Block Size"
218 #endif
219
220 #ifndef kIOBootPathKey
221 #define kIOBootPathKey "bootpath"
222 #endif
223 #ifndef kIOSelectedBootDeviceKey
224 #define kIOSelectedBootDeviceKey "boot-device"
225 #endif
226
227
228 enum { kIOHibernateMinPollersNeeded = 2 };
229
230 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
231
232 // copy from phys addr to MD
233
234 static IOReturn
235 IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
236 IOByteCount offset, addr64_t bytes, IOByteCount length)
237 {
238 addr64_t srcAddr = bytes;
239 IOByteCount remaining;
240
241 remaining = length = min(length, md->getLength() - offset);
242 while (remaining) { // (process another target segment?)
243 addr64_t dstAddr64;
244 IOByteCount dstLen;
245
246 dstAddr64 = md->getPhysicalSegment64(offset, &dstLen);
247 if (!dstAddr64)
248 break;
249
250 // Clip segment length to remaining
251 if (dstLen > remaining)
252 dstLen = remaining;
253
254 #if 1
255 bcopy_phys(srcAddr, dstAddr64, dstLen);
256 #else
257 copypv(srcAddr, dstAddr64, dstLen,
258 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
259 #endif
260 srcAddr += dstLen;
261 offset += dstLen;
262 remaining -= dstLen;
263 }
264
265 assert(!remaining);
266
267 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
268 }
269
270 // copy from MD to phys addr
271
272 static IOReturn
273 IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
274 IOByteCount offset, addr64_t bytes, IOByteCount length)
275 {
276 addr64_t dstAddr = bytes;
277 IOByteCount remaining;
278
279 remaining = length = min(length, md->getLength() - offset);
280 while (remaining) { // (process another target segment?)
281 addr64_t srcAddr64;
282 IOByteCount dstLen;
283
284 srcAddr64 = md->getPhysicalSegment64(offset, &dstLen);
285 if (!srcAddr64)
286 break;
287
288 // Clip segment length to remaining
289 if (dstLen > remaining)
290 dstLen = remaining;
291
292 #if 1
293 bcopy_phys(srcAddr64, dstAddr, dstLen);
294 #else
295 copypv(srcAddr, dstAddr64, dstLen,
296 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
297 #endif
298 dstAddr += dstLen;
299 offset += dstLen;
300 remaining -= dstLen;
301 }
302
303 assert(!remaining);
304
305 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
306 }
307
308 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
309
310 void
311 hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
312 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
313 {
314 count += ppnum;
315 switch (kind)
316 {
317 case kIOHibernatePageStateUnwiredSave:
318 // unwired save
319 for (; ppnum < count; ppnum++)
320 {
321 hibernate_page_bitset(page_list, FALSE, ppnum);
322 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
323 }
324 break;
325 case kIOHibernatePageStateWiredSave:
326 // wired save
327 for (; ppnum < count; ppnum++)
328 {
329 hibernate_page_bitset(page_list, FALSE, ppnum);
330 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
331 }
332 break;
333 case kIOHibernatePageStateFree:
334 // free page
335 for (; ppnum < count; ppnum++)
336 {
337 hibernate_page_bitset(page_list, TRUE, ppnum);
338 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
339 }
340 break;
341 default:
342 panic("hibernate_set_page_state");
343 }
344 }
345
346 static vm_offset_t
347 hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
348 {
349 uint32_t page = *pPage;
350 uint32_t count;
351 hibernate_bitmap_t * bitmap;
352
353 while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
354 {
355 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
356 if (!count)
357 break;
358 page += count;
359 if (page <= bitmap->last_page)
360 break;
361 }
362
363 *pPage = page;
364 if (bitmap)
365 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
366 else
367 count = 0;
368
369 return (count);
370 }
371
372 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
373
374 static IOReturn
375 IOHibernatePollerProbe(IOPolledFileIOVars * vars, IOService * target)
376 {
377 IOReturn err = kIOReturnError;
378 int32_t idx;
379 IOPolledInterface * poller;
380
381 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
382 {
383 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
384 err = poller->probe(target);
385 if (err)
386 {
387 HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
388 break;
389 }
390 }
391
392 return (err);
393 }
394
395 static IOReturn
396 IOHibernatePollerOpen(IOPolledFileIOVars * vars, uint32_t state, IOMemoryDescriptor * md)
397 {
398 IOReturn err = kIOReturnError;
399 int32_t idx;
400 IOPolledInterface * poller;
401
402 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
403 {
404 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
405 err = poller->open(state, md);
406 if (err)
407 {
408 HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
409 break;
410 }
411 }
412
413 return (err);
414 }
415
416 static IOReturn
417 IOHibernatePollerClose(IOPolledFileIOVars * vars, uint32_t state)
418 {
419 IOReturn err = kIOReturnError;
420 int32_t idx;
421 IOPolledInterface * poller;
422
423 for (idx = 0;
424 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
425 idx++)
426 {
427 err = poller->close(state);
428 if (err)
429 HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
430 }
431
432 return (err);
433 }
434
435 static void
436 IOHibernatePollerIOComplete(void * target,
437 void * parameter,
438 IOReturn status,
439 UInt64 actualByteCount)
440 {
441 IOPolledFileIOVars * vars = (IOPolledFileIOVars *) parameter;
442
443 vars->ioStatus = status;
444 }
445
446 static IOReturn
447 IOHibernatePollerIO(IOPolledFileIOVars * vars,
448 uint32_t operation, uint32_t bufferOffset,
449 uint64_t deviceOffset, uint64_t length)
450 {
451
452 IOReturn err = kIOReturnError;
453 IOPolledInterface * poller;
454 IOPolledCompletion completion;
455
456 completion.target = 0;
457 completion.action = &IOHibernatePollerIOComplete;
458 completion.parameter = vars;
459
460 vars->ioStatus = -1;
461
462 poller = (IOPolledInterface *) vars->pollers->getObject(0);
463 err = poller->startIO(operation, bufferOffset, deviceOffset + vars->block0, length, completion);
464 if (err)
465 HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
466
467 return (err);
468 }
469
470 static IOReturn
471 IOHibernatePollerIODone(IOPolledFileIOVars * vars, bool abortable)
472 {
473 IOReturn err = kIOReturnSuccess;
474 int32_t idx = 0;
475 IOPolledInterface * poller;
476
477 while (-1 == vars->ioStatus)
478 {
479 for (idx = 0;
480 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
481 idx++)
482 {
483 IOReturn newErr;
484 newErr = poller->checkForWork();
485 if ((newErr == kIOReturnAborted) && !abortable)
486 newErr = kIOReturnSuccess;
487 if (kIOReturnSuccess == err)
488 err = newErr;
489 }
490 }
491
492 if (err)
493 {
494 HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
495 }
496 else
497 {
498 err = vars->ioStatus;
499 if (kIOReturnSuccess != err)
500 HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
501 }
502
503 return (err);
504 }
505
506 IOReturn
507 IOPolledInterface::checkAllForWork(void)
508 {
509 IOReturn err = kIOReturnNotReady;
510 int32_t idx;
511 IOPolledInterface * poller;
512
513 IOHibernateVars * vars = &gIOHibernateVars;
514
515 if (!vars->fileVars || !vars->fileVars->pollers)
516 return (err);
517
518 for (idx = 0;
519 (poller = (IOPolledInterface *) vars->fileVars->pollers->getObject(idx));
520 idx++)
521 {
522 err = poller->checkForWork();
523 if (err)
524 HIBLOG("IOPolledInterface::checkAllForWork[%d] 0x%x\n", idx, err);
525 }
526
527 return (err);
528 }
529
530 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
531
532 struct _OpenFileContext
533 {
534 OSData * extents;
535 uint64_t size;
536 };
537
538 static void
539 file_extent_callback(void * ref, uint64_t start, uint64_t length)
540 {
541 _OpenFileContext * ctx = (_OpenFileContext *) ref;
542 IOPolledFileExtent extent;
543
544 extent.start = start;
545 extent.length = length;
546
547 ctx->extents->appendBytes(&extent, sizeof(extent));
548 ctx->size += length;
549 }
550
551 IOReturn
552 IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
553 IOPolledFileIOVars ** fileVars, OSData ** fileExtents,
554 OSData ** imagePath)
555 {
556 IOReturn err = kIOReturnError;
557 IOPolledFileIOVars * vars;
558 _OpenFileContext ctx;
559 OSData * extentsData;
560 OSNumber * num;
561 IORegistryEntry * part = 0;
562 OSDictionary * matching;
563 OSIterator * iter;
564 dev_t hibernate_image_dev;
565 uint64_t maxiobytes;
566
567 vars = &gFileVars;
568 do
569 {
570 HIBLOG("sizeof(IOHibernateImageHeader) == %ld\n", sizeof(IOHibernateImageHeader));
571 if (sizeof(IOHibernateImageHeader) != 512)
572 continue;
573
574 vars->io = false;
575 vars->buffer = (uint8_t *) ioBuffer->getBytesNoCopy();
576 vars->bufferHalf = 0;
577 vars->bufferOffset = 0;
578 vars->bufferSize = ioBuffer->getLength() >> 1;
579
580 extentsData = OSData::withCapacity(32);
581
582 ctx.extents = extentsData;
583 ctx.size = 0;
584 vars->fileRef = kern_open_file_for_direct_io(filename,
585 &file_extent_callback, &ctx,
586 &hibernate_image_dev,
587 &vars->block0,
588 &maxiobytes);
589 if (!vars->fileRef)
590 {
591 err = kIOReturnNoSpace;
592 break;
593 }
594 HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx\n", filename, ctx.size,
595 vars->block0, maxiobytes);
596 if (ctx.size < 1*1024*1024) // check against image size estimate!
597 {
598 err = kIOReturnNoSpace;
599 break;
600 }
601
602 if (maxiobytes < vars->bufferSize)
603 vars->bufferSize = maxiobytes;
604
605 vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
606
607 matching = IOService::serviceMatching("IOMedia");
608 num = OSNumber::withNumber(major(hibernate_image_dev), 32);
609 matching->setObject(kIOBSDMajorKey, num);
610 num->release();
611 num = OSNumber::withNumber(minor(hibernate_image_dev), 32);
612 matching->setObject(kIOBSDMinorKey, num);
613 num->release();
614 iter = IOService::getMatchingServices(matching);
615 matching->release();
616 if (iter)
617 {
618 part = (IORegistryEntry *) iter->getNextObject();
619 part->retain();
620 iter->release();
621 }
622 if (!part)
623 break;
624
625 int minor, major;
626 IORegistryEntry * next;
627 IORegistryEntry * child;
628 OSData * data;
629
630 num = (OSNumber *) part->getProperty(kIOBSDMajorKey);
631 if (!num)
632 break;
633 major = num->unsigned32BitValue();
634 num = (OSNumber *) part->getProperty(kIOBSDMinorKey);
635 if (!num)
636 break;
637 minor = num->unsigned32BitValue();
638
639 hibernate_image_dev = makedev(major, minor);
640
641 vars->pollers = OSArray::withCapacity(4);
642 if (!vars->pollers)
643 break;
644
645 vars->blockSize = 512;
646 next = part;
647 do
648 {
649 IOPolledInterface * poller;
650 OSObject * obj;
651
652 obj = next->getProperty(kIOPolledInterfaceSupportKey);
653 if (kOSBooleanFalse == obj)
654 {
655 vars->pollers->flushCollection();
656 break;
657 }
658 else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
659 vars->pollers->setObject(poller);
660 if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey))))
661 vars->blockSize = num->unsigned32BitValue();
662 child = next;
663 }
664 while ((next = child->getParentEntry(gIOServicePlane))
665 && child->isParent(next, gIOServicePlane, true));
666
667 HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
668 major, minor, vars->blockSize, vars->pollers->getCount());
669 if (vars->pollers->getCount() < kIOHibernateMinPollersNeeded)
670 continue;
671
672 err = IOHibernatePollerProbe(vars, (IOService *) part);
673 if (kIOReturnSuccess != err)
674 break;
675
676 err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer);
677 if (kIOReturnSuccess != err)
678 break;
679
680 *fileVars = vars;
681 *fileExtents = extentsData;
682
683 // make imagePath
684
685 if ((extentsData->getLength() >= sizeof(IOPolledFileExtent)))
686 {
687 char str2[24];
688
689 #if __i386__
690 if (!gIOCreateEFIDevicePathSymbol)
691 gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
692
693 snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
694
695 err = IOService::getPlatform()->callPlatformFunction(
696 gIOCreateEFIDevicePathSymbol, false,
697 (void *) part, (void *) str2, (void *) true,
698 (void *) &data);
699 #else
700 char str1[256];
701 int len = sizeof(str1);
702
703 if (!part->getPath(str1, &len, gIODTPlane))
704 err = kIOReturnNotFound;
705 else
706 {
707 snprintf(str2, sizeof(str2), ",%qx", vars->extentMap[0].start);
708 // (strip the plane name)
709 char * tail = strchr(str1, ':');
710 if (!tail)
711 tail = str1 - 1;
712 data = OSData::withBytes(tail + 1, strlen(tail + 1));
713 data->appendBytes(str2, strlen(str2));
714 }
715 #endif
716 if (kIOReturnSuccess == err)
717 *imagePath = data;
718 else
719 HIBLOG("error 0x%x getting path\n", err);
720 }
721 }
722 while (false);
723
724 if (kIOReturnSuccess != err)
725 {
726 HIBLOG("error 0x%x opening hibernation file\n", err);
727 if (vars->fileRef)
728 kern_close_file_for_direct_io(vars->fileRef);
729 }
730
731 if (part)
732 part->release();
733
734 return (err);
735 }
736
737 IOReturn
738 IOPolledFileClose( IOPolledFileIOVars * vars )
739 {
740 if (vars->pollers)
741 {
742 IOHibernatePollerClose(vars, kIOPolledPostflightState);
743 vars->pollers->release();
744 }
745
746 gIOHibernateFileRef = vars->fileRef;
747
748 bzero(vars, sizeof(IOPolledFileIOVars));
749
750 return (kIOReturnSuccess);
751 }
752
753 static IOReturn
754 IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
755 {
756 IOPolledFileExtent * extentMap;
757
758 extentMap = vars->extentMap;
759
760 vars->position = position;
761
762 while (position >= extentMap->length)
763 {
764 position -= extentMap->length;
765 extentMap++;
766 }
767
768 vars->currentExtent = extentMap;
769 vars->extentRemaining = extentMap->length - position;
770 vars->extentPosition = vars->position - position;
771
772 if (vars->bufferSize <= vars->extentRemaining)
773 vars->bufferLimit = vars->bufferSize;
774 else
775 vars->bufferLimit = vars->extentRemaining;
776
777 return (kIOReturnSuccess);
778 }
779
780 static IOReturn
781 IOPolledFileWrite(IOPolledFileIOVars * vars,
782 const uint8_t * bytes, IOByteCount size,
783 hibernate_cryptvars_t * cryptvars)
784 {
785 IOReturn err = kIOReturnSuccess;
786 IOByteCount copy;
787 bool flush = false;
788
789 do
790 {
791 if (!bytes && !size)
792 {
793 // seek to end of block & flush
794 size = vars->position & (vars->blockSize - 1);
795 if (size)
796 size = vars->blockSize - size;
797 flush = true;
798 // use some garbage for the fill
799 bytes = vars->buffer + vars->bufferOffset;
800 }
801
802 copy = vars->bufferLimit - vars->bufferOffset;
803 if (copy > size)
804 copy = size;
805 else
806 flush = true;
807
808 if (bytes)
809 {
810 bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
811 bytes += copy;
812 }
813 else
814 bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
815
816 size -= copy;
817 vars->bufferOffset += copy;
818 vars->position += copy;
819
820 if (flush && vars->bufferOffset)
821 {
822 uint64_t offset = (vars->position - vars->bufferOffset
823 - vars->extentPosition + vars->currentExtent->start);
824 uint32_t length = (vars->bufferOffset);
825
826 #if CRYPTO
827 if (cryptvars && vars->encryptStart && (vars->position > vars->encryptStart))
828 {
829 uint32_t encryptLen, encryptStart;
830 encryptLen = vars->position - vars->encryptStart;
831 if (encryptLen > length)
832 encryptLen = length;
833 encryptStart = length - encryptLen;
834
835 // encrypt the buffer
836 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
837 &cryptvars->aes_iv[0],
838 encryptLen / AES_BLOCK_SIZE,
839 vars->buffer + vars->bufferHalf + encryptStart,
840 &cryptvars->ctx.encrypt);
841 // save initial vector for following encrypts
842 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
843 &cryptvars->aes_iv[0],
844 AES_BLOCK_SIZE);
845 }
846 #endif /* CRYPTO */
847
848 if (vars->io)
849 {
850 err = IOHibernatePollerIODone(vars, true);
851 if (kIOReturnSuccess != err)
852 break;
853 }
854
855 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
856 //if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
857
858 err = IOHibernatePollerIO(vars, kIOPolledWrite, vars->bufferHalf, offset, length);
859 if (kIOReturnSuccess != err)
860 break;
861 vars->io = true;
862
863 vars->extentRemaining -= vars->bufferOffset;
864 if (!vars->extentRemaining)
865 {
866 vars->currentExtent++;
867 vars->extentRemaining = vars->currentExtent->length;
868 vars->extentPosition = vars->position;
869 if (!vars->extentRemaining)
870 {
871 err = kIOReturnOverrun;
872 break;
873 }
874 }
875
876 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
877 vars->bufferOffset = 0;
878 if (vars->bufferSize <= vars->extentRemaining)
879 vars->bufferLimit = vars->bufferSize;
880 else
881 vars->bufferLimit = vars->extentRemaining;
882
883 flush = false;
884 }
885 }
886 while (size);
887
888 return (err);
889 }
890
891 static IOReturn
892 IOPolledFileRead(IOPolledFileIOVars * vars,
893 uint8_t * bytes, IOByteCount size,
894 hibernate_cryptvars_t * cryptvars)
895 {
896 IOReturn err = kIOReturnSuccess;
897 IOByteCount copy;
898
899 // bytesWritten += size;
900
901 do
902 {
903 copy = vars->bufferLimit - vars->bufferOffset;
904 if (copy > size)
905 copy = size;
906
907 if (bytes)
908 {
909 bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
910 bytes += copy;
911 }
912 size -= copy;
913 vars->bufferOffset += copy;
914 // vars->position += copy;
915
916 if (vars->bufferOffset == vars->bufferLimit)
917 {
918 if (vars->io)
919 {
920 err = IOHibernatePollerIODone(vars, false);
921 if (kIOReturnSuccess != err)
922 break;
923 }
924 else
925 cryptvars = 0;
926
927 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
928
929 vars->position += vars->lastRead;
930 vars->extentRemaining -= vars->lastRead;
931 vars->bufferLimit = vars->lastRead;
932
933 if (!vars->extentRemaining)
934 {
935 vars->currentExtent++;
936 vars->extentRemaining = vars->currentExtent->length;
937 vars->extentPosition = vars->position;
938 if (!vars->extentRemaining)
939 {
940 err = kIOReturnOverrun;
941 break;
942 }
943 }
944
945 uint64_t length;
946 uint64_t lastReadLength = vars->lastRead;
947 uint64_t offset = (vars->position
948 - vars->extentPosition + vars->currentExtent->start);
949 if (vars->extentRemaining <= vars->bufferSize)
950 length = vars->extentRemaining;
951 else
952 length = vars->bufferSize;
953 vars->lastRead = length;
954
955 //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
956
957 err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
958 if (kIOReturnSuccess != err)
959 break;
960 vars->io = true;
961
962 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
963 vars->bufferOffset = 0;
964
965 #if CRYPTO
966 if (cryptvars)
967 {
968 uint8_t thisVector[AES_BLOCK_SIZE];
969 // save initial vector for following decrypts
970 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
971 bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
972 &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
973 // decrypt the buffer
974 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
975 &thisVector[0],
976 lastReadLength / AES_BLOCK_SIZE,
977 vars->buffer + vars->bufferHalf,
978 &cryptvars->ctx.decrypt);
979 }
980 #endif CRYPTO
981 }
982 }
983 while (size);
984
985 return (err);
986 }
987
988 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
989
990 IOReturn
991 IOHibernateSystemSleep(void)
992 {
993 IOReturn err;
994 OSData * data;
995 OSObject * obj;
996 OSString * str;
997 OSNumber * num;
998 OSDictionary *sleepOverrideOptions;
999
1000 IOHibernateVars * vars = &gIOHibernateVars;
1001
1002 if (vars->fileVars && vars->fileVars->fileRef)
1003 // already on the way down
1004 return (kIOReturnSuccess);
1005
1006 gIOHibernateState = kIOHibernateStateInactive;
1007
1008 /* The invocation of IOPMSleepSystemWithOptions() may override
1009 * existing hibernation settings.
1010 */
1011 sleepOverrideOptions = (OSDictionary *)OSDynamicCast( OSDictionary,
1012 IOService::getPMRootDomain()->copyProperty(kRootDomainSleepOptionsKey));
1013
1014
1015 /* Hibernate mode overriden by sleep otions ? */
1016 obj = NULL;
1017
1018 if (sleepOverrideOptions) {
1019 obj = sleepOverrideOptions->getObject(kIOHibernateModeKey);
1020 if (obj) obj->retain();
1021 }
1022
1023 if(!obj) {
1024 obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateModeKey);
1025 }
1026
1027 if (obj && (num = OSDynamicCast(OSNumber, obj)) )
1028 {
1029 gIOHibernateMode = num->unsigned32BitValue();
1030 if (kIOHibernateModeSleep & gIOHibernateMode)
1031 // default to discard clean for safe sleep
1032 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
1033 | kIOHibernateModeDiscardCleanActive);
1034 }
1035
1036 if (obj) obj->release();
1037
1038 /* Hibernate free rotio overriden by sleep options ? */
1039 obj = NULL;
1040
1041 if (sleepOverrideOptions) {
1042 obj = sleepOverrideOptions->getObject(kIOHibernateFreeRatioKey);
1043 if (obj) obj->retain();
1044 }
1045
1046 if(!obj) {
1047 obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeRatioKey);
1048 }
1049 if (obj && (num = OSDynamicCast(OSNumber, obj)))
1050 {
1051 gIOHibernateFreeRatio = num->unsigned32BitValue();
1052 }
1053 if (obj) obj->release();
1054
1055 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeTimeKey)))
1056 {
1057 if ((num = OSDynamicCast(OSNumber, obj)))
1058 gIOHibernateFreeTime = num->unsigned32BitValue();
1059 obj->release();
1060 }
1061 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
1062 {
1063 if ((str = OSDynamicCast(OSString, obj)))
1064 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
1065 sizeof(gIOHibernateFilename));
1066 obj->release();
1067 }
1068
1069 if (sleepOverrideOptions)
1070 sleepOverrideOptions->release();
1071
1072 if (!gIOHibernateMode || !gIOHibernateFilename[0])
1073 return (kIOReturnUnsupported);
1074
1075 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
1076
1077
1078 do
1079 {
1080 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1081 4 * page_size, page_size);
1082 vars->ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1083 2 * kDefaultIOSize, page_size);
1084
1085 if (!vars->srcBuffer || !vars->ioBuffer)
1086 {
1087 err = kIOReturnNoMemory;
1088 break;
1089 }
1090
1091 err = IOPolledFileOpen(gIOHibernateFilename, vars->ioBuffer,
1092 &vars->fileVars, &vars->fileExtents, &data);
1093 if (KERN_SUCCESS != err)
1094 {
1095 HIBLOG("IOPolledFileOpen(%x)\n", err);
1096 break;
1097 }
1098 if (vars->fileVars->fileRef)
1099 {
1100 // invalidate the image file
1101 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1102 int err = kern_write_file(vars->fileVars->fileRef, 0,
1103 (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1104 if (KERN_SUCCESS != err)
1105 HIBLOG("kern_write_file(%d)\n", err);
1106 }
1107
1108 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1109
1110 boolean_t encryptedswap;
1111 err = hibernate_setup(gIOHibernateCurrentHeader,
1112 gIOHibernateFreeRatio, gIOHibernateFreeTime,
1113 &vars->page_list, &vars->page_list_wired, &encryptedswap);
1114 if (KERN_SUCCESS != err)
1115 {
1116 HIBLOG("hibernate_setup(%d)\n", err);
1117 break;
1118 }
1119
1120 if (encryptedswap)
1121 gIOHibernateMode ^= kIOHibernateModeEncrypt;
1122
1123 vars->videoAllocSize = kVideoMapSize;
1124 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
1125 vars->videoMapping = 0;
1126
1127 // generate crypt keys
1128 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
1129 vars->wiredCryptKey[i] = random();
1130 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
1131 vars->cryptKey[i] = random();
1132
1133 // set nvram
1134
1135 IORegistryEntry * regEntry;
1136 if (!gIOOptionsEntry)
1137 {
1138 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
1139 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
1140 if (regEntry && !gIOOptionsEntry)
1141 regEntry->release();
1142 }
1143 if (!gIOChosenEntry)
1144 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1145
1146 if (gIOOptionsEntry)
1147 {
1148 const OSSymbol * sym;
1149
1150 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1151 if (sym)
1152 {
1153 gIOOptionsEntry->setProperty(sym, data);
1154 sym->release();
1155 }
1156 data->release();
1157
1158 #ifdef __ppc__
1159 size_t len;
1160 char valueString[16];
1161
1162 vars->saveBootDevice = gIOOptionsEntry->copyProperty(kIOSelectedBootDeviceKey);
1163 if (gIOChosenEntry)
1164 {
1165 OSData * bootDevice = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOBootPathKey));
1166 if (bootDevice)
1167 {
1168 sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
1169 OSString * str2 = OSString::withCStringNoCopy((const char *) bootDevice->getBytesNoCopy());
1170 if (sym && str2)
1171 gIOOptionsEntry->setProperty(sym, str2);
1172 if (sym)
1173 sym->release();
1174 if (str2)
1175 str2->release();
1176 }
1177 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMemorySignatureKey));
1178 if (data)
1179 {
1180 vars->haveFastBoot = true;
1181
1182 len = sprintf(valueString, "0x%lx", *((UInt32 *)data->getBytesNoCopy()));
1183 data = OSData::withBytes(valueString, len + 1);
1184 sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
1185 if (sym && data)
1186 gIOOptionsEntry->setProperty(sym, data);
1187 if (sym)
1188 sym->release();
1189 if (data)
1190 data->release();
1191 }
1192 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
1193 if (data)
1194 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
1195 }
1196 #endif /* __ppc__ */
1197 #ifdef __i386__
1198 struct AppleRTCHibernateVars
1199 {
1200 uint8_t signature[4];
1201 uint32_t revision;
1202 uint8_t booterSignature[20];
1203 uint8_t wiredCryptKey[16];
1204 };
1205 AppleRTCHibernateVars rtcVars;
1206
1207 rtcVars.signature[0] = 'A';
1208 rtcVars.signature[1] = 'A';
1209 rtcVars.signature[2] = 'P';
1210 rtcVars.signature[3] = 'L';
1211 rtcVars.revision = 1;
1212 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
1213 if (gIOHibernateBootSignature[0])
1214 {
1215 char c;
1216 uint8_t value = 0;
1217 for (uint32_t i = 0;
1218 (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
1219 i++)
1220 {
1221 if (c >= 'a')
1222 c -= 'a' - 10;
1223 else if (c >= 'A')
1224 c -= 'A' - 10;
1225 else if (c >= '0')
1226 c -= '0';
1227 else
1228 continue;
1229 value = (value << 4) | c;
1230 if (i & 1)
1231 rtcVars.booterSignature[i >> 1] = value;
1232 }
1233 }
1234 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
1235 if (data)
1236 {
1237 IOService::getPMRootDomain()->setProperty(kIOHibernateRTCVariablesKey, data);
1238
1239 if( gIOOptionsEntry )
1240 {
1241 if( gIOHibernateMode & kIOHibernateModeSwitch )
1242 {
1243 const OSSymbol *sym;
1244 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSwitchVarsKey);
1245 if( sym )
1246 {
1247 gIOOptionsEntry->setProperty(sym, data); /* intentional insecure backup of rtc boot vars */
1248 sym->release();
1249 }
1250 }
1251 }
1252
1253 data->release();
1254 }
1255 if (gIOChosenEntry)
1256 {
1257 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
1258 if (data)
1259 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
1260 }
1261 #else /* !__i386__ */
1262 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1263 {
1264 data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1265 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
1266 if (sym && data)
1267 gIOOptionsEntry->setProperty(sym, data);
1268 if (sym)
1269 sym->release();
1270 if (data)
1271 data->release();
1272 if (false && gIOHibernateBootSignature[0])
1273 {
1274 data = OSData::withCapacity(16);
1275 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
1276 if (sym && data)
1277 {
1278 char c;
1279 uint8_t value = 0;
1280 for (uint32_t i = 0; (c = gIOHibernateBootSignature[i]); i++)
1281 {
1282 if (c >= 'a')
1283 c -= 'a' - 10;
1284 else if (c >= 'A')
1285 c -= 'A' - 10;
1286 else if (c >= '0')
1287 c -= '0';
1288 else
1289 continue;
1290 value = (value << 4) | c;
1291 if (i & 1)
1292 data->appendBytes(&value, sizeof(value));
1293 }
1294 gIOOptionsEntry->setProperty(sym, data);
1295 }
1296 if (sym)
1297 sym->release();
1298 if (data)
1299 data->release();
1300 }
1301 }
1302 if (!vars->haveFastBoot)
1303 {
1304 // set boot volume to zero
1305 IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
1306 if (platform && (kIOReturnSuccess == platform->readXPRAM(kXPRamAudioVolume,
1307 &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume))))
1308 {
1309 uint8_t newVolume;
1310 newVolume = vars->saveBootAudioVolume & 0xf8;
1311 platform->writeXPRAM(kXPRamAudioVolume,
1312 &newVolume, sizeof(newVolume));
1313 }
1314 }
1315 #endif /* !__i386__ */
1316 }
1317 // --
1318
1319 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
1320 gIOHibernateState = kIOHibernateStateHibernating;
1321 }
1322 while (false);
1323
1324 return (err);
1325 }
1326
1327 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1328
1329 DECLARE_IOHIBERNATEPROGRESSALPHA
1330
1331 static void
1332 ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
1333 {
1334 uint32_t rowBytes, pixelShift;
1335 uint32_t x, y;
1336 int32_t blob;
1337 uint32_t alpha, in, color, result;
1338 uint8_t * out;
1339 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1340
1341 rowBytes = display->rowBytes;
1342 pixelShift = display->depth >> 4;
1343 if (pixelShift < 1) return;
1344
1345 screen += ((display->width
1346 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1347 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1348
1349 for (y = 0; y < kIOHibernateProgressHeight; y++)
1350 {
1351 out = screen + y * rowBytes;
1352 for (blob = 0; blob < kIOHibernateProgressCount; blob++)
1353 {
1354 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
1355 for (x = 0; x < kIOHibernateProgressWidth; x++)
1356 {
1357 alpha = gIOHibernateProgressAlpha[y][x];
1358 result = color;
1359 if (alpha)
1360 {
1361 if (0xff != alpha)
1362 {
1363 if (1 == pixelShift)
1364 {
1365 in = *((uint16_t *)out) & 0x1f; // 16
1366 in = (in << 3) | (in >> 2);
1367 }
1368 else
1369 in = *((uint32_t *)out) & 0xff; // 32
1370 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
1371 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
1372 }
1373 if (1 == pixelShift)
1374 {
1375 result >>= 3;
1376 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1377 }
1378 else
1379 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1380 }
1381 out += (1 << pixelShift);
1382 }
1383 out += (kIOHibernateProgressSpacing << pixelShift);
1384 }
1385 }
1386 }
1387
1388
1389 static void
1390 ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
1391 {
1392 uint32_t rowBytes, pixelShift;
1393 uint32_t x, y;
1394 int32_t blob, lastBlob;
1395 uint32_t alpha, in, color, result;
1396 uint8_t * out;
1397 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1398
1399 pixelShift = display->depth >> 4;
1400 if (pixelShift < 1)
1401 return;
1402
1403 rowBytes = display->rowBytes;
1404
1405 screen += ((display->width
1406 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1407 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1408
1409 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
1410
1411 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
1412
1413 for (y = 0; y < kIOHibernateProgressHeight; y++)
1414 {
1415 out = screen + y * rowBytes;
1416 for (blob = firstBlob; blob <= lastBlob; blob++)
1417 {
1418 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
1419 for (x = 0; x < kIOHibernateProgressWidth; x++)
1420 {
1421 alpha = gIOHibernateProgressAlpha[y][x];
1422 result = color;
1423 if (alpha)
1424 {
1425 if (0xff != alpha)
1426 {
1427 in = display->progressSaveUnder[blob][saveindex[blob]++];
1428 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
1429 }
1430 if (1 == pixelShift)
1431 {
1432 result >>= 3;
1433 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1434 }
1435 else
1436 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1437 }
1438 out += (1 << pixelShift);
1439 }
1440 out += (kIOHibernateProgressSpacing << pixelShift);
1441 }
1442 }
1443 }
1444
1445 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1446
1447 IOReturn
1448 IOHibernateSystemHasSlept(void)
1449 {
1450 IOHibernateVars * vars = &gIOHibernateVars;
1451 OSObject * obj;
1452 OSData * data;
1453
1454 obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
1455 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1456 if (obj && !vars->previewBuffer)
1457 obj->release();
1458
1459 vars->consoleMapping = NULL;
1460 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
1461 {
1462 vars->previewBuffer->release();
1463 vars->previewBuffer = 0;
1464 }
1465
1466 if (vars->previewBuffer && (data = OSDynamicCast(OSData,
1467 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
1468 {
1469 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
1470 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", flags);
1471
1472 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1473
1474 if (kIOHibernatePreviewUpdates & flags)
1475 {
1476 PE_Video consoleInfo;
1477 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1478
1479 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1480
1481 graphicsInfo->width = consoleInfo.v_width;
1482 graphicsInfo->height = consoleInfo.v_height;
1483 graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
1484 graphicsInfo->depth = consoleInfo.v_depth;
1485 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1486
1487 HIBPRINT("video %p %d %d %d\n",
1488 vars->consoleMapping, gIOHibernateGraphicsInfo->depth,
1489 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height);
1490 if (vars->consoleMapping)
1491 ProgressInit(graphicsInfo, vars->consoleMapping,
1492 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1493 }
1494 }
1495
1496 if (gIOOptionsEntry)
1497 gIOOptionsEntry->sync();
1498
1499 return (kIOReturnSuccess);
1500 }
1501
1502 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1503
1504 IOReturn
1505 IOHibernateSystemWake(void)
1506 {
1507 IOHibernateVars * vars = &gIOHibernateVars;
1508
1509 hibernate_teardown(vars->page_list, vars->page_list_wired);
1510
1511 if (vars->videoMapping)
1512 {
1513 if (vars->videoMapSize)
1514 // remove mappings
1515 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1516 if (vars->videoAllocSize)
1517 // dealloc range
1518 kmem_free(kernel_map, trunc_page_32(vars->videoMapping), vars->videoAllocSize);
1519 }
1520
1521 if (vars->previewBuffer)
1522 {
1523 vars->previewBuffer->release();
1524 vars->previewBuffer = 0;
1525 }
1526
1527 if (vars->fileVars)
1528 {
1529 IOPolledFileClose(vars->fileVars);
1530 }
1531
1532 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1533
1534 #ifdef __ppc__
1535 OSData * data = OSData::withCapacity(4);
1536 if (gIOOptionsEntry && data)
1537 {
1538 const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1539 if (sym)
1540 {
1541 gIOOptionsEntry->setProperty(sym, data);
1542 sym->release();
1543 }
1544 sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
1545 if (sym)
1546 {
1547 if (vars->saveBootDevice)
1548 {
1549 gIOOptionsEntry->setProperty(sym, vars->saveBootDevice);
1550 vars->saveBootDevice->release();
1551 }
1552 sym->release();
1553 }
1554 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
1555 if (sym)
1556 {
1557 gIOOptionsEntry->setProperty(sym, data);
1558 sym->release();
1559 }
1560 sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
1561 if (sym)
1562 {
1563 gIOOptionsEntry->removeProperty(sym);
1564 sym->release();
1565 }
1566 }
1567 if (data)
1568 data->release();
1569
1570 if (gIOOptionsEntry)
1571 {
1572 if (!vars->haveFastBoot)
1573 {
1574 // reset boot audio volume
1575 IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
1576 if (platform)
1577 platform->writeXPRAM(kXPRamAudioVolume,
1578 &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume));
1579 }
1580
1581 // sync now to hardware if the booter has not
1582 if (kIOHibernateStateInactive == gIOHibernateState)
1583 gIOOptionsEntry->sync();
1584 else
1585 // just sync the variables in case a later panic syncs nvram (it won't sync variables)
1586 gIOOptionsEntry->syncOFVariables();
1587 }
1588 #endif
1589
1590 #ifdef __i386__
1591 IOService::getPMRootDomain()->removeProperty(kIOHibernateRTCVariablesKey);
1592
1593 /*
1594 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1595 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1596 */
1597 if (gIOOptionsEntry) {
1598 const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1599
1600 if (sym) {
1601 gIOOptionsEntry->removeProperty(sym);
1602 sym->release();
1603 }
1604 }
1605 #endif
1606
1607 if (vars->srcBuffer)
1608 vars->srcBuffer->release();
1609 if (vars->ioBuffer)
1610 vars->ioBuffer->release();
1611 if (vars->fileExtents)
1612 vars->fileExtents->release();
1613
1614 bzero(vars, sizeof(*vars));
1615
1616 // gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1617
1618 return (kIOReturnSuccess);
1619 }
1620
1621 IOReturn
1622 IOHibernateSystemPostWake(void)
1623 {
1624 if (gIOHibernateFileRef)
1625 {
1626 // invalidate the image file
1627 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1628 int err = kern_write_file(gIOHibernateFileRef, 0,
1629 (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1630 if (KERN_SUCCESS != err)
1631 HIBLOG("kern_write_file(%d)\n", err);
1632
1633 kern_close_file_for_direct_io(gIOHibernateFileRef);
1634 gIOHibernateFileRef = 0;
1635 }
1636 return (kIOReturnSuccess);
1637 }
1638
1639 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1640
1641 SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
1642 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
1643 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
1644 SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
1645 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
1646 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
1647 SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
1648 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
1649 &gIOHibernateMode, 0, "");
1650
1651 void
1652 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1653 {
1654 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
1655 if (data)
1656 {
1657 rootDomain->setProperty(kIOHibernateStateKey, data);
1658 data->release();
1659 }
1660
1661 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))
1662 gIOHibernateMode = kIOHibernateModeOn;
1663 else
1664 gIOHibernateFilename[0] = 0;
1665
1666 sysctl_register_oid(&sysctl__kern_hibernatefile);
1667 sysctl_register_oid(&sysctl__kern_bootsignature);
1668 sysctl_register_oid(&sysctl__kern_hibernatemode);
1669 }
1670
1671
1672 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1673
1674 static void
1675 hibernate_setup_for_wake(void)
1676 {
1677 #if __ppc__
1678 // go slow (state needed for wake)
1679 ml_set_processor_speed(1);
1680 #endif
1681 }
1682
1683 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1684
1685 #define C_ASSERT(e) typedef char __C_ASSERT__[(e) ? 1 : -1]
1686
1687 extern "C" uint32_t
1688 hibernate_write_image(void)
1689 {
1690 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
1691 IOHibernateVars * vars = &gIOHibernateVars;
1692 IOPolledFileExtent * fileExtents;
1693
1694 C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
1695
1696 uint32_t pageCount, pagesDone;
1697 IOReturn err;
1698 vm_offset_t ppnum;
1699 IOItemCount page, count;
1700 uint8_t * src;
1701 uint8_t * data;
1702 IOByteCount pageCompressedSize;
1703 uint64_t compressedSize, uncompressedSize;
1704 uint64_t image1Size = 0;
1705 uint32_t bitmap_size;
1706 bool iterDone, pollerOpen, needEncryptStart;
1707 uint32_t restore1Sum, sum, sum1, sum2;
1708 uint32_t tag;
1709 uint32_t pageType;
1710 uint32_t pageAndCount[2];
1711
1712 AbsoluteTime startTime, endTime;
1713 AbsoluteTime allTime, compTime, decoTime;
1714 uint64_t nsec;
1715 uint32_t lastProgressStamp = 0;
1716 uint32_t progressStamp;
1717 uint32_t blob, lastBlob = (uint32_t) -1L;
1718
1719 hibernate_cryptvars_t _cryptvars;
1720 hibernate_cryptvars_t * cryptvars = 0;
1721
1722 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
1723 return (false /* sleep */ );
1724
1725 restore1Sum = sum1 = sum2 = 0;
1726
1727 #if CRYPTO
1728 // encryption data. "iv" is the "initial vector".
1729 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1730 {
1731 static const unsigned char first_iv[AES_BLOCK_SIZE]
1732 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
1733 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
1734
1735 cryptvars = &gIOHibernateCryptWakeContext;
1736 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
1737 aes_encrypt_key(vars->cryptKey,
1738 kIOHibernateAESKeySize,
1739 &cryptvars->ctx.encrypt);
1740 aes_decrypt_key(vars->cryptKey,
1741 kIOHibernateAESKeySize,
1742 &cryptvars->ctx.decrypt);
1743
1744 cryptvars = &_cryptvars;
1745 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
1746 aes_encrypt_key(vars->wiredCryptKey,
1747 kIOHibernateAESKeySize,
1748 &cryptvars->ctx.encrypt);
1749
1750 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1751 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1752 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
1753 bzero(gIOHibernateCryptWakeVars, sizeof(hibernate_cryptwakevars_t));
1754 }
1755 #endif /* CRYPTO */
1756
1757 hibernate_setup_for_wake();
1758
1759 hibernate_page_list_setall(vars->page_list,
1760 vars->page_list_wired,
1761 &pageCount);
1762
1763 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
1764
1765 fileExtents = (IOPolledFileExtent *) vars->fileExtents->getBytesNoCopy();
1766
1767 #if 0
1768 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
1769 for (page = 0; page < count; page++)
1770 {
1771 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
1772 fileExtents[page].start, fileExtents[page].length,
1773 fileExtents[page].start + fileExtents[page].length);
1774 }
1775 #endif
1776
1777 needEncryptStart = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
1778
1779 AbsoluteTime_to_scalar(&compTime) = 0;
1780 AbsoluteTime_to_scalar(&decoTime) = 0;
1781
1782 clock_get_uptime(&allTime);
1783
1784 do
1785 {
1786 compressedSize = 0;
1787 uncompressedSize = 0;
1788 iterDone = false;
1789 pageType = 0; // wired pages first
1790
1791 IOPolledFileSeek(vars->fileVars, sizeof(IOHibernateImageHeader));
1792
1793 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
1794 ml_get_interrupts_enabled());
1795 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledBeforeSleepState, vars->ioBuffer);
1796 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
1797 pollerOpen = (kIOReturnSuccess == err);
1798 if (!pollerOpen)
1799 break;
1800
1801 // copy file block extent list if larger than header
1802
1803 count = vars->fileExtents->getLength();
1804 if (count > sizeof(header->fileExtentMap))
1805 {
1806 count -= sizeof(header->fileExtentMap);
1807 err = IOPolledFileWrite(vars->fileVars,
1808 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
1809 if (kIOReturnSuccess != err)
1810 break;
1811 }
1812
1813 // copy out restore1 code
1814
1815 page = atop_32(sectHIBB);
1816 count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
1817 header->restore1CodePage = page;
1818 header->restore1PageCount = count;
1819 header->restore1CodeOffset = ((uint32_t) &hibernate_machine_entrypoint) - sectHIBB;
1820 header->restore1StackOffset = ((uint32_t) &gIOHibernateRestoreStackEnd[0]) - 64 - sectHIBB;
1821
1822 // sum __HIB sect, with zeros for the stack
1823 src = (uint8_t *) trunc_page(sectHIBB);
1824 for (page = 0; page < count; page++)
1825 {
1826 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
1827 restore1Sum += hibernate_sum(src, page_size);
1828 else
1829 restore1Sum += 0x10000001;
1830 src += page_size;
1831 }
1832 sum1 = restore1Sum;
1833
1834 // write the __HIB sect, with zeros for the stack
1835
1836 src = (uint8_t *) trunc_page(sectHIBB);
1837 count = ((uint32_t) &gIOHibernateRestoreStack[0]) - trunc_page(sectHIBB);
1838 if (count)
1839 {
1840 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
1841 if (kIOReturnSuccess != err)
1842 break;
1843 }
1844 err = IOPolledFileWrite(vars->fileVars,
1845 (uint8_t *) 0,
1846 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
1847 cryptvars);
1848 if (kIOReturnSuccess != err)
1849 break;
1850 src = &gIOHibernateRestoreStackEnd[0];
1851 count = round_page(sectHIBB + sectSizeHIB) - ((uint32_t) src);
1852 if (count)
1853 {
1854 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
1855 if (kIOReturnSuccess != err)
1856 break;
1857 }
1858
1859 // write the preview buffer
1860
1861 addr64_t phys64;
1862 IOByteCount segLen;
1863
1864 if (vars->previewBuffer)
1865 {
1866 ppnum = 0;
1867 count = 0;
1868 do
1869 {
1870 phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen);
1871 pageAndCount[0] = atop_64(phys64);
1872 pageAndCount[1] = atop_32(segLen);
1873 err = IOPolledFileWrite(vars->fileVars,
1874 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1875 cryptvars);
1876 if (kIOReturnSuccess != err)
1877 break;
1878 count += segLen;
1879 ppnum += sizeof(pageAndCount);
1880 }
1881 while (phys64);
1882 if (kIOReturnSuccess != err)
1883 break;
1884
1885 src = (uint8_t *) vars->previewBuffer->getSourceSegment(0, NULL);
1886 count = vars->previewBuffer->getLength();
1887
1888 header->previewPageListSize = ppnum;
1889 header->previewSize = count + ppnum;
1890
1891 for (page = 0; page < count; page += page_size)
1892 sum1 += hibernate_sum(src + page, page_size);
1893
1894 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
1895 if (kIOReturnSuccess != err)
1896 break;
1897 }
1898
1899 // mark areas for no save
1900
1901 for (count = 0;
1902 (phys64 = vars->ioBuffer->getPhysicalSegment64(count, &segLen));
1903 count += segLen)
1904 {
1905 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1906 atop_64(phys64), atop_32(segLen),
1907 kIOHibernatePageStateFree);
1908 pageCount -= atop_32(segLen);
1909 }
1910
1911 for (count = 0;
1912 (phys64 = vars->srcBuffer->getPhysicalSegment64(count, &segLen));
1913 count += segLen)
1914 {
1915 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1916 atop_64(phys64), atop_32(segLen),
1917 kIOHibernatePageStateFree);
1918 pageCount -= atop_32(segLen);
1919 }
1920
1921 // copy out bitmap of pages available for trashing during restore
1922
1923 bitmap_size = vars->page_list_wired->list_size;
1924 src = (uint8_t *) vars->page_list_wired;
1925 err = IOPolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
1926 if (kIOReturnSuccess != err)
1927 break;
1928
1929 // mark more areas for no save, but these are not available
1930 // for trashing during restore
1931
1932 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
1933
1934 page = atop_32(sectHIBB);
1935 count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
1936 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1937 page, count,
1938 kIOHibernatePageStateFree);
1939 pageCount -= count;
1940
1941 if (vars->previewBuffer) for (count = 0;
1942 (phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen));
1943 count += segLen)
1944 {
1945 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1946 atop_64(phys64), atop_32(segLen),
1947 kIOHibernatePageStateFree);
1948 pageCount -= atop_32(segLen);
1949 }
1950
1951 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
1952
1953 ppnum = 0;
1954 pagesDone = 0;
1955 lastBlob = 0;
1956
1957 HIBLOG("writing %d pages\n", pageCount);
1958
1959 do
1960 {
1961 count = hibernate_page_list_iterate(pageType ? vars->page_list : vars->page_list_wired,
1962 &ppnum);
1963 // kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
1964
1965 iterDone = !count;
1966
1967 pageAndCount[0] = ppnum;
1968 pageAndCount[1] = count;
1969 err = IOPolledFileWrite(vars->fileVars,
1970 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1971 cryptvars);
1972 if (kIOReturnSuccess != err)
1973 break;
1974
1975 for (page = 0; page < count; page++)
1976 {
1977 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(ppnum), page_size);
1978 if (err)
1979 {
1980 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%d] %x\n", __LINE__, ppnum, err);
1981 break;
1982 }
1983
1984 sum = hibernate_sum(src, page_size);
1985
1986 clock_get_uptime(&startTime);
1987
1988 pageCompressedSize = WKdm_compress ((WK_word*) src, (WK_word*) (src + page_size), PAGE_SIZE_IN_WORDS);
1989
1990 clock_get_uptime(&endTime);
1991 ADD_ABSOLUTETIME(&compTime, &endTime);
1992 SUB_ABSOLUTETIME(&compTime, &startTime);
1993
1994 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1995 pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
1996
1997 if (pageCompressedSize > page_size)
1998 {
1999 // HIBLOG("------------lose: %d\n", pageCompressedSize);
2000 pageCompressedSize = page_size;
2001 }
2002
2003 if (pageCompressedSize != page_size)
2004 data = (src + page_size);
2005 else
2006 data = src;
2007
2008 tag = pageCompressedSize | kIOHibernateTagSignature;
2009
2010 if (pageType)
2011 sum2 += sum;
2012 else
2013 sum1 += sum;
2014
2015 if (needEncryptStart && (ppnum >= atop_32(sectDATAB)))
2016 {
2017 // start encrypting partway into the data about to be written
2018 vars->fileVars->encryptStart = (vars->fileVars->position + AES_BLOCK_SIZE - 1)
2019 & ~(AES_BLOCK_SIZE - 1);
2020 needEncryptStart = false;
2021 }
2022
2023 err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
2024 if (kIOReturnSuccess != err)
2025 break;
2026
2027 err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
2028 if (kIOReturnSuccess != err)
2029 break;
2030
2031 compressedSize += pageCompressedSize;
2032 if (pageCompressedSize)
2033 uncompressedSize += page_size;
2034 ppnum++;
2035 pagesDone++;
2036
2037 if (vars->consoleMapping && (0 == (1023 & pagesDone)))
2038 {
2039 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
2040 if (blob != lastBlob)
2041 {
2042 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
2043 lastBlob = blob;
2044 }
2045 }
2046 if (0 == (8191 & pagesDone))
2047 {
2048 clock_get_uptime(&endTime);
2049 SUB_ABSOLUTETIME(&endTime, &allTime);
2050 absolutetime_to_nanoseconds(endTime, &nsec);
2051 progressStamp = nsec / 750000000ULL;
2052 if (progressStamp != lastProgressStamp)
2053 {
2054 lastProgressStamp = progressStamp;
2055 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
2056 }
2057 }
2058 }
2059 if (kIOReturnSuccess != err)
2060 break;
2061 if (iterDone && !pageType)
2062 {
2063 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2064 if (kIOReturnSuccess != err)
2065 break;
2066
2067 iterDone = false;
2068 pageType = 1;
2069 ppnum = 0;
2070 image1Size = vars->fileVars->position;
2071 if (cryptvars)
2072 {
2073 bcopy(&cryptvars->aes_iv[0],
2074 &gIOHibernateCryptWakeContext.aes_iv[0],
2075 sizeof(cryptvars->aes_iv));
2076 cryptvars = &gIOHibernateCryptWakeContext;
2077 }
2078 HIBLOG("image1Size %qd\n", image1Size);
2079 }
2080 }
2081 while (!iterDone);
2082 if (kIOReturnSuccess != err)
2083 break;
2084 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2085 if (kIOReturnSuccess != err)
2086 break;
2087
2088 // Header:
2089
2090 header->imageSize = vars->fileVars->position;
2091 header->image1Size = image1Size;
2092 header->bitmapSize = bitmap_size;
2093 header->pageCount = pageCount;
2094 header->encryptStart = vars->fileVars->encryptStart;
2095
2096 header->restore1Sum = restore1Sum;
2097 header->image1Sum = sum1;
2098 header->image2Sum = sum2;
2099
2100 count = vars->fileExtents->getLength();
2101 if (count > sizeof(header->fileExtentMap))
2102 {
2103 header->fileExtentMapSize = count;
2104 count = sizeof(header->fileExtentMap);
2105 }
2106 else
2107 header->fileExtentMapSize = sizeof(header->fileExtentMap);
2108 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
2109
2110 IOPolledFileSeek(vars->fileVars, 0);
2111 err = IOPolledFileWrite(vars->fileVars,
2112 (uint8_t *) header, sizeof(IOHibernateImageHeader),
2113 cryptvars);
2114 if (kIOReturnSuccess != err)
2115 break;
2116 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2117 if (kIOReturnSuccess != err)
2118 break;
2119 err = IOHibernatePollerIODone(vars->fileVars, true);
2120 if (kIOReturnSuccess != err)
2121 break;
2122 }
2123 while (false);
2124
2125 clock_get_uptime(&endTime);
2126 SUB_ABSOLUTETIME(&endTime, &allTime);
2127 absolutetime_to_nanoseconds(endTime, &nsec);
2128 HIBLOG("all time: %qd ms, ",
2129 nsec / 1000000ULL);
2130
2131 absolutetime_to_nanoseconds(compTime, &nsec);
2132 HIBLOG("comp time: %qd ms, ",
2133 nsec / 1000000ULL);
2134
2135 absolutetime_to_nanoseconds(decoTime, &nsec);
2136 HIBLOG("deco time: %qd ms, ",
2137 nsec / 1000000ULL);
2138
2139 HIBLOG("\nimage %qd, uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
2140 header->imageSize,
2141 uncompressedSize, atop_32(uncompressedSize), compressedSize,
2142 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
2143 sum1, sum2);
2144
2145 if (pollerOpen)
2146 IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
2147
2148 if (vars->consoleMapping)
2149 ProgressUpdate(gIOHibernateGraphicsInfo,
2150 vars->consoleMapping, 0, kIOHibernateProgressCount);
2151
2152 HIBLOG("hibernate_write_image done(%x)\n", err);
2153
2154 // should we come back via regular wake, set the state in memory.
2155 gIOHibernateState = kIOHibernateStateInactive;
2156
2157 if (kIOReturnSuccess == err)
2158 {
2159 if (kIOHibernateModeSleep & gIOHibernateMode)
2160 {
2161 return (kIOHibernatePostWriteSleep);
2162 }
2163 else if(kIOHibernateModeRestart & gIOHibernateMode)
2164 {
2165 return (kIOHibernatePostWriteRestart);
2166 }
2167 else
2168 {
2169 /* by default, power down */
2170 return (kIOHibernatePostWriteHalt);
2171 }
2172 }
2173 else if (kIOReturnAborted == err)
2174 {
2175 return (kIOHibernatePostWriteWake);
2176 }
2177 else
2178 {
2179 /* on error, sleep */
2180 return (kIOHibernatePostWriteSleep);
2181 }
2182 }
2183
2184 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2185
2186 extern "C" void
2187 hibernate_machine_init(void)
2188 {
2189 IOReturn err;
2190 uint32_t sum;
2191 uint32_t pagesDone;
2192 AbsoluteTime allTime, endTime;
2193 uint64_t nsec;
2194 uint32_t lastProgressStamp = 0;
2195 uint32_t progressStamp;
2196 uint64_t progressZeroPosition = 0;
2197 uint32_t blob, lastBlob = (uint32_t) -1L;
2198 hibernate_cryptvars_t * cryptvars = 0;
2199
2200 IOHibernateVars * vars = &gIOHibernateVars;
2201
2202 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
2203 return;
2204
2205 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2206 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2207
2208 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, image1Size %qx, conflictCount %d, nextFree %x\n",
2209 gIOHibernateState, pagesDone, sum, gIOHibernateCurrentHeader->image1Size,
2210 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
2211
2212 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
2213 {
2214 HIBLOG("regular wake\n");
2215 return;
2216 }
2217
2218 HIBPRINT("diag %x %x %x %x\n",
2219 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
2220 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
2221
2222 HIBPRINT("video %x %d %d %d\n",
2223 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2224 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height);
2225
2226 if ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)
2227 hibernate_page_list_discard(vars->page_list);
2228
2229 boot_args *args = (boot_args *) PE_state.bootArgs;
2230
2231 if (vars->videoMapping
2232 && gIOHibernateGraphicsInfo->physicalAddress
2233 && (args->Video.v_baseAddr == gIOHibernateGraphicsInfo->physicalAddress))
2234 {
2235 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2236 * gIOHibernateGraphicsInfo->rowBytes);
2237 IOMapPages(kernel_map,
2238 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2239 vars->videoMapSize, kIOMapInhibitCache );
2240 }
2241
2242 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2243
2244 if (gIOHibernateWakeMapSize)
2245 {
2246 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(gIOHibernateWakeMap),
2247 gIOHibernateWakeMapSize);
2248 if (kIOReturnSuccess == err)
2249 hibernate_newruntime_map(src, gIOHibernateWakeMapSize,
2250 gIOHibernateCurrentHeader->systemTableOffset);
2251 gIOHibernateWakeMap = 0;
2252 gIOHibernateWakeMapSize = 0;
2253 }
2254
2255 uint32_t decoOffset;
2256
2257 clock_get_uptime(&allTime);
2258
2259 HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2260 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0);
2261 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
2262
2263 if (gIOHibernateCurrentHeader->previewSize)
2264 progressZeroPosition = gIOHibernateCurrentHeader->previewSize
2265 + gIOHibernateCurrentHeader->fileExtentMapSize
2266 - sizeof(gIOHibernateCurrentHeader->fileExtentMap)
2267 + ptoa_64(gIOHibernateCurrentHeader->restore1PageCount);
2268
2269 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2270
2271 if (vars->videoMapSize)
2272 {
2273 lastBlob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
2274 / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
2275 ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, 0, lastBlob);
2276 }
2277
2278 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
2279 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2280 {
2281 cryptvars = &gIOHibernateCryptWakeContext;
2282 bcopy(&gIOHibernateCryptWakeVars->aes_iv[0],
2283 &cryptvars->aes_iv[0],
2284 sizeof(cryptvars->aes_iv));
2285 }
2286
2287 // kick off the read ahead
2288 vars->fileVars->io = false;
2289 vars->fileVars->bufferHalf = 0;
2290 vars->fileVars->bufferLimit = 0;
2291 vars->fileVars->lastRead = 0;
2292 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2293
2294 IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
2295 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2296 // --
2297
2298 HIBLOG("hibernate_machine_init reading\n");
2299
2300 uint32_t * header = (uint32_t *) src;
2301 sum = 0;
2302
2303 do
2304 {
2305 unsigned int count;
2306 unsigned int page;
2307 uint32_t tag;
2308 vm_offset_t ppnum, compressedSize;
2309
2310 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2311 if (kIOReturnSuccess != err)
2312 break;
2313
2314 ppnum = header[0];
2315 count = header[1];
2316
2317 // HIBPRINT("(%x, %x)\n", ppnum, count);
2318
2319 if (!count)
2320 break;
2321
2322 for (page = 0; page < count; page++)
2323 {
2324 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2325 if (kIOReturnSuccess != err)
2326 break;
2327
2328 compressedSize = kIOHibernateTagLength & tag;
2329 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength))
2330 {
2331 err = kIOReturnIPCError;
2332 break;
2333 }
2334
2335 if (!compressedSize)
2336 {
2337 ppnum++;
2338 pagesDone++;
2339 continue;
2340 }
2341
2342 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2343 if (kIOReturnSuccess != err)
2344 break;
2345
2346 if (compressedSize < page_size)
2347 {
2348 decoOffset = page_size;
2349 WKdm_decompress((WK_word*) src, (WK_word*) (src + decoOffset), PAGE_SIZE_IN_WORDS);
2350 }
2351 else
2352 decoOffset = 0;
2353
2354 sum += hibernate_sum((src + decoOffset), page_size);
2355
2356 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2357 if (err)
2358 {
2359 HIBLOG("IOMemoryDescriptorReadToPhysical [%d] %x\n", ppnum, err);
2360 break;
2361 }
2362
2363 ppnum++;
2364 pagesDone++;
2365
2366 if (vars->videoMapSize && (0 == (1023 & pagesDone)))
2367 {
2368 blob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
2369 / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
2370 if (blob != lastBlob)
2371 {
2372 ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, lastBlob, blob);
2373 lastBlob = blob;
2374 }
2375 }
2376
2377 if (0 == (8191 & pagesDone))
2378 {
2379 clock_get_uptime(&endTime);
2380 SUB_ABSOLUTETIME(&endTime, &allTime);
2381 absolutetime_to_nanoseconds(endTime, &nsec);
2382 progressStamp = nsec / 750000000ULL;
2383 if (progressStamp != lastProgressStamp)
2384 {
2385 lastProgressStamp = progressStamp;
2386 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2387 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2388 }
2389 }
2390 }
2391 }
2392 while (true);
2393
2394 if (kIOReturnSuccess != err)
2395 panic("Hibernate restore error %x", err);
2396
2397 gIOHibernateCurrentHeader->actualImage2Sum = sum;
2398
2399 if (vars->fileVars->io)
2400 (void) IOHibernatePollerIODone(vars->fileVars, false);
2401
2402 err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
2403
2404 if (vars->videoMapSize)
2405 ProgressUpdate(gIOHibernateGraphicsInfo,
2406 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2407
2408 clock_get_uptime(&endTime);
2409 SUB_ABSOLUTETIME(&endTime, &allTime);
2410 absolutetime_to_nanoseconds(endTime, &nsec);
2411
2412 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %qd ms\n",
2413 pagesDone, sum, nsec / 1000000ULL);
2414 }
2415
2416 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
mirror of XNU