]> git.saurik.com Git - apple/xnu.git/blame - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-6153.11.26.tar.gz
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
CommitLineData
3a60a9f5 1/*
39037602 2 * Copyright (c) 2004-2016 Apple Inc. All rights reserved.
3a60a9f5 3 *
2d21ac55 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
39037602 5 *
2d21ac55
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
39037602 14 *
2d21ac55
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
39037602 17 *
2d21ac55
A		 18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5
A		 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55
A		 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
39037602 25 *
2d21ac55 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
3a60a9f5
A		 27 */
28
3a60a9f5 29/*
0a7de745
A		 30 *
31 * Sleep:
32 *
33 * - PMRootDomain calls IOHibernateSystemSleep() before system sleep
34 * (devices awake, normal execution context)
35 * - IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
36 * grabs its extents and searches for a polling driver willing to work with that IOMedia.
37 * The BSD code makes an ioctl to the storage driver to get the partition base offset to
38 * the disk, and other ioctls to get the transfer constraints
39 * If successful, the file is written to make sure its initially not bootable (in case of
40 * later failure) and nvram set to point to the first block of the file. (Has to be done
41 * here so blocking is possible in nvram support).
42 * hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
43 * page out any pages it wants to (currently zero, but probably some percentage of memory).
44 * Its assumed just allocating pages will cause the VM system to naturally select the best
45 * pages for eviction. It also copies processor flags needed for the restore path and sets
46 * a flag in the boot processor proc info.
47 * gIOHibernateState = kIOHibernateStateHibernating.
48 * - Regular sleep progresses - some drivers may inspect the root domain property
49 * kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
50 * as usual but leaves motherboard I/O on.
51 * - Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
52 * at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
53 * all ppc RC bits out of the hash table and caches into the mapping structures.
54 * - hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
55 * hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
56 * All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
57 * by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
58 * The image header and block list are written. The header includes the second file extent so
59 * only the header block is needed to read the file, regardless of filesystem.
60 * The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
61 * (only) is used to decompress the image during wake/boot.
62 * Some additional pages are removed from the bitmaps - the buffers used for hibernation.
63 * The bitmaps are written to the image.
64 * More areas are removed from the bitmaps (after they have been written to the image) - the
65 * segment "__HIB" pages and interrupt stack.
66 * Each wired page is compressed and written and then each non-wired page. Compression and
67 * disk writes are in parallel.
68 * The image header is written to the start of the file and the polling driver closed.
69 * The machine powers down (or sleeps).
70 *
71 * Boot/Wake:
72 *
73 * - BootX sees the boot-image nvram variable containing the device and block number of the image,
74 * reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
75 * - BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
76 * in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
77 * that is in the kernel's __HIB section.
78 * - BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
79 * hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
80 * only code & data in that section is safe to call since all the other wired pages are still
81 * compressed in the image.
82 * - hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
83 * It uses the bitmaps to work out which pages can be uncompressed from the image to their final
84 * location directly, and copies those that can't to interim free pages. When the image has been
85 * completed, the copies are uncompressed, overwriting the wired image pages.
86 * hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
87 * is used to get pages into place for 64bit.
88 * - the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
89 * changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
90 * are removed from the software strutures, and the hash table is reinitialized.
91 * - After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
92 * of memory, using the polled mode driver, before other threads can run or any devices are turned on.
93 * This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
94 * for the remaining non wired pages.
95 * - The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
96 * (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
97 * is closed via bsd.
98 *
99 * Polled Mode I/O:
100 *
101 * IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
102 * registry, specifying an object of calls IOPolledInterface.
103 *
104 * Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
105 * partition) that the image is going to live, looking for polled interface properties. If it finds
106 * one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
107 * interfaces found are kept in an ordered list.
108 *
109 * There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
110 * few different contexts things happen in:
111 *
112 * - there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
113 * up and running) and after wake - this is safe to allocate memory and do anything. The device
114 * ignores sleep requests from that point since its a waste of time if it goes to sleep and
115 * immediately wakes back up for the image write.
116 *
117 * - there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
118 * immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
119 * from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
120 * used to flush and set the disk to sleep.
121 *
122 * - there is an Open/Close (AfterSleep) pair made around the image read operations that happen
123 * immediately after sleep. These can't block or allocate memory. This is happening after the platform
124 * expert has woken the low level bits of the system, but most of the I/O system has not. There is only
125 * one thread running.
126 *
127 * For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
128 * (prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
129 * the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
130 * Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
131 * that is called for the hardware to check for events, and complete the I/O via the callback.
132 * The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
133 * to restrict I/O ops.
134 */
3a60a9f5
A		 135
136#include <sys/systm.h>
137
138#include <IOKit/IOWorkLoop.h>
139#include <IOKit/IOCommandGate.h>
140#include <IOKit/IOTimerEventSource.h>
141#include <IOKit/IOPlatformExpert.h>
142#include <IOKit/IOKitDebug.h>
143#include <IOKit/IOTimeStamp.h>
144#include <IOKit/pwr_mgt/RootDomain.h>
145#include <IOKit/pwr_mgt/IOPMPrivate.h>
146#include <IOKit/IOMessage.h>
147#include <IOKit/IODeviceTreeSupport.h>
148#include <IOKit/IOBSD.h>
a39ff7e2 149#include <IOKit/IOKitKeysPrivate.h>
3a60a9f5
A		 150#include "RootDomainUserClient.h"
151#include <IOKit/pwr_mgt/IOPowerConnection.h>
152#include "IOPMPowerStateQueue.h"
153#include <IOKit/IOBufferMemoryDescriptor.h>
6d2010ae 154#include <IOKit/AppleKeyStoreInterface.h>
316670eb 155#include <libkern/crypto/aes.h>
3a60a9f5
A		 156
157#include <sys/uio.h>
158#include <sys/conf.h>
159#include <sys/stat.h>
160#include <sys/fcntl.h> // (FWRITE, ...)
3a60a9f5 161#include <sys/sysctl.h>
6d2010ae 162#include <sys/kdebug.h>
fe8ab488 163#include <stdint.h>
3a60a9f5
A		 164
165#include <IOKit/IOHibernatePrivate.h>
166#include <IOKit/IOPolledInterface.h>
167#include <IOKit/IONVRAM.h>
168#include "IOHibernateInternal.h"
39236c6e 169#include <vm/WKdm_new.h>
39037602 170#include <vm/vm_protos.h>
3a60a9f5 171#include "IOKitKernelInternal.h"
6d2010ae
A		 172#include <pexpert/device_tree.h>
173
174#include <machine/pal_routines.h>
175#include <machine/pal_hibernate.h>
39236c6e 176#include <i386/tsc.h>
39037602 177#include <i386/cpuid.h>
5ba3f43e 178#include <san/kasan.h>
6d2010ae 179
0a7de745
A		 180extern "C" addr64_t kvtophys(vm_offset_t va);
181extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
3a60a9f5
A		 182
183/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
184
0a7de745
A		 185#define DISABLE_TRIM 0
186#define TRIM_DELAY 25000
39236c6e 187
0a7de745
A		 188extern unsigned int save_kdebug_enable;
189extern uint32_t gIOHibernateState;
190uint32_t gIOHibernateMode;
191static char gIOHibernateBootSignature[256 + 1];
192static char gIOHibernateFilename[MAXPATHLEN + 1];
a39ff7e2 193
0a7de745 194static uuid_string_t gIOHibernateBridgeBootSessionUUIDString;
a39ff7e2 195
0a7de745
A		 196static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
197uint32_t gIOHibernateFreeTime = 0 * 1000; // max time to spend freeing pages (ms)
198static uint64_t gIOHibernateCompression = 0x80; // default compression 50%
39037602 199boolean_t gIOHibernateStandbyDisabled;
3a60a9f5 200
0a7de745
A		 201static IODTNVRAM * gIOOptionsEntry;
202static IORegistryEntry * gIOChosenEntry;
3e170ce0 203
0a7de745
A		 204static const OSSymbol * gIOHibernateBootImageKey;
205static const OSSymbol * gIOHibernateBootSignatureKey;
206static const OSSymbol * gIOBridgeBootSessionUUIDKey;
3e170ce0 207
b0d623f7 208#if defined(__i386__) || defined(__x86_64__)
3e170ce0 209
0a7de745 210static const OSSymbol * gIOHibernateRTCVariablesKey;
060df5ea
A		 211static const OSSymbol * gIOHibernateBoot0082Key;
212static const OSSymbol * gIOHibernateBootNextKey;
0a7de745
A		 213static OSData * gIOHibernateBoot0082Data;
214static OSData * gIOHibernateBootNextData;
215static OSObject * gIOHibernateBootNextSave;
3e170ce0 216
3e170ce0 217#endif /* defined(__i386__) || defined(__x86_64__) */
3a60a9f5 218
316670eb 219static IOLock * gFSLock;
0a7de745 220uint32_t gFSState;
813fb2f6 221static thread_call_t gIOHibernateTrimCalloutEntry;
0a7de745
A		 222static IOPolledFileIOVars gFileVars;
223static IOHibernateVars gIOHibernateVars;
224static IOPolledFileCryptVars gIOHibernateCryptWakeContext;
225static hibernate_graphics_t _hibernateGraphics;
226static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
227static hibernate_statistics_t _hibernateStats;
228static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats;
229
230enum{
231 kFSIdle = 0,
232 kFSOpening = 2,
233 kFSOpened = 3,
234 kFSTimedOut = 4,
235 kFSTrimDelay = 5
316670eb
A		 236};
237
238static IOReturn IOHibernateDone(IOHibernateVars * vars);
3e170ce0
A		 239static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature);
240static void IOSetBootImageNVRAM(OSData * data);
5ba3f43e 241static void IOHibernateSystemPostWakeTrim(void * p1, void * p2);
316670eb 242
3a60a9f5
A		 243/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
244
3a60a9f5 245enum { kDefaultIOSize = 128 * 1024 };
22ba694c 246enum { kVideoMapSize = 80 * 1024 * 1024 };
3a60a9f5 247
3a60a9f5
A		 248/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
249
250// copy from phys addr to MD
251
252static IOReturn
253IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
0a7de745 254 IOByteCount offset, addr64_t bytes, IOByteCount length)
3a60a9f5 255{
0a7de745
A		 256 addr64_t srcAddr = bytes;
257 IOByteCount remaining;
3a60a9f5 258
0a7de745
A		 259 remaining = length = min(length, md->getLength() - offset);
260 while (remaining) { // (process another target segment?)
261 addr64_t dstAddr64;
262 IOByteCount dstLen;
3a60a9f5 263
0a7de745
A		 264 dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
265 if (!dstAddr64) {
266 break;
267 }
3a60a9f5 268
0a7de745
A		 269 // Clip segment length to remaining
270 if (dstLen > remaining) {
271 dstLen = remaining;
272 }
3a60a9f5
A		 273
274#if 1
0a7de745 275 bcopy_phys(srcAddr, dstAddr64, dstLen);
3a60a9f5 276#else
0a7de745
A		 277 copypv(srcAddr, dstAddr64, dstLen,
278 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
3a60a9f5 279#endif
0a7de745
A		 280 srcAddr += dstLen;
281 offset += dstLen;
282 remaining -= dstLen;
283 }
3a60a9f5 284
0a7de745 285 assert(!remaining);
3a60a9f5 286
0a7de745 287 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
3a60a9f5
A		 288}
289
290// copy from MD to phys addr
291
292static IOReturn
293IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
0a7de745 294 IOByteCount offset, addr64_t bytes, IOByteCount length)
3a60a9f5 295{
0a7de745
A		 296 addr64_t dstAddr = bytes;
297 IOByteCount remaining;
3a60a9f5 298
0a7de745
A		 299 remaining = length = min(length, md->getLength() - offset);
300 while (remaining) { // (process another target segment?)
301 addr64_t srcAddr64;
302 IOByteCount dstLen;
3a60a9f5 303
0a7de745
A		 304 srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
305 if (!srcAddr64) {
306 break;
307 }
3a60a9f5 308
0a7de745
A		 309 // Clip segment length to remaining
310 if (dstLen > remaining) {
311 dstLen = remaining;
312 }
3a60a9f5
A		 313
314#if 1
0a7de745 315 bcopy_phys(srcAddr64, dstAddr, dstLen);
3a60a9f5 316#else
0a7de745
A		 317 copypv(srcAddr, dstAddr64, dstLen,
318 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
3a60a9f5 319#endif
0a7de745
A		 320 dstAddr += dstLen;
321 offset += dstLen;
322 remaining -= dstLen;
323 }
3a60a9f5 324
0a7de745 325 assert(!remaining);
3a60a9f5 326
0a7de745 327 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
3a60a9f5
A		 328}
329
330/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
331
332void
333hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
0a7de745 334 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
3a60a9f5 335{
0a7de745
A		 336 count += ppnum;
337 switch (kind) {
338 case kIOHibernatePageStateUnwiredSave:
339 // unwired save
340 for (; ppnum < count; ppnum++) {
341 hibernate_page_bitset(page_list, FALSE, ppnum);
342 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
343 }
344 break;
345 case kIOHibernatePageStateWiredSave:
346 // wired save
347 for (; ppnum < count; ppnum++) {
348 hibernate_page_bitset(page_list, FALSE, ppnum);
349 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
350 }
351 break;
352 case kIOHibernatePageStateFree:
353 // free page
354 for (; ppnum < count; ppnum++) {
355 hibernate_page_bitset(page_list, TRUE, ppnum);
356 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
357 }
358 break;
359 default:
360 panic("hibernate_set_page_state");
361 }
3a60a9f5
A		 362}
363
3e170ce0
A		 364static vm_offset_t
365hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
366{
0a7de745
A		 367 uint32_t page = *pPage;
368 uint32_t count;
369 hibernate_bitmap_t * bitmap;
370
371 while ((bitmap = hibernate_page_bitmap_pin(list, &page))) {
372 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
373 if (!count) {
374 break;
375 }
376 page += count;
377 if (page <= bitmap->last_page) {
378 break;
379 }
380 }
381
382 *pPage = page;
383 if (bitmap) {
384 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
385 } else {
386 count = 0;
387 }
388
389 return count;
fe8ab488 390}
fe8ab488
A		 391
392/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
393
3a60a9f5
A		 394IOReturn
395IOHibernateSystemSleep(void)
396{
0a7de745
A		 397 IOReturn err;
398 OSData * nvramData;
399 OSObject * obj;
400 OSString * str;
401 OSNumber * num;
402 bool dsSSD, vmflush, swapPinned;
403 IOHibernateVars * vars;
404 uint64_t setFileSize = 0;
405
406 gIOHibernateState = kIOHibernateStateInactive;
407
408 gIOHibernateDebugFlags = 0;
409 if (kIOLogHibernate & gIOKitDebug) {
410 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
411 }
412
413 if (IOService::getPMRootDomain()->getHibernateSettings(
414 &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime)) {
415 if (kIOHibernateModeSleep & gIOHibernateMode) {
416 // default to discard clean for safe sleep
417 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
418 | kIOHibernateModeDiscardCleanActive);
419 }
420 }
421
422 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey))) {
423 if ((str = OSDynamicCast(OSString, obj))) {
424 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
2d21ac55 425 sizeof(gIOHibernateFilename));
0a7de745
A		 426 }
427 obj->release();
428 }
db609669 429
0a7de745
A		 430 if (!gIOHibernateMode || !gIOHibernateFilename[0]) {
431 return kIOReturnUnsupported;
432 }
3e170ce0 433
0a7de745 434 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
6d2010ae 435
0a7de745
A		 436 vars = IONew(IOHibernateVars, 1);
437 if (!vars) {
438 return kIOReturnNoMemory;
439 }
440 bzero(vars, sizeof(*vars));
6d2010ae 441
0a7de745
A		 442 IOLockLock(gFSLock);
443 if (!gIOHibernateTrimCalloutEntry) {
444 gIOHibernateTrimCalloutEntry = thread_call_allocate(&IOHibernateSystemPostWakeTrim, &gFSLock);
445 }
446 IOHibernateSystemPostWakeTrim(NULL, NULL);
447 thread_call_cancel(gIOHibernateTrimCalloutEntry);
448 if (kFSIdle != gFSState) {
449 HIBLOG("hibernate file busy\n");
450 IOLockUnlock(gFSLock);
451 IODelete(vars, IOHibernateVars, 1);
452 return kIOReturnBusy;
3e170ce0 453 }
0a7de745
A		 454 gFSState = kFSOpening;
455 IOLockUnlock(gFSLock);
456
457 swapPinned = false;
458 do{
459 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
460 2 * page_size + WKdm_SCRATCH_BUF_SIZE_INTERNAL, page_size);
461
462 vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
463 ptoa_64(gIOHibernateHandoffPageCount), page_size);
464
465 if (!vars->srcBuffer || !vars->handoffBuffer) {
466 err = kIOReturnNoMemory;
467 break;
468 }
469
470 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey))) {
471 if ((num = OSDynamicCast(OSNumber, obj))) {
472 vars->fileMinSize = num->unsigned64BitValue();
473 }
474 obj->release();
475 }
476 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey))) {
477 if ((num = OSDynamicCast(OSNumber, obj))) {
478 vars->fileMaxSize = num->unsigned64BitValue();
479 }
480 obj->release();
481 }
482
483 boolean_t encryptedswap = true;
484 uint32_t pageCount;
485 AbsoluteTime startTime, endTime;
486 uint64_t nsec;
487
488 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
489 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
490 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
491
492 vmflush = ((kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
493 err = hibernate_alloc_page_lists(&vars->page_list,
494 &vars->page_list_wired,
495 &vars->page_list_pal);
496 if (KERN_SUCCESS != err) {
cb323159 497 HIBLOG("%s err, hibernate_alloc_page_lists return 0x%x\n", __FUNCTION__, err);
0a7de745
A		 498 break;
499 }
500
501 err = hibernate_pin_swap(TRUE);
502 if (KERN_SUCCESS != err) {
cb323159 503 HIBLOG("%s error, hibernate_pin_swap return 0x%x\n", __FUNCTION__, err);
0a7de745
A		 504 break;
505 }
506 swapPinned = true;
507
508 if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode)) {
509 hibernate_page_list_setall(vars->page_list,
510 vars->page_list_wired,
511 vars->page_list_pal,
512 true /* preflight */,
513 vmflush /* discard */,
514 &pageCount);
515 PE_Video consoleInfo;
516 bzero(&consoleInfo, sizeof(consoleInfo));
517 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
518
519 // estimate: 6% increase in pages compressed
520 // screen preview 2 images compressed 0%
521 setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
522 + vars->page_list->list_size
523 + (consoleInfo.v_width * consoleInfo.v_height * 8);
524 enum { setFileRound = 1024 * 1024ULL };
525 setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
526
527 HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
528 pageCount, (100ULL * gIOHibernateCompression) >> 8,
529 setFileSize, vars->fileMinSize);
530
531 if (!(kIOHibernateModeFileResize & gIOHibernateMode)
532 && (setFileSize < vars->fileMinSize)) {
533 setFileSize = vars->fileMinSize;
534 }
535 }
536
537 vars->volumeCryptKeySize = sizeof(vars->volumeCryptKey);
538 err = IOPolledFileOpen(gIOHibernateFilename,
539 (kIOPolledFileCreate | kIOPolledFileHibernate),
540 setFileSize, 0,
541 gIOHibernateCurrentHeader, sizeof(gIOHibernateCurrentHeader),
542 &vars->fileVars, &nvramData,
543 &vars->volumeCryptKey[0], &vars->volumeCryptKeySize);
544
545 if (KERN_SUCCESS != err) {
546 IOLockLock(gFSLock);
547 if (kFSOpening != gFSState) {
548 err = kIOReturnTimeout;
549 }
550 IOLockUnlock(gFSLock);
551 }
552
553 if (KERN_SUCCESS != err) {
554 HIBLOG("IOPolledFileOpen(%x)\n", err);
555 break;
556 }
557
558 // write extents for debug data usage in EFI
559 IOWriteExtentsToFile(vars->fileVars, kIOHibernateHeaderOpenSignature);
560
561 err = IOPolledFilePollersSetup(vars->fileVars, kIOPolledPreflightState);
562 if (KERN_SUCCESS != err) {
563 break;
564 }
565
566 clock_get_uptime(&startTime);
567 err = hibernate_setup(gIOHibernateCurrentHeader,
568 vmflush,
569 vars->page_list, vars->page_list_wired, vars->page_list_pal);
570 clock_get_uptime(&endTime);
571 SUB_ABSOLUTETIME(&endTime, &startTime);
572 absolutetime_to_nanoseconds(endTime, &nsec);
573
574 boolean_t haveSwapPin, hibFileSSD;
575 haveSwapPin = vm_swap_files_pinned();
576
577 hibFileSSD = (kIOPolledFileSSD & vars->fileVars->flags);
578
579 HIBLOG("hibernate_setup(%d) took %qd ms, swapPin(%d) ssd(%d)\n",
580 err, nsec / 1000000ULL,
581 haveSwapPin, hibFileSSD);
582 if (KERN_SUCCESS != err) {
583 break;
584 }
585
586 gIOHibernateStandbyDisabled = ((!haveSwapPin || !hibFileSSD));
587
588 dsSSD = ((0 != (kIOPolledFileSSD & vars->fileVars->flags))
589 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
590
591 if (dsSSD) {
592 gIOHibernateCurrentHeader->options |= kIOHibernateOptionSSD | kIOHibernateOptionColor;
593 } else {
594 gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
595 }
596
597
598#if defined(__i386__) || defined(__x86_64__)
599 if (vars->volumeCryptKeySize &&
600 (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey))) {
601 uintptr_t smcVars[2];
602 smcVars[0] = vars->volumeCryptKeySize;
603 smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
604
605 IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
606 bzero(smcVars, sizeof(smcVars));
607 }
6d2010ae 608#endif
0b4c1975 609
0b4c1975 610
0a7de745
A		 611 if (encryptedswap || vars->volumeCryptKeySize) {
612 gIOHibernateMode ^= kIOHibernateModeEncrypt;
613 }
3a60a9f5 614
0a7de745
A		 615 if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options) {
616 vars->videoAllocSize = kVideoMapSize;
617 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize, VM_KERN_MEMORY_IOKIT)) {
618 vars->videoMapping = 0;
619 }
620 }
3a60a9f5 621
0a7de745
A		 622 // generate crypt keys
623 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++) {
624 vars->wiredCryptKey[i] = random();
625 }
626 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++) {
627 vars->cryptKey[i] = random();
628 }
3a60a9f5 629
0a7de745 630 // set nvram
3a60a9f5 631
0a7de745
A		 632 IOSetBootImageNVRAM(nvramData);
633 nvramData->release();
3a60a9f5 634
b0d623f7 635#if defined(__i386__) || defined(__x86_64__)
0c530ab8 636 {
0a7de745
A		 637 struct AppleRTCHibernateVars {
638 uint8_t signature[4];
639 uint32_t revision;
640 uint8_t booterSignature[20];
641 uint8_t wiredCryptKey[16];
642 };
643 AppleRTCHibernateVars rtcVars;
644 OSData * data;
645
646 rtcVars.signature[0] = 'A';
647 rtcVars.signature[1] = 'A';
648 rtcVars.signature[2] = 'P';
649 rtcVars.signature[3] = 'L';
650 rtcVars.revision = 1;
651 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
652
653 if (gIOChosenEntry
654 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOHibernateBootSignatureKey)))
655 && (sizeof(rtcVars.booterSignature) <= data->getLength())) {
656 bcopy(data->getBytesNoCopy(), &rtcVars.booterSignature[0], sizeof(rtcVars.booterSignature));
657 } else if (gIOHibernateBootSignature[0]) {
658 char c;
659 uint8_t value = 0;
660 uint32_t in, out, digits;
661 for (in = out = digits = 0;
662 (c = gIOHibernateBootSignature[in]) && (in < sizeof(gIOHibernateBootSignature));
663 in++) {
664 if ((c >= 'a') && (c <= 'f')) {
665 c -= 'a' - 10;
666 } else if ((c >= 'A') && (c <= 'F')) {
667 c -= 'A' - 10;
668 } else if ((c >= '0') && (c <= '9')) {
669 c -= '0';
670 } else {
671 if (c == '=') {
672 out = digits = value = 0;
673 }
674 continue;
675 }
676 value = (value << 4) | c;
677 if (digits & 1) {
678 rtcVars.booterSignature[out++] = value;
679 if (out >= sizeof(rtcVars.booterSignature)) {
680 break;
681 }
682 }
683 digits++;
684 }
685 }
d26ffc64 686#if DEBUG || DEVELOPMENT
0a7de745
A		 687 if (kIOLogHibernate & gIOKitDebug) {
688 IOKitKernelLogBuffer("H> rtc:",
689 &rtcVars, sizeof(rtcVars), &kprintf);
690 }
d26ffc64
A		 691#endif /* DEBUG || DEVELOPMENT */
692
0a7de745
A		 693 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
694 if (data) {
695 if (gIOHibernateRTCVariablesKey) {
696 IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
697 }
698 data->release();
060df5ea 699 }
0a7de745
A		 700 if (gIOChosenEntry && gIOOptionsEntry) {
701 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
702 if (data) {
703 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
704 }
705 // set BootNext
706 if (!gIOHibernateBoot0082Data) {
cb323159 707 OSData * fileData = NULL;
0a7de745
A		 708 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
709 if (data && data->getLength() >= 4) {
710 fileData = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-file-path"));
711 }
712 if (data) {
713 // AppleNVRAM_EFI_LOAD_OPTION
714 struct {
715 uint32_t Attributes;
716 uint16_t FilePathLength;
717 uint16_t Desc;
718 } loadOptionHeader;
719 loadOptionHeader.Attributes = 1;
720 loadOptionHeader.FilePathLength = data->getLength();
721 loadOptionHeader.Desc = 0;
722 if (fileData) {
723 loadOptionHeader.FilePathLength -= 4;
724 loadOptionHeader.FilePathLength += fileData->getLength();
725 }
726 gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
727 if (gIOHibernateBoot0082Data) {
728 gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
729 if (fileData) {
730 gIOHibernateBoot0082Data->appendBytes(data->getBytesNoCopy(), data->getLength() - 4);
731 gIOHibernateBoot0082Data->appendBytes(fileData);
732 } else {
733 gIOHibernateBoot0082Data->appendBytes(data);
734 }
735 }
736 }
737 }
738 if (!gIOHibernateBootNextData) {
739 uint16_t bits = 0x0082;
740 gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
741 }
d26ffc64
A		 742
743#if DEBUG || DEVELOPMENT
0a7de745
A		 744 if (kIOLogHibernate & gIOKitDebug) {
745 IOKitKernelLogBuffer("H> bootnext:",
746 gIOHibernateBoot0082Data->getBytesNoCopy(), gIOHibernateBoot0082Data->getLength(), &kprintf);
747 }
d26ffc64 748#endif /* DEBUG || DEVELOPMENT */
0a7de745
A		 749 if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData) {
750 gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
751 gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
752 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
753 }
754 // BootNext
755 }
3e170ce0 756 }
3e170ce0 757#endif /* !i386 && !x86_64 */
0a7de745
A		 758 }while (false);
759
760 if (swapPinned) {
761 hibernate_pin_swap(FALSE);
762 }
763
764 IOLockLock(gFSLock);
765 if ((kIOReturnSuccess == err) && (kFSOpening != gFSState)) {
766 HIBLOG("hibernate file close due timeout\n");
767 err = kIOReturnTimeout;
768 }
769 if (kIOReturnSuccess == err) {
770 gFSState = kFSOpened;
771 gIOHibernateVars = *vars;
772 gFileVars = *vars->fileVars;
773 gFileVars.allocated = false;
774 gIOHibernateVars.fileVars = &gFileVars;
775 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
776 gIOHibernateState = kIOHibernateStateHibernating;
d26ffc64
A		 777
778#if DEBUG || DEVELOPMENT
0a7de745
A		 779 if (kIOLogHibernate & gIOKitDebug) {
780 OSData * data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
781 if (data) {
782 uintptr_t * smcVars = (typeof(smcVars))data->getBytesNoCopy();
783 IOKitKernelLogBuffer("H> smc:",
784 (const void *)smcVars[1], smcVars[0], &kprintf);
785 }
786 }
d26ffc64 787#endif /* DEBUG || DEVELOPMENT */
0a7de745
A		 788 } else {
789 IOPolledFileIOVars * fileVars = vars->fileVars;
790 IOHibernateDone(vars);
791 IOPolledFileClose(&fileVars,
3e170ce0 792#if DISABLE_TRIM
0a7de745 793 0, NULL, 0, 0, 0);
3e170ce0 794#else
0a7de745 795 0, NULL, 0, sizeof(IOHibernateImageHeader), setFileSize);
3e170ce0 796#endif
0a7de745
A		 797 gFSState = kFSIdle;
798 }
799 IOLockUnlock(gFSLock);
316670eb 800
0a7de745
A		 801 if (vars->fileVars) {
802 IODelete(vars->fileVars, IOPolledFileIOVars, 1);
803 }
804 IODelete(vars, IOHibernateVars, 1);
3a60a9f5 805
0a7de745 806 return err;
3a60a9f5
A		 807}
808
809/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
810
3e170ce0
A		 811static void
812IOSetBootImageNVRAM(OSData * data)
813{
0a7de745
A		 814 IORegistryEntry * regEntry;
815
816 if (!gIOOptionsEntry) {
817 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
818 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
819 if (regEntry && !gIOOptionsEntry) {
820 regEntry->release();
821 }
822 }
823 if (gIOOptionsEntry && gIOHibernateBootImageKey) {
824 if (data) {
825 gIOOptionsEntry->setProperty(gIOHibernateBootImageKey, data);
d26ffc64 826#if DEBUG || DEVELOPMENT
0a7de745
A		 827 if (kIOLogHibernate & gIOKitDebug) {
828 IOKitKernelLogBuffer("H> boot-image:",
829 data->getBytesNoCopy(), data->getLength(), &kprintf);
830 }
d26ffc64 831#endif /* DEBUG || DEVELOPMENT */
0a7de745
A		 832 } else {
833 gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
834 gIOOptionsEntry->sync();
835 }
3e170ce0 836 }
3e170ce0
A		 837}
838
839/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
a39ff7e2 840/*
3e170ce0
A		 841 * Writes header to disk with signature, block size and file extents data.
842 * If there are more than 2 extents, then they are written on second block.
843 */
844static IOReturn
845IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature)
846{
0a7de745
A		 847 IOHibernateImageHeader hdr;
848 IOItemCount count;
849 IOReturn err = kIOReturnSuccess;
850 int rc;
851 IOPolledFileExtent * fileExtents;
852
853 fileExtents = (typeof(fileExtents))vars->fileExtents->getBytesNoCopy();
854
855 memset(&hdr, 0, sizeof(IOHibernateImageHeader));
856 count = vars->fileExtents->getLength();
857 if (count > sizeof(hdr.fileExtentMap)) {
858 hdr.fileExtentMapSize = count;
859 count = sizeof(hdr.fileExtentMap);
860 } else {
861 hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap);
862 }
863
864 bcopy(fileExtents, &hdr.fileExtentMap[0], count);
865
866 // copy file block extent list if larger than header
867 if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap)) {
868 count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap);
869 rc = kern_write_file(vars->fileRef, vars->blockSize,
870 (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)),
871 count, IO_SKIP_ENCRYPTION);
872 if (rc != 0) {
873 HIBLOG("kern_write_file returned %d\n", rc);
874 err = kIOReturnIOError;
875 goto exit;
876 }
877 }
878 hdr.signature = signature;
879 hdr.deviceBlockSize = vars->blockSize;
880
881 rc = kern_write_file(vars->fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION);
882 if (rc != 0) {
883 HIBLOG("kern_write_file returned %d\n", rc);
884 err = kIOReturnIOError;
885 goto exit;
886 }
3e170ce0
A		 887
888exit:
0a7de745 889 return err;
3e170ce0
A		 890}
891
3e170ce0
A		 892/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
893
2d21ac55
A		 894DECLARE_IOHIBERNATEPROGRESSALPHA
895
896static void
897ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
898{
0a7de745
A		 899 uint32_t rowBytes, pixelShift;
900 uint32_t x, y;
901 int32_t blob;
902 uint32_t alpha, in, color, result;
903 uint8_t * out;
904 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
905
906 rowBytes = display->rowBytes;
907 pixelShift = display->depth >> 4;
908 if (pixelShift < 1) {
909 return;
910 }
911
912 screen += ((display->width
913 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
914 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
915
916 for (y = 0; y < kIOHibernateProgressHeight; y++) {
917 out = screen + y * rowBytes;
918 for (blob = 0; blob < kIOHibernateProgressCount; blob++) {
919 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
920 for (x = 0; x < kIOHibernateProgressWidth; x++) {
921 alpha = gIOHibernateProgressAlpha[y][x];
922 result = color;
923 if (alpha) {
924 if (0xff != alpha) {
925 if (1 == pixelShift) {
926 in = *((uint16_t *)out) & 0x1f; // 16
927 in = (in << 3) | (in >> 2);
928 } else {
929 in = *((uint32_t *)out) & 0xff; // 32
930 }
931 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
932 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
933 }
934 if (1 == pixelShift) {
935 result >>= 3;
936 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
937 } else {
938 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
939 }
940 }
941 out += (1 << pixelShift);
942 }
943 out += (kIOHibernateProgressSpacing << pixelShift);
944 }
945 }
2d21ac55
A		 946}
947
948
949static void
950ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
951{
0a7de745
A		 952 uint32_t rowBytes, pixelShift;
953 uint32_t x, y;
954 int32_t blob, lastBlob;
955 uint32_t alpha, in, color, result;
956 uint8_t * out;
957 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
958
959 pixelShift = display->depth >> 4;
960 if (pixelShift < 1) {
961 return;
962 }
963
964 rowBytes = display->rowBytes;
965
966 screen += ((display->width
967 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
968 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
969
970 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
971
972 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
973
974 for (y = 0; y < kIOHibernateProgressHeight; y++) {
975 out = screen + y * rowBytes;
976 for (blob = firstBlob; blob <= lastBlob; blob++) {
977 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
978 for (x = 0; x < kIOHibernateProgressWidth; x++) {
979 alpha = gIOHibernateProgressAlpha[y][x];
980 result = color;
981 if (alpha) {
982 if (0xff != alpha) {
983 in = display->progressSaveUnder[blob][saveindex[blob]++];
984 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
985 }
986 if (1 == pixelShift) {
987 result >>= 3;
988 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
989 } else {
990 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
991 }
992 }
993 out += (1 << pixelShift);
994 }
995 out += (kIOHibernateProgressSpacing << pixelShift);
996 }
997 }
2d21ac55
A		 998}
999
1000/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1001
316670eb
A		 1002IOReturn
1003IOHibernateIOKitSleep(void)
1004{
0a7de745
A		 1005 IOReturn ret = kIOReturnSuccess;
1006 IOLockLock(gFSLock);
1007 if (kFSOpening == gFSState) {
1008 gFSState = kFSTimedOut;
1009 HIBLOG("hibernate file open timed out\n");
1010 ret = kIOReturnTimeout;
1011 }
1012 IOLockUnlock(gFSLock);
1013 return ret;
316670eb
A		 1014}
1015
1016/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1017
3a60a9f5
A		 1018IOReturn
1019IOHibernateSystemHasSlept(void)
1020{
0a7de745
A		 1021 IOReturn ret = kIOReturnSuccess;
1022 IOHibernateVars * vars = &gIOHibernateVars;
cb323159 1023 OSObject * obj = NULL;
0a7de745
A		 1024 OSData * data;
1025
1026 IOLockLock(gFSLock);
1027 if ((kFSOpened != gFSState) && gIOHibernateMode) {
1028 ret = kIOReturnTimeout;
1029 }
1030 IOLockUnlock(gFSLock);
1031 if (kIOReturnSuccess != ret) {
1032 return ret;
1033 }
1034
1035 if (gIOHibernateMode) {
1036 obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
1037 }
1038 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1039 if (obj && !vars->previewBuffer) {
1040 obj->release();
1041 }
1042
1043 vars->consoleMapping = NULL;
1044 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare())) {
1045 vars->previewBuffer->release();
cb323159 1046 vars->previewBuffer = NULL;
0a7de745
A		 1047 }
1048
1049 if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1050 && vars->previewBuffer
1051 && (data = OSDynamicCast(OSData,
1052 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey)))) {
1053 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
1054 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
1055
1056 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1057
1058 if (kIOHibernatePreviewUpdates & flags) {
1059 PE_Video consoleInfo;
1060 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1061
1062 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1063
1064 graphicsInfo->width = consoleInfo.v_width;
1065 graphicsInfo->height = consoleInfo.v_height;
1066 graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
1067 graphicsInfo->depth = consoleInfo.v_depth;
1068 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1069
1070 HIBPRINT("video %p %d %d %d\n",
1071 vars->consoleMapping, graphicsInfo->depth,
1072 graphicsInfo->width, graphicsInfo->height);
1073 if (vars->consoleMapping) {
1074 ProgressInit(graphicsInfo, vars->consoleMapping,
1075 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1076 }
1077 }
1078 }
1079
1080 if (gIOOptionsEntry) {
1081 gIOOptionsEntry->sync();
1082 }
1083
1084 return ret;
3a60a9f5
A		 1085}
1086
1087/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1088
6d2010ae
A		 1089static DeviceTreeNode *
1090MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry)
1091{
0a7de745
A		 1092 DeviceTreeNodeProperty * prop;
1093 DeviceTreeNode * child;
1094 IORegistryEntry * childRegEntry;
1095 const char * nameProp;
1096 unsigned int propLen, idx;
1097
1098 prop = (DeviceTreeNodeProperty *) (entry + 1);
1099 for (idx = 0; idx < entry->nProperties; idx++) {
1100 if (regEntry && (0 != strcmp("name", prop->name))) {
1101 regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
6d2010ae 1102// HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
0a7de745
A		 1103 }
1104 prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
6d2010ae 1105 }
6d2010ae 1106
0a7de745
A		 1107 child = (DeviceTreeNode *) prop;
1108 for (idx = 0; idx < entry->nChildren; idx++) {
1109 if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen)) {
1110 panic("no name");
1111 }
1112 childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
6d2010ae 1113// HIBPRINT("%s == %p\n", nameProp, childRegEntry);
0a7de745
A		 1114 child = MergeDeviceTree(child, childRegEntry);
1115 }
1116 return child;
6d2010ae
A		 1117}
1118
3a60a9f5
A		 1119IOReturn
1120IOHibernateSystemWake(void)
1121{
0a7de745
A		 1122 if (kFSOpened == gFSState) {
1123 IOPolledFilePollersClose(gIOHibernateVars.fileVars, kIOPolledPostflightState);
1124 IOHibernateDone(&gIOHibernateVars);
1125 } else {
1126 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1127 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1128 }
1129 return kIOReturnSuccess;
316670eb 1130}
3a60a9f5 1131
316670eb
A		 1132static IOReturn
1133IOHibernateDone(IOHibernateVars * vars)
1134{
0a7de745
A		 1135 IOReturn err;
1136 OSData * data;
1137
1138 hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
1139
1140 if (vars->videoMapping) {
1141 if (vars->videoMapSize) {
1142 // remove mappings
1143 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1144 }
1145 if (vars->videoAllocSize) {
1146 // dealloc range
1147 kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
1148 }
1149 }
1150
1151 if (vars->previewBuffer) {
1152 vars->previewBuffer->release();
cb323159 1153 vars->previewBuffer = NULL;
0a7de745
A		 1154 }
1155
1156 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
1157 IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
1158 gIOHibernateCurrentHeader->options, 32);
1159 } else {
1160 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1161 }
1162
1163 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1164 && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus)) {
1165 IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
1166 &gIOHibernateGraphicsInfo->gfxStatus,
1167 sizeof(gIOHibernateGraphicsInfo->gfxStatus));
1168 } else {
1169 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1170 }
1171
1172 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
3a60a9f5 1173
b0d623f7 1174#if defined(__i386__) || defined(__x86_64__)
060df5ea 1175 IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
6d2010ae 1176 IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
2d21ac55
A		 1177
1178 /*
1179 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1180 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1181 */
1182 if (gIOOptionsEntry) {
0a7de745
A		 1183 if (gIOHibernateRTCVariablesKey) {
1184 if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
1185 gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
1186 }
060df5ea 1187 }
060df5ea 1188
0a7de745
A		 1189 if (gIOHibernateBootNextKey) {
1190 if (gIOHibernateBootNextSave) {
1191 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
1192 gIOHibernateBootNextSave->release();
1193 gIOHibernateBootNextSave = NULL;
1194 } else {
1195 gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
1196 }
1197 }
1198 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) {
1199 gIOOptionsEntry->sync();
2d21ac55
A		 1200 }
1201 }
0c530ab8 1202#endif
3a60a9f5 1203
0a7de745
A		 1204 if (vars->srcBuffer) {
1205 vars->srcBuffer->release();
1206 }
1207 bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
1208 if (vars->handoffBuffer) {
1209 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
1210 IOHibernateHandoff * handoff;
1211 bool done = false;
1212 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
1213 !done;
1214 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount]) {
1215 HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
1216 uint8_t * data = &handoff->data[0];
1217 switch (handoff->type) {
1218 case kIOHibernateHandoffTypeEnd:
1219 done = true;
1220 break;
1221
1222 case kIOHibernateHandoffTypeDeviceTree:
1223 MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
1224 break;
1225
1226 case kIOHibernateHandoffTypeKeyStore:
6d2010ae 1227#if defined(__i386__) || defined(__x86_64__)
0a7de745
A		 1228 {
1229 IOBufferMemoryDescriptor *
1230 md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
1231 if (md) {
1232 IOSetKeyStoreData(md);
1233 }
1234 }
6d2010ae 1235#endif
0a7de745 1236 break;
a39ff7e2 1237
0a7de745
A		 1238 default:
1239 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
1240 break;
1241 }
1242 }
a39ff7e2 1243#if defined(__i386__) || defined(__x86_64__)
0a7de745
A		 1244 if (vars->volumeCryptKeySize) {
1245 IOBufferMemoryDescriptor *
1246 bmd = IOBufferMemoryDescriptor::withBytes(&vars->volumeCryptKey[0],
1247 vars->volumeCryptKeySize, kIODirectionOutIn);
1248 if (!bmd) {
1249 panic("IOBufferMemoryDescriptor");
1250 }
1251 IOSetAPFSKeyStoreData(bmd);
1252 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
1253 }
a39ff7e2 1254#endif
0a7de745
A		 1255 }
1256 vars->handoffBuffer->release();
6d2010ae 1257 }
3a60a9f5 1258
0a7de745
A		 1259 if (gIOChosenEntry
1260 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey)))
1261 && (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength())) {
1262 bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0],
1263 sizeof(gIOHibernateBridgeBootSessionUUIDString));
1264 }
a39ff7e2 1265
0a7de745
A		 1266 if (vars->hwEncrypt) {
1267 err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, NULL, 0);
1268 HIBLOG("IOPolledFilePollersSetEncryptionKey(0,%x)\n", err);
1269 }
a39ff7e2 1270
0a7de745 1271 bzero(vars, sizeof(*vars));
3a60a9f5
A		 1272
1273// gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1274
0a7de745 1275 return kIOReturnSuccess;
3a60a9f5
A		 1276}
1277
5ba3f43e 1278static void
813fb2f6 1279IOHibernateSystemPostWakeTrim(void * p1, void * p2)
3a60a9f5 1280{
0a7de745
A		 1281 // invalidate & close the image file
1282 if (p1) {
1283 IOLockLock(gFSLock);
1284 }
1285 if (kFSTrimDelay == gFSState) {
1286 IOPolledFileIOVars * vars = &gFileVars;
1287 IOPolledFileClose(&vars,
39236c6e 1288#if DISABLE_TRIM
0a7de745 1289 0, NULL, 0, 0, 0);
39236c6e 1290#else
0a7de745
A		 1291 0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader),
1292 sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize);
39236c6e 1293#endif
0a7de745
A		 1294 gFSState = kFSIdle;
1295 }
1296 if (p1) {
1297 IOLockUnlock(gFSLock);
1298 }
813fb2f6 1299}
fe8ab488 1300
813fb2f6 1301IOReturn
5ba3f43e 1302IOHibernateSystemPostWake(bool now)
813fb2f6 1303{
0a7de745 1304 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
cb323159 1305 IOSetBootImageNVRAM(NULL);
0a7de745
A		 1306
1307 IOLockLock(gFSLock);
1308 if (kFSTrimDelay == gFSState) {
1309 thread_call_cancel(gIOHibernateTrimCalloutEntry);
1310 IOHibernateSystemPostWakeTrim(NULL, NULL);
1311 } else if (kFSOpened != gFSState) {
1312 gFSState = kFSIdle;
1313 } else {
1314 gFSState = kFSTrimDelay;
1315 if (now) {
1316 thread_call_cancel(gIOHibernateTrimCalloutEntry);
1317 IOHibernateSystemPostWakeTrim(NULL, NULL);
1318 } else {
1319 AbsoluteTime deadline;
1320 clock_interval_to_deadline(TRIM_DELAY, kMillisecondScale, &deadline );
1321 thread_call_enter1_delayed(gIOHibernateTrimCalloutEntry, NULL, deadline);
1322 }
1323 }
1324 IOLockUnlock(gFSLock);
1325
1326 return kIOReturnSuccess;
3a60a9f5
A		 1327}
1328
0a7de745
A		 1329uint32_t
1330IOHibernateWasScreenLocked(void)
316670eb 1331{
0a7de745
A		 1332 uint32_t ret = 0;
1333 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry) {
1334 OSData *
1335 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
1336 if (data) {
1337 ret = ((uint32_t *)data->getBytesNoCopy())[0];
1338 gIOChosenEntry->setProperty(kIOBooterScreenLockStateKey, data);
1339 }
1340 } else {
1341 gIOChosenEntry->removeProperty(kIOBooterScreenLockStateKey);
1342 }
1343
1344 return ret;
316670eb
A		 1345}
1346
3a60a9f5
A		 1347/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1348
a39ff7e2 1349SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
0a7de745
A		 1350 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1351 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
a39ff7e2 1352SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
0a7de745
A		 1353 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1354 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
a39ff7e2 1355SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
0a7de745
A		 1356 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1357 &gIOHibernateMode, 0, "");
39236c6e 1358SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
0a7de745
A		 1359 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1360 &_hibernateStats, hibernate_statistics_t, "");
a39ff7e2 1361SYSCTL_STRING(_kern_bridge, OID_AUTO, bootsessionuuid,
0a7de745
A		 1362 CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1363 gIOHibernateBridgeBootSessionUUIDString, sizeof(gIOHibernateBridgeBootSessionUUIDString), "");
39236c6e
A		 1364
1365SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
0a7de745
A		 1366 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1367 &_hibernateStats.graphicsReadyTime, 0, "");
39236c6e 1368SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
0a7de745
A		 1369 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1370 &_hibernateStats.wakeNotificationTime, 0, "");
39236c6e 1371SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
0a7de745
A		 1372 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1373 &_hibernateStats.lockScreenReadyTime, 0, "");
39236c6e 1374SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
0a7de745
A		 1375 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1376 &_hibernateStats.hidReadyTime, 0, "");
39236c6e 1377
3a60a9f5
A		 1378void
1379IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1380{
0a7de745
A		 1381 gIOHibernateBootImageKey = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1382 gIOHibernateBootSignatureKey = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
1383 gIOBridgeBootSessionUUIDKey = OSSymbol::withCStringNoCopy(kIOBridgeBootSessionUUIDKey);
3e170ce0
A		 1384
1385#if defined(__i386__) || defined(__x86_64__)
0a7de745
A		 1386 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1387 gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
1388 gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
1389 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
3e170ce0
A		 1390#endif /* defined(__i386__) || defined(__x86_64__) */
1391
0a7de745
A		 1392 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
1393 if (data) {
1394 rootDomain->setProperty(kIOHibernateStateKey, data);
1395 data->release();
1396 }
1397
1398 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename))) {
1399 gIOHibernateMode = kIOHibernateModeOn;
1400 } else {
1401 gIOHibernateFilename[0] = 0;
1402 }
1403
1404 sysctl_register_oid(&sysctl__kern_hibernatefile);
1405 sysctl_register_oid(&sysctl__kern_bootsignature);
1406 sysctl_register_oid(&sysctl__kern_hibernatemode);
1407 sysctl_register_oid(&sysctl__kern_hibernatestatistics);
1408 sysctl_register_oid(&sysctl__kern_hibernategraphicsready);
1409 sysctl_register_oid(&sysctl__kern_hibernatewakenotification);
1410 sysctl_register_oid(&sysctl__kern_hibernatelockscreenready);
1411 sysctl_register_oid(&sysctl__kern_hibernatehidready);
1412
1413 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1414
1415 if (gIOChosenEntry
1416 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey)))
1417 && (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength())) {
1418 sysctl_register_oid(&sysctl__kern_bridge_bootsessionuuid);
1419 bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0], sizeof(gIOHibernateBridgeBootSessionUUIDString));
1420 }
1421
1422 gFSLock = IOLockAlloc();
3a60a9f5
A		 1423}
1424
3a60a9f5
A		 1425/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1426
a39ff7e2 1427static IOReturn
3e170ce0 1428IOHibernatePolledFileWrite(IOPolledFileIOVars * vars,
0a7de745
A		 1429 const uint8_t * bytes, IOByteCount size,
1430 IOPolledFileCryptVars * cryptvars)
6d2010ae 1431{
0a7de745 1432 IOReturn err;
6d2010ae 1433
0a7de745
A		 1434 err = IOPolledFileWrite(vars, bytes, size, cryptvars);
1435 if ((kIOReturnSuccess == err) && hibernate_should_abort()) {
1436 err = kIOReturnAborted;
1437 }
6d2010ae 1438
0a7de745 1439 return err;
6d2010ae
A		 1440}
1441
3e170ce0 1442/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
6d2010ae 1443
2d21ac55 1444extern "C" uint32_t
3a60a9f5
A		 1445hibernate_write_image(void)
1446{
0a7de745
A		 1447 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
1448 IOHibernateVars * vars = &gIOHibernateVars;
1449 IOPolledFileExtent * fileExtents;
1450
1451 _static_assert_1_arg(sizeof(IOHibernateImageHeader) == 512);
1452
1453 uint32_t pageCount, pagesDone;
1454 IOReturn err;
1455 vm_offset_t ppnum, page;
1456 IOItemCount count;
1457 uint8_t * src;
1458 uint8_t * data;
1459 uint8_t * compressed;
1460 uint8_t * scratch;
1461 IOByteCount pageCompressedSize;
1462 uint64_t compressedSize, uncompressedSize;
1463 uint64_t image1Size = 0;
1464 uint32_t bitmap_size;
1465 bool iterDone, pollerOpen, needEncrypt;
1466 uint32_t restore1Sum, sum, sum1, sum2;
1467 int wkresult;
1468 uint32_t tag;
1469 uint32_t pageType;
1470 uint32_t pageAndCount[2];
1471 addr64_t phys64;
1472 IOByteCount segLen;
1473 uintptr_t hibernateBase;
1474 uintptr_t hibernateEnd;
1475
1476 AbsoluteTime startTime, endTime;
1477 AbsoluteTime allTime, compTime;
1478 uint64_t compBytes;
1479 uint64_t nsec;
1480 uint32_t lastProgressStamp = 0;
1481 uint32_t progressStamp;
1482 uint32_t blob, lastBlob = (uint32_t) -1L;
1483
1484 uint32_t wiredPagesEncrypted;
1485 uint32_t dirtyPagesEncrypted;
1486 uint32_t wiredPagesClear;
1487 uint32_t svPageCount;
1488 uint32_t zvPageCount;
1489
1490 IOPolledFileCryptVars _cryptvars;
cb323159 1491 IOPolledFileCryptVars * cryptvars = NULL;
0a7de745
A		 1492
1493 wiredPagesEncrypted = 0;
1494 dirtyPagesEncrypted = 0;
1495 wiredPagesClear = 0;
1496 svPageCount = 0;
1497 zvPageCount = 0;
1498
1499 if (!vars->fileVars
1500 || !vars->fileVars->pollers
1501 || !(kIOHibernateModeOn & gIOHibernateMode)) {
1502 return kIOHibernatePostWriteSleep;
1503 }
1504
1505 if (kIOHibernateModeSleep & gIOHibernateMode) {
1506 kdebug_enable = save_kdebug_enable;
1507 }
1508
1509 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START);
1510 IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
1511
1512 restore1Sum = sum1 = sum2 = 0;
3a60a9f5 1513
2d21ac55 1514#if CRYPTO
0a7de745
A		 1515 // encryption data. "iv" is the "initial vector".
1516 if (kIOHibernateModeEncrypt & gIOHibernateMode) {
1517 static const unsigned char first_iv[AES_BLOCK_SIZE]
1518 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
1519 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
1520
1521 cryptvars = &gIOHibernateCryptWakeContext;
1522 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1523 aes_encrypt_key(vars->cryptKey,
1524 kIOHibernateAESKeySize,
1525 &cryptvars->ctx.encrypt);
1526 aes_decrypt_key(vars->cryptKey,
1527 kIOHibernateAESKeySize,
1528 &cryptvars->ctx.decrypt);
1529
1530 cryptvars = &_cryptvars;
1531 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1532 for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++) {
1533 vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
1534 }
1535 aes_encrypt_key(vars->wiredCryptKey,
1536 kIOHibernateAESKeySize,
1537 &cryptvars->ctx.encrypt);
1538
1539 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1540 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1541 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
1542 }
2d21ac55 1543#endif /* CRYPTO */
3a60a9f5 1544
0a7de745
A		 1545 hibernate_page_list_setall(vars->page_list,
1546 vars->page_list_wired,
1547 vars->page_list_pal,
1548 false /* !preflight */,
1549 /* discard_all */
1550 ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
1551 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
1552 &pageCount);
3a60a9f5 1553
0a7de745 1554 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
3a60a9f5 1555
0a7de745 1556 fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy();
3a60a9f5
A		 1557
1558#if 0
0a7de745
A		 1559 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
1560 for (page = 0; page < count; page++) {
1561 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
1562 fileExtents[page].start, fileExtents[page].length,
1563 fileExtents[page].start + fileExtents[page].length);
1564 }
3a60a9f5
A		 1565#endif
1566
0a7de745
A		 1567 needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
1568 AbsoluteTime_to_scalar(&compTime) = 0;
1569 compBytes = 0;
1570
1571 clock_get_uptime(&allTime);
1572 IOService::getPMRootDomain()->pmStatsRecordEvent(
1573 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
1574 do{
1575 compressedSize = 0;
1576 uncompressedSize = 0;
1577 svPageCount = 0;
1578 zvPageCount = 0;
1579
1580 IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
1581
1582 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
1583 ml_get_interrupts_enabled());
1584 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState,
1585 // abortable if not low battery
1586 !IOService::getPMRootDomain()->mustHibernate());
1587 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
1588 pollerOpen = (kIOReturnSuccess == err);
1589 if (!pollerOpen) {
1590 break;
1591 }
1592
1593 if (vars->volumeCryptKeySize) {
1594 err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
1595 HIBLOG("IOPolledFilePollersSetEncryptionKey(%x)\n", err);
1596 vars->hwEncrypt = (kIOReturnSuccess == err);
1597 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
1598 if (vars->hwEncrypt) {
1599 header->options |= kIOHibernateOptionHWEncrypt;
1600 }
1601 }
1602
1603 // copy file block extent list if larger than header
1604
1605 count = vars->fileVars->fileExtents->getLength();
1606 if (count > sizeof(header->fileExtentMap)) {
1607 count -= sizeof(header->fileExtentMap);
1608 err = IOHibernatePolledFileWrite(vars->fileVars,
1609 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
1610 if (kIOReturnSuccess != err) {
1611 break;
1612 }
1613 }
1614
1615 hibernateBase = HIB_BASE; /* Defined in PAL headers */
1616 hibernateEnd = (segHIBB + segSizeHIB);
1617
1618 // copy out restore1 code
1619
1620 for (count = 0;
1621 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1622 count += segLen) {
1623 for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++) {
1624 gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
1625 }
1626 }
1627
1628 page = atop_32(kvtophys(hibernateBase));
1629 count = atop_32(round_page(hibernateEnd) - hibernateBase);
1630 header->restore1CodePhysPage = page;
1631 header->restore1CodeVirt = hibernateBase;
1632 header->restore1PageCount = count;
1633 header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
1634 header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
1635
1636 if (uuid_parse(&gIOHibernateBridgeBootSessionUUIDString[0], &header->bridgeBootSessionUUID[0])) {
1637 bzero(&header->bridgeBootSessionUUID[0], sizeof(header->bridgeBootSessionUUID));
1638 }
1639
1640 // sum __HIB seg, with zeros for the stack
1641 src = (uint8_t *) trunc_page(hibernateBase);
1642 for (page = 0; page < count; page++) {
1643 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0])) {
1644 restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
1645 } else {
1646 restore1Sum += 0x00000000;
1647 }
1648 src += page_size;
1649 }
1650 sum1 = restore1Sum;
1651
1652 // write the __HIB seg, with zeros for the stack
1653
1654 src = (uint8_t *) trunc_page(hibernateBase);
1655 count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
1656 if (count) {
1657 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
1658 if (kIOReturnSuccess != err) {
1659 break;
1660 }
1661 }
1662 err = IOHibernatePolledFileWrite(vars->fileVars,
cb323159 1663 (uint8_t *) NULL,
0a7de745
A		 1664 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
1665 cryptvars);
1666 if (kIOReturnSuccess != err) {
1667 break;
1668 }
1669 src = &gIOHibernateRestoreStackEnd[0];
1670 count = round_page(hibernateEnd) - ((uintptr_t) src);
1671 if (count) {
1672 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
1673 if (kIOReturnSuccess != err) {
1674 break;
1675 }
1676 }
1677
1678 if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) {
1679 vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
1680 vars->fileVars->encryptEnd = UINT64_MAX;
1681 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1682 }
1683
1684 // write the preview buffer
1685
1686 if (vars->previewBuffer) {
1687 ppnum = 0;
1688 count = 0;
1689 do{
1690 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
1691 pageAndCount[0] = atop_64(phys64);
1692 pageAndCount[1] = atop_32(segLen);
1693 err = IOHibernatePolledFileWrite(vars->fileVars,
1694 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1695 cryptvars);
1696 if (kIOReturnSuccess != err) {
1697 break;
1698 }
1699 count += segLen;
1700 ppnum += sizeof(pageAndCount);
1701 }while (phys64);
1702 if (kIOReturnSuccess != err) {
1703 break;
1704 }
1705
1706 src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
316670eb
A		 1707
1708 ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
1709
0a7de745
A		 1710 count = vars->previewBuffer->getLength();
1711
1712 header->previewPageListSize = ppnum;
1713 header->previewSize = count + ppnum;
1714
1715 for (page = 0; page < count; page += page_size) {
1716 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
1717 sum1 += hibernate_sum_page(src + page, atop_64(phys64));
1718 }
1719 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
1720 if (kIOReturnSuccess != err) {
1721 break;
1722 }
1723 }
1724
1725 // mark areas for no save
1726 IOMemoryDescriptor * ioBuffer;
1727 ioBuffer = IOPolledFileGetIOBuffer(vars->fileVars);
1728 for (count = 0;
1729 (phys64 = ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1730 count += segLen) {
1731 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1732 atop_64(phys64), atop_32(segLen),
1733 kIOHibernatePageStateFree);
1734 pageCount -= atop_32(segLen);
1735 }
1736
1737 for (count = 0;
1738 (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1739 count += segLen) {
1740 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1741 atop_64(phys64), atop_32(segLen),
1742 kIOHibernatePageStateFree);
1743 pageCount -= atop_32(segLen);
1744 }
1745
1746 // copy out bitmap of pages available for trashing during restore
1747
1748 bitmap_size = vars->page_list_wired->list_size;
1749 src = (uint8_t *) vars->page_list_wired;
1750 err = IOHibernatePolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
1751 if (kIOReturnSuccess != err) {
1752 break;
1753 }
1754
1755 // mark more areas for no save, but these are not available
1756 // for trashing during restore
1757
1758 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
1759
1760
1761 page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
1762 count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
1763 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1764 page, count,
1765 kIOHibernatePageStateFree);
1766 pageCount -= count;
1767
1768 if (vars->previewBuffer) {
1769 for (count = 0;
1770 (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1771 count += segLen) {
1772 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1773 atop_64(phys64), atop_32(segLen),
1774 kIOHibernatePageStateFree);
1775 pageCount -= atop_32(segLen);
1776 }
1777 }
1778
1779 for (count = 0;
1780 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1781 count += segLen) {
1782 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1783 atop_64(phys64), atop_32(segLen),
1784 kIOHibernatePageStateFree);
1785 pageCount -= atop_32(segLen);
1786 }
6d2010ae 1787
5ba3f43e
A		 1788#if KASAN
1789 vm_size_t shadow_pages_free = atop_64(shadow_ptop) - atop_64(shadow_pnext);
1790
1791 /* no need to save unused shadow pages */
1792 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
0a7de745
A		 1793 atop_64(shadow_pnext),
1794 shadow_pages_free,
1795 kIOHibernatePageStateFree);
5ba3f43e
A		 1796#endif
1797
0a7de745
A		 1798 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
1799 compressed = src + page_size;
1800 scratch = compressed + page_size;
1801
1802 pagesDone = 0;
1803 lastBlob = 0;
1804
1805 HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
1806 bitmap_size, header->previewSize,
1807 pageCount, vars->fileVars->position);
1808
1809 enum
1810 // pageType
d9a64523 1811 {
0a7de745
A		 1812 kWired = 0x02,
1813 kEncrypt = 0x01,
1814 kWiredEncrypt = kWired | kEncrypt,
1815 kWiredClear = kWired,
1816 kUnwiredEncrypt = kEncrypt
1817 };
1818
1819 bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features()));
1820
1821 for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--) {
1822 if (kUnwiredEncrypt == pageType) {
1823 // start unwired image
1824 if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) {
1825 vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
1826 vars->fileVars->encryptEnd = UINT64_MAX;
1827 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1828 }
1829 bcopy(&cryptvars->aes_iv[0],
1830 &gIOHibernateCryptWakeContext.aes_iv[0],
1831 sizeof(cryptvars->aes_iv));
1832 cryptvars = &gIOHibernateCryptWakeContext;
1833 }
1834 for (iterDone = false, ppnum = 0; !iterDone;) {
1835 if (cpuAES && (pageType == kWiredClear)) {
1836 count = 0;
1837 } else {
1838 count = hibernate_page_list_iterate((kWired & pageType) ? vars->page_list_wired : vars->page_list,
1839 &ppnum);
1840 }
0b4c1975 1841// kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
0a7de745
A		 1842 iterDone = !count;
1843
1844 if (!cpuAES) {
1845 if (count && (kWired & pageType) && needEncrypt) {
1846 uint32_t checkIndex;
1847 for (checkIndex = 0;
1848 (checkIndex < count)
1849 && (((kEncrypt & pageType) == 0) == pmap_is_noencrypt(ppnum + checkIndex));
1850 checkIndex++) {
1851 }
1852 if (!checkIndex) {
1853 ppnum++;
1854 continue;
1855 }
1856 count = checkIndex;
1857 }
1858 }
1859
1860 switch (pageType) {
1861 case kWiredEncrypt: wiredPagesEncrypted += count; break;
1862 case kWiredClear: wiredPagesClear += count; break;
1863 case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
1864 }
1865
1866 if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */
1867 } else {
1868 pageAndCount[0] = ppnum;
1869 pageAndCount[1] = count;
1870 err = IOHibernatePolledFileWrite(vars->fileVars,
1871 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1872 cryptvars);
1873 if (kIOReturnSuccess != err) {
1874 break;
1875 }
1876 }
1877
1878 for (page = ppnum; page < (ppnum + count); page++) {
1879 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
1880 if (err) {
1881 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
1882 break;
1883 }
1884
1885 sum = hibernate_sum_page(src, page);
1886 if (kWired & pageType) {
1887 sum1 += sum;
1888 } else {
1889 sum2 += sum;
1890 }
1891
1892 clock_get_uptime(&startTime);
1893 wkresult = WKdm_compress_new((const WK_word*) src,
1894 (WK_word*) compressed,
1895 (WK_word*) scratch,
1896 page_size - 4);
1897
1898 clock_get_uptime(&endTime);
1899 ADD_ABSOLUTETIME(&compTime, &endTime);
1900 SUB_ABSOLUTETIME(&compTime, &startTime);
1901
1902 compBytes += page_size;
1903 pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
1904
1905 if (pageCompressedSize == 0) {
1906 pageCompressedSize = 4;
1907 data = src;
1908
1909 if (*(uint32_t *)src) {
1910 svPageCount++;
1911 } else {
1912 zvPageCount++;
1913 }
1914 } else {
1915 if (pageCompressedSize != page_size) {
1916 data = compressed;
1917 } else {
1918 data = src;
1919 }
1920 }
1921
1922 tag = pageCompressedSize | kIOHibernateTagSignature;
1923 err = IOHibernatePolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
1924 if (kIOReturnSuccess != err) {
1925 break;
1926 }
1927
1928 err = IOHibernatePolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
1929 if (kIOReturnSuccess != err) {
1930 break;
1931 }
1932
1933 compressedSize += pageCompressedSize;
1934 uncompressedSize += page_size;
1935 pagesDone++;
1936
1937 if (vars->consoleMapping && (0 == (1023 & pagesDone))) {
1938 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
1939 if (blob != lastBlob) {
1940 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
1941 lastBlob = blob;
1942 }
1943 }
1944 if (0 == (8191 & pagesDone)) {
1945 clock_get_uptime(&endTime);
1946 SUB_ABSOLUTETIME(&endTime, &allTime);
1947 absolutetime_to_nanoseconds(endTime, &nsec);
1948 progressStamp = nsec / 750000000ULL;
1949 if (progressStamp != lastProgressStamp) {
1950 lastProgressStamp = progressStamp;
1951 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
1952 }
1953 }
1954 }
1955 if (kIOReturnSuccess != err) {
1956 break;
1957 }
1958 ppnum = page;
1959 }
39037602 1960
0a7de745
A		 1961 if (kIOReturnSuccess != err) {
1962 break;
1963 }
1964
1965 if ((kEncrypt & pageType) && vars->fileVars->encryptStart) {
1966 vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
1967 HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
d9a64523 1968 }
0a7de745
A		 1969
1970 if (kWiredEncrypt != pageType) {
1971 // end of image1/2 - fill to next block
cb323159 1972 err = IOHibernatePolledFileWrite(vars->fileVars, NULL, 0, cryptvars);
0a7de745
A		 1973 if (kIOReturnSuccess != err) {
1974 break;
1975 }
1976 }
1977 if (kWiredClear == pageType) {
1978 // enlarge wired image for test
3e170ce0 1979// err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
7ddcb079 1980
0a7de745
A		 1981 // end wired image
1982 header->encryptStart = vars->fileVars->encryptStart;
1983 header->encryptEnd = vars->fileVars->encryptEnd;
1984 image1Size = vars->fileVars->position;
1985 HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
1986 image1Size, header->encryptStart, header->encryptEnd);
1987 }
1988 }
1989 if (kIOReturnSuccess != err) {
1990 if (kIOReturnOverrun == err) {
1991 // update actual compression ratio on not enough space (for retry)
1992 gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
1993 }
1994
1995 // update partial amount written (for IOPolledFileClose cleanup/unmap)
1996 header->imageSize = vars->fileVars->position;
1997 break;
1998 }
1999
2000 // Header:
2001
2002 header->imageSize = vars->fileVars->position;
2003 header->image1Size = image1Size;
2004 header->bitmapSize = bitmap_size;
2005 header->pageCount = pageCount;
2006
2007 header->restore1Sum = restore1Sum;
2008 header->image1Sum = sum1;
2009 header->image2Sum = sum2;
2010 header->sleepTime = gIOLastSleepTime.tv_sec;
2011
2012 header->compression = (compressedSize << 8) / uncompressedSize;
2013 gIOHibernateCompression = header->compression;
2014
2015 count = vars->fileVars->fileExtents->getLength();
2016 if (count > sizeof(header->fileExtentMap)) {
2017 header->fileExtentMapSize = count;
2018 count = sizeof(header->fileExtentMap);
2019 } else {
2020 header->fileExtentMapSize = sizeof(header->fileExtentMap);
2021 }
2022 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
2023
2024 header->deviceBase = vars->fileVars->block0;
2025 header->deviceBlockSize = vars->fileVars->blockSize;
2026
2027 IOPolledFileSeek(vars->fileVars, 0);
2028 err = IOHibernatePolledFileWrite(vars->fileVars,
2029 (uint8_t *) header, sizeof(IOHibernateImageHeader),
2030 cryptvars);
2031 if (kIOReturnSuccess != err) {
2032 break;
2033 }
cb323159 2034 err = IOHibernatePolledFileWrite(vars->fileVars, NULL, 0, cryptvars);
0a7de745
A		 2035 }while (false);
2036
2037 clock_get_uptime(&endTime);
2038
2039 IOService::getPMRootDomain()->pmStatsRecordEvent(
2040 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
2041
2042 SUB_ABSOLUTETIME(&endTime, &allTime);
2043 absolutetime_to_nanoseconds(endTime, &nsec);
2044 HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
2045
2046 absolutetime_to_nanoseconds(compTime, &nsec);
2047 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2048 compBytes,
2049 nsec / 1000000ULL,
2050 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2051
2052 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2053 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
2054 vars->fileVars->cryptBytes,
2055 nsec / 1000000ULL,
2056 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2057
2058 HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
2059 header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
2060 uncompressedSize, atop_32(uncompressedSize), compressedSize,
2061 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
2062 sum1, sum2);
2063
2064 HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
2065 svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
2066
2067 if (pollerOpen) {
2068 IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted );
2069 }
2070
2071 if (vars->consoleMapping) {
2072 ProgressUpdate(gIOHibernateGraphicsInfo,
2073 vars->consoleMapping, 0, kIOHibernateProgressCount);
2074 }
2075
2076 HIBLOG("hibernate_write_image done(%x)\n", err);
2077
2078 // should we come back via regular wake, set the state in memory.
2079 gIOHibernateState = kIOHibernateStateInactive;
3a60a9f5 2080
5ba3f43e 2081 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, wiredPagesEncrypted,
0a7de745
A		 2082 wiredPagesClear, dirtyPagesEncrypted);
2083
2084 if (kIOReturnSuccess == err) {
2085 if (kIOHibernateModeSleep & gIOHibernateMode) {
2086 return kIOHibernatePostWriteSleep;
2087 } else if (kIOHibernateModeRestart & gIOHibernateMode) {
2088 return kIOHibernatePostWriteRestart;
2089 } else {
2090 /* by default, power down */
2091 return kIOHibernatePostWriteHalt;
2092 }
2093 } else if (kIOReturnAborted == err) {
2094 return kIOHibernatePostWriteWake;
2095 } else {
2096 /* on error, sleep */
2097 return kIOHibernatePostWriteSleep;
2098 }
3a60a9f5
A		 2099}
2100
2101/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2102
a39ff7e2 2103extern "C" void
3a60a9f5
A		 2104hibernate_machine_init(void)
2105{
0a7de745
A		 2106 IOReturn err;
2107 uint32_t sum;
2108 uint32_t pagesDone;
2109 uint32_t pagesRead = 0;
2110 AbsoluteTime startTime, compTime;
2111 AbsoluteTime allTime, endTime;
2112 AbsoluteTime startIOTime, endIOTime;
2113 uint64_t nsec, nsecIO;
2114 uint64_t compBytes;
2115 uint32_t lastProgressStamp = 0;
2116 uint32_t progressStamp;
cb323159 2117 IOPolledFileCryptVars * cryptvars = NULL;
0a7de745
A		 2118
2119 IOHibernateVars * vars = &gIOHibernateVars;
2120 bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
2121
2122 if (!vars->fileVars || !vars->fileVars->pollers) {
2123 return;
2124 }
2125
2126 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2127 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2128
2129 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) {
2130 HIBLOG("regular wake\n");
2131 return;
2132 }
2133
2134 HIBPRINT("diag %x %x %x %x\n",
a39ff7e2 2135 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
6d2010ae 2136 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
3a60a9f5 2137
0a7de745
A		 2138#define t40ms(x) (tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)
2139#define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
2140 tStat(booterStart, booterStart);
2141 gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart;
2142 tStat(booterDuration0, booterTime0);
2143 tStat(booterDuration1, booterTime1);
2144 tStat(booterDuration2, booterTime2);
2145 tStat(booterDuration, booterTime);
2146 tStat(booterConnectDisplayDuration, connectDisplayTime);
2147 tStat(booterSplashDuration, splashTime);
2148 tStat(trampolineDuration, trampolineTime);
2149
2150 gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size;
2151 gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize;
2152 gIOHibernateStats->image1Pages = pagesDone;
39236c6e 2153
5ba3f43e
A		 2154 /* HIBERNATE_stats */
2155 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 14), gIOHibernateStats->smcStart,
0a7de745
A		 2156 gIOHibernateStats->booterStart, gIOHibernateStats->booterDuration,
2157 gIOHibernateStats->trampolineDuration);
2158
2159 HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
2160 gIOHibernateStats->booterStart,
2161 gIOHibernateStats->smcStart,
2162 gIOHibernateStats->booterDuration0,
2163 gIOHibernateStats->booterDuration1,
2164 gIOHibernateStats->booterDuration2,
2165 gIOHibernateStats->booterDuration,
2166 gIOHibernateStats->booterConnectDisplayDuration,
2167 gIOHibernateStats->booterSplashDuration,
2168 gIOHibernateStats->trampolineDuration);
2169
2170 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
39236c6e
A		 2171 gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
2172 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
bd504ef0 2173
0a7de745
A		 2174 if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
2175 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))) {
2176 hibernate_page_list_discard(vars->page_list);
2177 }
0c530ab8 2178
cb323159 2179 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : NULL;
6d2010ae 2180
0a7de745
A		 2181 if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount) {
2182 panic("handoff overflow");
2183 }
6d2010ae 2184
0a7de745
A		 2185 IOHibernateHandoff * handoff;
2186 bool done = false;
2187 bool foundCryptData = false;
2188 bool foundVolumeEncryptData = false;
6d2010ae 2189
0a7de745
A		 2190 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
2191 !done;
2192 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount]) {
6d2010ae 2193// HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
0a7de745
A		 2194 uint8_t * data = &handoff->data[0];
2195 switch (handoff->type) {
2196 case kIOHibernateHandoffTypeEnd:
2197 done = true;
2198 break;
6d2010ae 2199
0a7de745
A		 2200 case kIOHibernateHandoffTypeGraphicsInfo:
2201 if (handoff->bytecount == sizeof(*gIOHibernateGraphicsInfo)) {
2202 bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
2203 }
2204 break;
6d2010ae 2205
0a7de745
A		 2206 case kIOHibernateHandoffTypeCryptVars:
2207 if (cryptvars) {
2208 hibernate_cryptwakevars_t *
2209 wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
2210 bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
2211 }
2212 foundCryptData = true;
2213 bzero(data, handoff->bytecount);
2214 break;
6d2010ae 2215
0a7de745
A		 2216 case kIOHibernateHandoffTypeVolumeCryptKey:
2217 if (handoff->bytecount == vars->volumeCryptKeySize) {
2218 bcopy(data, &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
2219 foundVolumeEncryptData = true;
2220 } else {
2221 panic("kIOHibernateHandoffTypeVolumeCryptKey(%d)", handoff->bytecount);
2222 }
2223 break;
a39ff7e2 2224
0a7de745 2225 case kIOHibernateHandoffTypeMemoryMap:
bd504ef0 2226
0a7de745 2227 clock_get_uptime(&allTime);
bd504ef0 2228
0a7de745
A		 2229 hibernate_newruntime_map(data, handoff->bytecount,
2230 gIOHibernateCurrentHeader->systemTableOffset);
bd504ef0 2231
0a7de745 2232 clock_get_uptime(&endTime);
a39ff7e2 2233
0a7de745
A		 2234 SUB_ABSOLUTETIME(&endTime, &allTime);
2235 absolutetime_to_nanoseconds(endTime, &nsec);
a39ff7e2 2236
0a7de745 2237 HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
bd504ef0 2238
0a7de745 2239 break;
6d2010ae 2240
0a7de745 2241 case kIOHibernateHandoffTypeDeviceTree:
6d2010ae
A		 2242 {
2243// DTEntry chosen = NULL;
2244// HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen));
2245 }
0a7de745 2246 break;
6d2010ae 2247
0a7de745
A		 2248 default:
2249 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
2250 break;
2251 }
a39ff7e2 2252 }
a39ff7e2 2253
0a7de745
A		 2254 if (vars->hwEncrypt && !foundVolumeEncryptData) {
2255 panic("no volumeCryptKey");
2256 } else if (cryptvars && !foundCryptData) {
2257 panic("hibernate handoff");
2258 }
6d2010ae 2259
0a7de745 2260 HIBPRINT("video 0x%llx %d %d %d status %x\n",
a39ff7e2
A		 2261 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2262 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
316670eb 2263
0a7de745
A		 2264 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress) {
2265 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2266 * gIOHibernateGraphicsInfo->rowBytes);
2267 if (vars->videoMapSize > vars->videoAllocSize) {
2268 vars->videoMapSize = 0;
2269 } else {
2270 IOMapPages(kernel_map,
2271 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2272 vars->videoMapSize, kIOMapInhibitCache );
2273 }
2274 }
3a60a9f5 2275
0a7de745
A		 2276 if (vars->videoMapSize) {
2277 ProgressUpdate(gIOHibernateGraphicsInfo,
2278 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2279 }
3a60a9f5 2280
0a7de745
A		 2281 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2282 uint8_t * compressed = src + page_size;
2283 uint8_t * scratch = compressed + page_size;
2284 uint32_t decoOffset;
2285
2286 clock_get_uptime(&allTime);
2287 AbsoluteTime_to_scalar(&compTime) = 0;
2288 compBytes = 0;
2289
2290 HIBLOG("IOPolledFilePollersOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2291 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledAfterSleepState, false);
2292 clock_get_uptime(&startIOTime);
2293 endTime = startIOTime;
2294 SUB_ABSOLUTETIME(&endTime, &allTime);
2295 absolutetime_to_nanoseconds(endTime, &nsec);
2296 HIBLOG("IOPolledFilePollersOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
2297
2298 if (vars->hwEncrypt) {
2299 err = IOPolledFilePollersSetEncryptionKey(vars->fileVars,
2300 &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
2301 HIBLOG("IOPolledFilePollersSetEncryptionKey(%x) %ld\n", err, vars->volumeCryptKeySize);
2302 if (kIOReturnSuccess != err) {
2303 panic("IOPolledFilePollersSetEncryptionKey(0x%x)", err);
2304 }
cb323159 2305 cryptvars = NULL;
0a7de745 2306 }
3a60a9f5 2307
0a7de745 2308 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
3a60a9f5 2309
0a7de745
A		 2310 // kick off the read ahead
2311 vars->fileVars->bufferHalf = 0;
2312 vars->fileVars->bufferLimit = 0;
2313 vars->fileVars->lastRead = 0;
2314 vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
2315 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2316 vars->fileVars->cryptBytes = 0;
2317 AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
36401178 2318
cb323159 2319 err = IOPolledFileRead(vars->fileVars, NULL, 0, cryptvars);
0a7de745
A		 2320 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2321 // --
3e170ce0 2322
0a7de745 2323 HIBLOG("hibernate_machine_init reading\n");
3e170ce0 2324
0a7de745
A		 2325 uint32_t * header = (uint32_t *) src;
2326 sum = 0;
a39ff7e2 2327
0a7de745
A		 2328 while (kIOReturnSuccess == err) {
2329 unsigned int count;
2330 unsigned int page;
2331 uint32_t tag;
2332 vm_offset_t ppnum, compressedSize;
3e170ce0 2333
0a7de745
A		 2334 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2335 if (kIOReturnSuccess != err) {
2336 break;
39236c6e 2337 }
0a7de745
A		 2338
2339 ppnum = header[0];
2340 count = header[1];
2341
2342// HIBPRINT("(%x, %x)\n", ppnum, count);
2343
2344 if (!count) {
2345 break;
2346 }
2347
2348 for (page = 0; page < count; page++) {
2349 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2350 if (kIOReturnSuccess != err) {
2351 break;
2352 }
2353
2354 compressedSize = kIOHibernateTagLength & tag;
2355 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength)) {
2356 err = kIOReturnIPCError;
2357 break;
2358 }
2359
2360 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2361 if (kIOReturnSuccess != err) {
2362 break;
2363 }
2364
2365 if (compressedSize < page_size) {
2366 decoOffset = page_size;
2367 clock_get_uptime(&startTime);
2368
2369 if (compressedSize == 4) {
2370 int i;
2371 uint32_t *s, *d;
2372
2373 s = (uint32_t *)src;
2374 d = (uint32_t *)(uintptr_t)compressed;
2375
2376 for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++) {
2377 *d++ = *s;
2378 }
2379 } else {
2380 WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, compressedSize);
2381 }
2382 clock_get_uptime(&endTime);
2383 ADD_ABSOLUTETIME(&compTime, &endTime);
2384 SUB_ABSOLUTETIME(&compTime, &startTime);
2385 compBytes += page_size;
2386 } else {
2387 decoOffset = 0;
2388 }
2389
2390 sum += hibernate_sum_page((src + decoOffset), ppnum);
2391 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2392 if (err) {
2393 HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
2394 break;
2395 }
2396
2397 ppnum++;
2398 pagesDone++;
2399 pagesRead++;
2400
2401 if (0 == (8191 & pagesDone)) {
2402 clock_get_uptime(&endTime);
2403 SUB_ABSOLUTETIME(&endTime, &allTime);
2404 absolutetime_to_nanoseconds(endTime, &nsec);
2405 progressStamp = nsec / 750000000ULL;
2406 if (progressStamp != lastProgressStamp) {
2407 lastProgressStamp = progressStamp;
2408 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2409 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2410 }
2411 }
3a60a9f5 2412 }
3a60a9f5 2413 }
0a7de745
A		 2414 if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages)) {
2415 err = kIOReturnLockedRead;
2416 }
3a60a9f5 2417
0a7de745
A		 2418 if (kIOReturnSuccess != err) {
2419 panic("Hibernate restore error %x", err);
2420 }
36401178 2421
0a7de745
A		 2422 gIOHibernateCurrentHeader->actualImage2Sum = sum;
2423 gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
3a60a9f5 2424
0a7de745 2425 clock_get_uptime(&endIOTime);
39236c6e 2426
0a7de745 2427 err = IOPolledFilePollersClose(vars->fileVars, kIOPolledAfterSleepState);
3a60a9f5 2428
0a7de745 2429 clock_get_uptime(&endTime);
b0d623f7 2430
0a7de745
A		 2431 IOService::getPMRootDomain()->pmStatsRecordEvent(
2432 kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
2433 IOService::getPMRootDomain()->pmStatsRecordEvent(
2434 kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
b0d623f7 2435
0a7de745
A		 2436 SUB_ABSOLUTETIME(&endTime, &allTime);
2437 absolutetime_to_nanoseconds(endTime, &nsec);
3a60a9f5 2438
0a7de745
A		 2439 SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
2440 absolutetime_to_nanoseconds(endIOTime, &nsecIO);
39236c6e 2441
0a7de745
A		 2442 gIOHibernateStats->kernelImageReadDuration = nsec / 1000000ULL;
2443 gIOHibernateStats->imagePages = pagesDone;
39236c6e 2444
0a7de745
A		 2445 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
2446 pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
2447 nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
6d2010ae 2448
0a7de745
A		 2449 absolutetime_to_nanoseconds(compTime, &nsec);
2450 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2451 compBytes,
2452 nsec / 1000000ULL,
2453 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
6d2010ae 2454
0a7de745
A		 2455 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2456 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
2457 vars->fileVars->cryptBytes,
2458 nsec / 1000000ULL,
2459 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
6d2010ae 2460
0a7de745 2461 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 2), pagesRead, pagesDone);
3a60a9f5
A		 2462}
2463
2464/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
99c3a104 2465
0a7de745
A		 2466void
2467IOHibernateSetWakeCapabilities(uint32_t capability)
39236c6e 2468{
0a7de745
A		 2469 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
2470 gIOHibernateStats->wakeCapability = capability;
8a3053a0 2471
0a7de745
A		 2472 if (kIOPMSystemCapabilityGraphics & capability) {
2473 vm_compressor_do_warmup();
2474 }
8a3053a0 2475 }
39236c6e
A		 2476}
2477
2478/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2479
0a7de745
A		 2480void
2481IOHibernateSystemRestart(void)
99c3a104 2482{
0a7de745
A		 2483 static uint8_t noteStore[32] __attribute__((aligned(32)));
2484 IORegistryEntry * regEntry;
2485 const OSSymbol * sym;
2486 OSData * noteProp;
2487 OSData * data;
2488 uintptr_t * smcVars;
2489 uint8_t * smcBytes;
2490 size_t len;
2491 addr64_t element;
2492
2493 data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
2494 if (!data) {
2495 return;
2496 }
2497
2498 smcVars = (typeof(smcVars))data->getBytesNoCopy();
2499 smcBytes = (typeof(smcBytes))smcVars[1];
2500 len = smcVars[0];
2501 if (len > sizeof(noteStore)) {
2502 len = sizeof(noteStore);
2503 }
2504 noteProp = OSData::withCapacity(3 * sizeof(element));
2505 if (!noteProp) {
2506 return;
2507 }
2508 element = len;
2509 noteProp->appendBytes(&element, sizeof(element));
2510 element = crc32(0, smcBytes, len);
2511 noteProp->appendBytes(&element, sizeof(element));
2512
2513 bcopy(smcBytes, noteStore, len);
2514 element = (addr64_t) &noteStore[0];
2515 element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
2516 noteProp->appendBytes(&element, sizeof(element));
2517
2518 if (!gIOOptionsEntry) {
2519 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
2520 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
2521 if (regEntry && !gIOOptionsEntry) {
2522 regEntry->release();
2523 }
2524 }
2525
2526 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
2527 if (gIOOptionsEntry && sym) {
2528 gIOOptionsEntry->setProperty(sym, noteProp);
2529 }
2530 if (noteProp) {
2531 noteProp->release();
2532 }
2533 if (sym) {
2534 sym->release();
2535 }
99c3a104 2536}
mirror of XNU