]> git.saurik.com Git - apple/xnu.git/blame - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-4570.31.3.tar.gz
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
CommitLineData
3a60a9f5 1/*
39037602 2 * Copyright (c) 2004-2016 Apple Inc. All rights reserved.
3a60a9f5 3 *
2d21ac55 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
39037602 5 *
2d21ac55
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
39037602 14 *
2d21ac55
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
39037602 17 *
2d21ac55
A		 18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5
A		 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55
A		 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
39037602 25 *
2d21ac55 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
3a60a9f5
A		 27 */
28
3a60a9f5
A		 29/*
30
31Sleep:
32
33- PMRootDomain calls IOHibernateSystemSleep() before system sleep
34(devices awake, normal execution context)
35- IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
36 grabs its extents and searches for a polling driver willing to work with that IOMedia.
37 The BSD code makes an ioctl to the storage driver to get the partition base offset to
38 the disk, and other ioctls to get the transfer constraints
39 If successful, the file is written to make sure its initially not bootable (in case of
40 later failure) and nvram set to point to the first block of the file. (Has to be done
41 here so blocking is possible in nvram support).
42 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
43 page out any pages it wants to (currently zero, but probably some percentage of memory).
44 Its assumed just allocating pages will cause the VM system to naturally select the best
45 pages for eviction. It also copies processor flags needed for the restore path and sets
46 a flag in the boot processor proc info.
47 gIOHibernateState = kIOHibernateStateHibernating.
48- Regular sleep progresses - some drivers may inspect the root domain property
49 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
50 as usual but leaves motherboard I/O on.
51- Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
52 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
53 all ppc RC bits out of the hash table and caches into the mapping structures.
54- hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
55 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
56 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
57 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
58 The image header and block list are written. The header includes the second file extent so
59 only the header block is needed to read the file, regardless of filesystem.
316670eb 60 The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
3a60a9f5
A		 61 (only) is used to decompress the image during wake/boot.
62 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
63 The bitmaps are written to the image.
64 More areas are removed from the bitmaps (after they have been written to the image) - the
316670eb 65 segment "__HIB" pages and interrupt stack.
3a60a9f5
A		 66 Each wired page is compressed and written and then each non-wired page. Compression and
67 disk writes are in parallel.
68 The image header is written to the start of the file and the polling driver closed.
69 The machine powers down (or sleeps).
70
71Boot/Wake:
72
73- BootX sees the boot-image nvram variable containing the device and block number of the image,
74 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
75- BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
76 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
77 that is in the kernel's __HIB section.
78- BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
79 hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
80 only code & data in that section is safe to call since all the other wired pages are still
81 compressed in the image.
82- hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
83 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
84 location directly, and copies those that can't to interim free pages. When the image has been
85 completed, the copies are uncompressed, overwriting the wired image pages.
86 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
87 is used to get pages into place for 64bit.
88- the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
89 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
90 are removed from the software strutures, and the hash table is reinitialized.
91- After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
92 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
93 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
94 for the remaining non wired pages.
95- The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
96 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
97 is closed via bsd.
98
99Polled Mode I/O:
100
101IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
102registry, specifying an object of calls IOPolledInterface.
103
104Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
105partition) that the image is going to live, looking for polled interface properties. If it finds
106one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
107interfaces found are kept in an ordered list.
108
109There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
110few different contexts things happen in:
111
112- there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
113up and running) and after wake - this is safe to allocate memory and do anything. The device
114ignores sleep requests from that point since its a waste of time if it goes to sleep and
115immediately wakes back up for the image write.
116
117- there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
118immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
119from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
120used to flush and set the disk to sleep.
121
122- there is an Open/Close (AfterSleep) pair made around the image read operations that happen
123immediately after sleep. These can't block or allocate memory. This is happening after the platform
124expert has woken the low level bits of the system, but most of the I/O system has not. There is only
125one thread running.
126
127For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
128(prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
129the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
130Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
131that is called for the hardware to check for events, and complete the I/O via the callback.
132The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
133to restrict I/O ops.
134*/
135
136#include <sys/systm.h>
137
138#include <IOKit/IOWorkLoop.h>
139#include <IOKit/IOCommandGate.h>
140#include <IOKit/IOTimerEventSource.h>
141#include <IOKit/IOPlatformExpert.h>
142#include <IOKit/IOKitDebug.h>
143#include <IOKit/IOTimeStamp.h>
144#include <IOKit/pwr_mgt/RootDomain.h>
145#include <IOKit/pwr_mgt/IOPMPrivate.h>
146#include <IOKit/IOMessage.h>
147#include <IOKit/IODeviceTreeSupport.h>
148#include <IOKit/IOBSD.h>
149#include "RootDomainUserClient.h"
150#include <IOKit/pwr_mgt/IOPowerConnection.h>
151#include "IOPMPowerStateQueue.h"
152#include <IOKit/IOBufferMemoryDescriptor.h>
6d2010ae 153#include <IOKit/AppleKeyStoreInterface.h>
316670eb 154#include <libkern/crypto/aes.h>
3a60a9f5
A		 155
156#include <sys/uio.h>
157#include <sys/conf.h>
158#include <sys/stat.h>
159#include <sys/fcntl.h> // (FWRITE, ...)
3a60a9f5 160#include <sys/sysctl.h>
6d2010ae 161#include <sys/kdebug.h>
fe8ab488 162#include <stdint.h>
3a60a9f5
A		 163
164#include <IOKit/IOHibernatePrivate.h>
165#include <IOKit/IOPolledInterface.h>
166#include <IOKit/IONVRAM.h>
167#include "IOHibernateInternal.h"
39236c6e 168#include <vm/WKdm_new.h>
39037602 169#include <vm/vm_protos.h>
3a60a9f5 170#include "IOKitKernelInternal.h"
6d2010ae
A		 171#include <pexpert/device_tree.h>
172
173#include <machine/pal_routines.h>
174#include <machine/pal_hibernate.h>
39236c6e 175#include <i386/tsc.h>
39037602 176#include <i386/cpuid.h>
5ba3f43e 177#include <san/kasan.h>
6d2010ae
A		 178
179extern "C" addr64_t kvtophys(vm_offset_t va);
99c3a104 180extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
3a60a9f5
A		 181
182/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
183
39236c6e 184#define DISABLE_TRIM 0
39037602 185#define TRIM_DELAY 25000
39236c6e 186
0b4c1975 187extern unsigned int save_kdebug_enable;
3a60a9f5
A		 188extern uint32_t gIOHibernateState;
189uint32_t gIOHibernateMode;
190static char gIOHibernateBootSignature[256+1];
191static char gIOHibernateFilename[MAXPATHLEN+1];
db609669
A		 192static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
193uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
194static uint64_t gIOHibernateCompression = 0x80; // default compression 50%
39037602 195boolean_t gIOHibernateStandbyDisabled;
3a60a9f5
A		 196
197static IODTNVRAM * gIOOptionsEntry;
198static IORegistryEntry * gIOChosenEntry;
3e170ce0
A		 199
200static const OSSymbol * gIOHibernateBootImageKey;
201
b0d623f7 202#if defined(__i386__) || defined(__x86_64__)
3e170ce0 203
060df5ea
A		 204static const OSSymbol * gIOHibernateRTCVariablesKey;
205static const OSSymbol * gIOHibernateBoot0082Key;
206static const OSSymbol * gIOHibernateBootNextKey;
207static OSData * gIOHibernateBoot0082Data;
208static OSData * gIOHibernateBootNextData;
209static OSObject * gIOHibernateBootNextSave;
3e170ce0
A		 210
211static IOPolledFileIOVars * gDebugImageFileVars;
212static IOLock * gDebugImageLock;
213
214#endif /* defined(__i386__) || defined(__x86_64__) */
3a60a9f5 215
316670eb 216static IOLock * gFSLock;
5ba3f43e 217 uint32_t gFSState;
813fb2f6 218static thread_call_t gIOHibernateTrimCalloutEntry;
3a60a9f5
A		 219static IOPolledFileIOVars gFileVars;
220static IOHibernateVars gIOHibernateVars;
3e170ce0 221static IOPolledFileCryptVars gIOHibernateCryptWakeContext;
6d2010ae
A		 222static hibernate_graphics_t _hibernateGraphics;
223static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
39236c6e
A		 224static hibernate_statistics_t _hibernateStats;
225static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats;
3a60a9f5 226
316670eb
A		 227enum
228{
813fb2f6
A		 229 kFSIdle = 0,
230 kFSOpening = 2,
231 kFSOpened = 3,
232 kFSTimedOut = 4,
233 kFSTrimDelay = 5
316670eb
A		 234};
235
236static IOReturn IOHibernateDone(IOHibernateVars * vars);
3e170ce0
A		 237static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature);
238static void IOSetBootImageNVRAM(OSData * data);
5ba3f43e 239static void IOHibernateSystemPostWakeTrim(void * p1, void * p2);
316670eb 240
3a60a9f5
A		 241/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
242
3a60a9f5 243enum { kDefaultIOSize = 128 * 1024 };
22ba694c 244enum { kVideoMapSize = 80 * 1024 * 1024 };
3a60a9f5 245
3a60a9f5
A		 246/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
247
248// copy from phys addr to MD
249
250static IOReturn
251IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
252 IOByteCount offset, addr64_t bytes, IOByteCount length)
253{
254 addr64_t srcAddr = bytes;
255 IOByteCount remaining;
256
257 remaining = length = min(length, md->getLength() - offset);
258 while (remaining) { // (process another target segment?)
259 addr64_t dstAddr64;
260 IOByteCount dstLen;
261
b0d623f7 262 dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
3a60a9f5
A		 263 if (!dstAddr64)
264 break;
265
266 // Clip segment length to remaining
267 if (dstLen > remaining)
268 dstLen = remaining;
269
270#if 1
271 bcopy_phys(srcAddr, dstAddr64, dstLen);
272#else
273 copypv(srcAddr, dstAddr64, dstLen,
274 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
275#endif
276 srcAddr += dstLen;
277 offset += dstLen;
278 remaining -= dstLen;
279 }
280
281 assert(!remaining);
282
283 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
284}
285
286// copy from MD to phys addr
287
288static IOReturn
289IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
290 IOByteCount offset, addr64_t bytes, IOByteCount length)
291{
292 addr64_t dstAddr = bytes;
293 IOByteCount remaining;
294
295 remaining = length = min(length, md->getLength() - offset);
296 while (remaining) { // (process another target segment?)
297 addr64_t srcAddr64;
298 IOByteCount dstLen;
299
b0d623f7 300 srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
3a60a9f5
A		 301 if (!srcAddr64)
302 break;
303
304 // Clip segment length to remaining
305 if (dstLen > remaining)
306 dstLen = remaining;
307
308#if 1
309 bcopy_phys(srcAddr64, dstAddr, dstLen);
310#else
311 copypv(srcAddr, dstAddr64, dstLen,
312 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
313#endif
314 dstAddr += dstLen;
315 offset += dstLen;
316 remaining -= dstLen;
317 }
318
319 assert(!remaining);
320
321 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
322}
323
324/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
325
326void
327hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
328 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
329{
330 count += ppnum;
331 switch (kind)
332 {
333 case kIOHibernatePageStateUnwiredSave:
334 // unwired save
335 for (; ppnum < count; ppnum++)
336 {
337 hibernate_page_bitset(page_list, FALSE, ppnum);
338 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
339 }
340 break;
341 case kIOHibernatePageStateWiredSave:
342 // wired save
343 for (; ppnum < count; ppnum++)
344 {
345 hibernate_page_bitset(page_list, FALSE, ppnum);
346 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
347 }
348 break;
349 case kIOHibernatePageStateFree:
350 // free page
351 for (; ppnum < count; ppnum++)
352 {
353 hibernate_page_bitset(page_list, TRUE, ppnum);
354 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
355 }
356 break;
357 default:
358 panic("hibernate_set_page_state");
359 }
360}
361
3e170ce0
A		 362static vm_offset_t
363hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
364{
365 uint32_t page = *pPage;
366 uint32_t count;
367 hibernate_bitmap_t * bitmap;
fe8ab488 368
3e170ce0
A		 369 while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
370 {
371 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
372 if (!count)
373 break;
374 page += count;
375 if (page <= bitmap->last_page)
376 break;
fe8ab488 377 }
fe8ab488 378
3e170ce0
A		 379 *pPage = page;
380 if (bitmap)
381 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
382 else
383 count = 0;
384
385 return (count);
fe8ab488 386}
fe8ab488
A		 387
388/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
389
3a60a9f5
A		 390IOReturn
391IOHibernateSystemSleep(void)
392{
393 IOReturn err;
3e170ce0 394 OSData * nvramData;
3a60a9f5
A		 395 OSObject * obj;
396 OSString * str;
db609669 397 OSNumber * num;
39037602 398 bool dsSSD, vmflush, swapPinned;
316670eb 399 IOHibernateVars * vars;
3e170ce0 400 uint64_t setFileSize = 0;
3a60a9f5
A		 401
402 gIOHibernateState = kIOHibernateStateInactive;
403
b0d623f7
A		 404 gIOHibernateDebugFlags = 0;
405 if (kIOLogHibernate & gIOKitDebug)
406 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
407
0b4c1975
A		 408 if (IOService::getPMRootDomain()->getHibernateSettings(
409 &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime))
6d2010ae 410 {
3a60a9f5
A		 411 if (kIOHibernateModeSleep & gIOHibernateMode)
412 // default to discard clean for safe sleep
413 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
414 | kIOHibernateModeDiscardCleanActive);
6d2010ae 415 }
3a60a9f5 416
3a60a9f5
A		 417 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
418 {
419 if ((str = OSDynamicCast(OSString, obj)))
2d21ac55
A		 420 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
421 sizeof(gIOHibernateFilename));
3a60a9f5
A		 422 obj->release();
423 }
424
425 if (!gIOHibernateMode || !gIOHibernateFilename[0])
426 return (kIOReturnUnsupported);
427
428 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
429
316670eb
A		 430 vars = IONew(IOHibernateVars, 1);
431 if (!vars) return (kIOReturnNoMemory);
432 bzero(vars, sizeof(*vars));
433
434 IOLockLock(gFSLock);
813fb2f6
A		 435 if (!gIOHibernateTrimCalloutEntry)
436 {
437 gIOHibernateTrimCalloutEntry = thread_call_allocate(&IOHibernateSystemPostWakeTrim, &gFSLock);
438 }
439 IOHibernateSystemPostWakeTrim(NULL, NULL);
5c9f4661 440 thread_call_cancel(gIOHibernateTrimCalloutEntry);
316670eb
A		 441 if (kFSIdle != gFSState)
442 {
443 HIBLOG("hibernate file busy\n");
444 IOLockUnlock(gFSLock);
445 IODelete(vars, IOHibernateVars, 1);
446 return (kIOReturnBusy);
447 }
448 gFSState = kFSOpening;
449 IOLockUnlock(gFSLock);
2d21ac55 450
39037602 451 swapPinned = false;
3a60a9f5
A		 452 do
453 {
0c530ab8 454 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
39037602 455 2 * page_size + WKdm_SCRATCH_BUF_SIZE_INTERNAL, page_size);
3a60a9f5 456
6d2010ae
A		 457 vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
458 ptoa_64(gIOHibernateHandoffPageCount), page_size);
459
3e170ce0 460 if (!vars->srcBuffer || !vars->handoffBuffer)
3a60a9f5
A		 461 {
462 err = kIOReturnNoMemory;
463 break;
464 }
465
db609669
A		 466 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey)))
467 {
468 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMinSize = num->unsigned64BitValue();
469 obj->release();
470 }
471 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey)))
472 {
473 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMaxSize = num->unsigned64BitValue();
474 obj->release();
475 }
476
477 boolean_t encryptedswap = true;
478 uint32_t pageCount;
479 AbsoluteTime startTime, endTime;
480 uint64_t nsec;
481
482 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
483 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
6d2010ae 484 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
db609669 485
39037602 486 vmflush = ((kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
39236c6e
A		 487 err = hibernate_alloc_page_lists(&vars->page_list,
488 &vars->page_list_wired,
489 &vars->page_list_pal);
39037602
A		 490 if (KERN_SUCCESS != err) break;
491
492 err = hibernate_pin_swap(TRUE);
493 if (KERN_SUCCESS != err) break;
494 swapPinned = true;
db609669
A		 495
496 if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode))
497 {
498 hibernate_page_list_setall(vars->page_list,
499 vars->page_list_wired,
500 vars->page_list_pal,
501 true /* preflight */,
39236c6e 502 vmflush /* discard */,
db609669
A		 503 &pageCount);
504 PE_Video consoleInfo;
505 bzero(&consoleInfo, sizeof(consoleInfo));
506 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
507
8a3053a0 508 // estimate: 6% increase in pages compressed
fe8ab488 509 // screen preview 2 images compressed 0%
8a3053a0 510 setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
db609669 511 + vars->page_list->list_size
8a3053a0
A		 512 + (consoleInfo.v_width * consoleInfo.v_height * 8);
513 enum { setFileRound = 1024*1024ULL };
514 setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
fe8ab488 515
db609669
A		 516 HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
517 pageCount, (100ULL * gIOHibernateCompression) >> 8,
518 setFileSize, vars->fileMinSize);
519
520 if (!(kIOHibernateModeFileResize & gIOHibernateMode)
521 && (setFileSize < vars->fileMinSize))
522 {
523 setFileSize = vars->fileMinSize;
524 }
525 }
526
3e170ce0
A		 527 // Invalidate the image file
528 if (gDebugImageLock) {
529 IOLockLock(gDebugImageLock);
530 if (gDebugImageFileVars != 0) {
3e170ce0
A		 531 IOSetBootImageNVRAM(0);
532 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
533 }
534 IOLockUnlock(gDebugImageLock);
535 }
db609669 536
3e170ce0
A		 537 err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, 0,
538 gIOHibernateCurrentHeader, sizeof(gIOHibernateCurrentHeader),
539 &vars->fileVars, &nvramData,
540 &vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
fe8ab488 541
3e170ce0
A		 542 if (KERN_SUCCESS != err)
543 {
544 IOLockLock(gFSLock);
545 if (kFSOpening != gFSState) err = kIOReturnTimeout;
546 IOLockUnlock(gFSLock);
547 }
8a3053a0 548
3a60a9f5
A		 549 if (KERN_SUCCESS != err)
550 {
551 HIBLOG("IOPolledFileOpen(%x)\n", err);
552 break;
553 }
3a60a9f5 554
3e170ce0
A		 555 // write extents for debug data usage in EFI
556 IOWriteExtentsToFile(vars->fileVars, kIOHibernateHeaderOpenSignature);
557
558 err = IOPolledFilePollersSetup(vars->fileVars, kIOPolledPreflightState);
559 if (KERN_SUCCESS != err) break;
560
39236c6e
A		 561 clock_get_uptime(&startTime);
562 err = hibernate_setup(gIOHibernateCurrentHeader,
39236c6e
A		 563 vmflush,
564 vars->page_list, vars->page_list_wired, vars->page_list_pal);
565 clock_get_uptime(&endTime);
566 SUB_ABSOLUTETIME(&endTime, &startTime);
567 absolutetime_to_nanoseconds(endTime, &nsec);
39037602
A		 568
569 boolean_t haveSwapPin, hibFileSSD;
570 haveSwapPin = vm_swap_files_pinned();
571
572 hibFileSSD = (kIOPolledFileSSD & vars->fileVars->flags);
573
574 HIBLOG("hibernate_setup(%d) took %qd ms, swapPin(%d) ssd(%d)\n",
575 err, nsec / 1000000ULL,
576 haveSwapPin, hibFileSSD);
3e170ce0 577 if (KERN_SUCCESS != err) break;
39236c6e 578
39037602
A		 579 gIOHibernateStandbyDisabled = ((!haveSwapPin || !hibFileSSD));
580
3e170ce0 581 dsSSD = ((0 != (kIOPolledFileSSD & vars->fileVars->flags))
0b4c1975 582 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
3e170ce0
A		 583
584 if (dsSSD) gIOHibernateCurrentHeader->options |= kIOHibernateOptionSSD | kIOHibernateOptionColor;
585 else gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
586
6d2010ae
A		 587
588#if defined(__i386__) || defined(__x86_64__)
3e170ce0
A		 589 if (!uuid_is_null(vars->volumeCryptKey) &&
590 (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey)))
591 {
592 uintptr_t smcVars[2];
593 smcVars[0] = sizeof(vars->volumeCryptKey);
594 smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
6d2010ae 595
3e170ce0
A		 596 IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
597 bzero(smcVars, sizeof(smcVars));
598 }
6d2010ae 599#endif
0b4c1975 600
0b4c1975 601
6d2010ae 602 if (encryptedswap || !uuid_is_null(vars->volumeCryptKey))
3a60a9f5
A		 603 gIOHibernateMode ^= kIOHibernateModeEncrypt;
604
0b4c1975
A		 605 if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
606 {
607 vars->videoAllocSize = kVideoMapSize;
3e170ce0 608 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize, VM_KERN_MEMORY_IOKIT))
0b4c1975
A		 609 vars->videoMapping = 0;
610 }
3a60a9f5
A		 611
612 // generate crypt keys
613 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
614 vars->wiredCryptKey[i] = random();
615 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
616 vars->cryptKey[i] = random();
617
618 // set nvram
619
3e170ce0
A		 620 IOSetBootImageNVRAM(nvramData);
621 nvramData->release();
3a60a9f5 622
b0d623f7 623#if defined(__i386__) || defined(__x86_64__)
3e170ce0 624 {
0c530ab8
A		 625 struct AppleRTCHibernateVars
626 {
627 uint8_t signature[4];
628 uint32_t revision;
629 uint8_t booterSignature[20];
630 uint8_t wiredCryptKey[16];
631 };
632 AppleRTCHibernateVars rtcVars;
3e170ce0 633 OSData * data;
0c530ab8
A		 634
635 rtcVars.signature[0] = 'A';
636 rtcVars.signature[1] = 'A';
637 rtcVars.signature[2] = 'P';
638 rtcVars.signature[3] = 'L';
639 rtcVars.revision = 1;
640 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
5ba3f43e
A		 641
642 if (gIOChosenEntry
643 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateBootSignatureKey)))
644 && (sizeof(rtcVars.booterSignature) <= data->getLength()))
645 {
646 bcopy(data->getBytesNoCopy(), &rtcVars.booterSignature[0], sizeof(rtcVars.booterSignature));
647 }
648 else if (gIOHibernateBootSignature[0])
0c530ab8
A		 649 {
650 char c;
651 uint8_t value = 0;
5ba3f43e
A		 652 uint32_t in, out, digits;
653 for (in = out = digits = 0;
654 (c = gIOHibernateBootSignature[in]) && (in < sizeof(gIOHibernateBootSignature));
655 in++)
0c530ab8 656 {
5ba3f43e
A		 657 if ((c >= 'a') && (c <= 'f')) c -= 'a' - 10;
658 else if ((c >= 'A') && (c <= 'F')) c -= 'A' - 10;
659 else if ((c >= '0') && (c <= '9')) c -= '0';
660 else
661 {
662 if (c == '=') out = digits = value = 0;
663 continue;
664 }
0c530ab8 665 value = (value << 4) | c;
5ba3f43e
A		 666 if (digits & 1)
667 {
668 rtcVars.booterSignature[out++] = value;
669 if (out >= sizeof(rtcVars.booterSignature)) break;
670 }
671 digits++;
0c530ab8
A		 672 }
673 }
674 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
675 if (data)
2d21ac55 676 {
060df5ea
A		 677 if (gIOHibernateRTCVariablesKey)
678 IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
060df5ea 679 data->release();
0c530ab8 680 }
5c9f4661 681 if (gIOChosenEntry && gIOOptionsEntry)
0c530ab8
A		 682 {
683 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
3e170ce0
A		 684 if (data) gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
685 // set BootNext
686 if (!gIOHibernateBoot0082Data)
060df5ea 687 {
5c9f4661 688 OSData * fileData = 0;
3e170ce0 689 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
5c9f4661 690 if (data->getLength() >= 4) fileData = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-file-path"));
3e170ce0 691 if (data)
060df5ea 692 {
3e170ce0
A		 693 // AppleNVRAM_EFI_LOAD_OPTION
694 struct {
695 uint32_t Attributes;
696 uint16_t FilePathLength;
697 uint16_t Desc;
698 } loadOptionHeader;
699 loadOptionHeader.Attributes = 1;
700 loadOptionHeader.FilePathLength = data->getLength();
701 loadOptionHeader.Desc = 0;
5c9f4661
A		 702 if (fileData)
703 {
704 loadOptionHeader.FilePathLength -= 4;
705 loadOptionHeader.FilePathLength += fileData->getLength();
706 }
3e170ce0
A		 707 gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
708 if (gIOHibernateBoot0082Data)
060df5ea 709 {
3e170ce0 710 gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
5c9f4661
A		 711 if (fileData)
712 {
713 gIOHibernateBoot0082Data->appendBytes(data->getBytesNoCopy(), data->getLength() - 4);
714 gIOHibernateBoot0082Data->appendBytes(fileData);
715 }
716 else gIOHibernateBoot0082Data->appendBytes(data);
060df5ea
A		 717 }
718 }
3e170ce0
A		 719 }
720 if (!gIOHibernateBootNextData)
721 {
722 uint16_t bits = 0x0082;
723 gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
724 }
725 if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData)
726 {
727 gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
728 gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
729 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
730 }
731 // BootNext
3a60a9f5
A		 732 }
733 }
3e170ce0 734#endif /* !i386 && !x86_64 */
316670eb
A		 735 }
736 while (false);
737
39037602
A		 738 if (swapPinned) hibernate_pin_swap(FALSE);
739
316670eb 740 IOLockLock(gFSLock);
3e170ce0
A		 741 if ((kIOReturnSuccess == err) && (kFSOpening != gFSState))
742 {
743 HIBLOG("hibernate file close due timeout\n");
744 err = kIOReturnTimeout;
745 }
746 if (kIOReturnSuccess == err)
316670eb
A		 747 {
748 gFSState = kFSOpened;
749 gIOHibernateVars = *vars;
750 gFileVars = *vars->fileVars;
3e170ce0 751 gFileVars.allocated = false;
316670eb 752 gIOHibernateVars.fileVars = &gFileVars;
3a60a9f5
A		 753 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
754 gIOHibernateState = kIOHibernateStateHibernating;
755 }
316670eb
A		 756 else
757 {
3e170ce0 758 IOPolledFileIOVars * fileVars = vars->fileVars;
316670eb 759 IOHibernateDone(vars);
3e170ce0
A		 760 IOPolledFileClose(&fileVars,
761#if DISABLE_TRIM
762 0, NULL, 0, 0, 0);
763#else
764 0, NULL, 0, sizeof(IOHibernateImageHeader), setFileSize);
765#endif
316670eb
A		 766 gFSState = kFSIdle;
767 }
768 IOLockUnlock(gFSLock);
769
770 if (vars->fileVars) IODelete(vars->fileVars, IOPolledFileIOVars, 1);
771 IODelete(vars, IOHibernateVars, 1);
3a60a9f5
A		 772
773 return (err);
774}
775
776/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
777
3e170ce0
A		 778static void
779IOSetBootImageNVRAM(OSData * data)
780{
781 IORegistryEntry * regEntry;
782
783 if (!gIOOptionsEntry)
784 {
785 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
786 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
787 if (regEntry && !gIOOptionsEntry)
788 regEntry->release();
789 }
790 if (gIOOptionsEntry && gIOHibernateBootImageKey)
791 {
792 if (data) gIOOptionsEntry->setProperty(gIOHibernateBootImageKey, data);
793 else
794 {
795 gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
796 gIOOptionsEntry->sync();
797 }
798 }
799}
800
801/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
802/*
803 * Writes header to disk with signature, block size and file extents data.
804 * If there are more than 2 extents, then they are written on second block.
805 */
806static IOReturn
807IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature)
808{
809 IOHibernateImageHeader hdr;
810 IOItemCount count;
811 IOReturn err = kIOReturnSuccess;
812 int rc;
813 IOPolledFileExtent * fileExtents;
814
39037602 815 fileExtents = (typeof(fileExtents)) vars->fileExtents->getBytesNoCopy();
3e170ce0
A		 816
817 memset(&hdr, 0, sizeof(IOHibernateImageHeader));
818 count = vars->fileExtents->getLength();
819 if (count > sizeof(hdr.fileExtentMap))
820 {
821 hdr.fileExtentMapSize = count;
822 count = sizeof(hdr.fileExtentMap);
823 }
824 else
825 hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap);
826
827 bcopy(fileExtents, &hdr.fileExtentMap[0], count);
828
829 // copy file block extent list if larger than header
830 if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap))
831 {
832 count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap);
833 rc = kern_write_file(vars->fileRef, vars->blockSize,
834 (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)),
835 count, IO_SKIP_ENCRYPTION);
836 if (rc != 0) {
837 HIBLOG("kern_write_file returned %d\n", rc);
838 err = kIOReturnIOError;
839 goto exit;
840 }
841 }
842 hdr.signature = signature;
843 hdr.deviceBlockSize = vars->blockSize;
844
845 rc = kern_write_file(vars->fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION);
846 if (rc != 0) {
847 HIBLOG("kern_write_file returned %d\n", rc);
848 err = kIOReturnIOError;
849 goto exit;
850 }
851
852exit:
853 return err;
854}
855
39037602
A		 856extern "C" boolean_t root_is_CF_drive;
857
3e170ce0
A		 858void
859IOOpenDebugDataFile(const char *fname, uint64_t size)
860{
861 IOReturn err;
862 OSData * imagePath = NULL;
863 uint64_t padding;
864
865 if (!gDebugImageLock) {
866 gDebugImageLock = IOLockAlloc();
867 }
868
39037602
A		 869 if (root_is_CF_drive) return;
870
3e170ce0
A		 871 // Try to get a lock, but don't block for getting lock
872 if (!IOLockTryLock(gDebugImageLock)) {
873 HIBLOG("IOOpenDebugDataFile: Failed to get lock\n");
874 return;
875 }
876
877 if (gDebugImageFileVars || !fname || !size) {
878 HIBLOG("IOOpenDebugDataFile: conditions failed\n");
879 goto exit;
880 }
881
882 padding = (PAGE_SIZE*2); // allocate couple more pages for header and fileextents
883 err = IOPolledFileOpen(fname, size+padding, 32ULL*1024*1024*1024,
884 NULL, 0,
885 &gDebugImageFileVars, &imagePath, NULL, 0);
886
887 if ((kIOReturnSuccess == err) && imagePath)
888 {
889 if ((gDebugImageFileVars->fileSize < (size+padding)) ||
890 (gDebugImageFileVars->fileExtents->getLength() > PAGE_SIZE)) {
891 // Can't use the file
892 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
893 HIBLOG("IOOpenDebugDataFile: too many file extents\n");
894 goto exit;
895 }
896
897 // write extents for debug data usage in EFI
898 IOWriteExtentsToFile(gDebugImageFileVars, kIOHibernateHeaderOpenSignature);
899 IOSetBootImageNVRAM(imagePath);
3e170ce0
A		 900 }
901
902exit:
903 IOLockUnlock(gDebugImageLock);
904
905 if (imagePath) imagePath->release();
906 return;
907}
908
39037602
A		 909void
910IOCloseDebugDataFile()
911{
912 IOSetBootImageNVRAM(0);
913
914 if (gDebugImageLock) {
915 IOLockLock(gDebugImageLock);
916 if (gDebugImageFileVars != 0) {
39037602
A		 917 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
918 }
919 IOLockUnlock(gDebugImageLock);
920 }
921
922
923}
924
3e170ce0
A		 925/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
926
2d21ac55
A		 927DECLARE_IOHIBERNATEPROGRESSALPHA
928
929static void
930ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
931{
932 uint32_t rowBytes, pixelShift;
933 uint32_t x, y;
934 int32_t blob;
935 uint32_t alpha, in, color, result;
936 uint8_t * out;
937 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
938
939 rowBytes = display->rowBytes;
940 pixelShift = display->depth >> 4;
941 if (pixelShift < 1) return;
942
943 screen += ((display->width
944 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
945 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
946
947 for (y = 0; y < kIOHibernateProgressHeight; y++)
948 {
949 out = screen + y * rowBytes;
950 for (blob = 0; blob < kIOHibernateProgressCount; blob++)
951 {
952 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
953 for (x = 0; x < kIOHibernateProgressWidth; x++)
954 {
955 alpha = gIOHibernateProgressAlpha[y][x];
956 result = color;
957 if (alpha)
958 {
959 if (0xff != alpha)
960 {
961 if (1 == pixelShift)
962 {
963 in = *((uint16_t *)out) & 0x1f; // 16
964 in = (in << 3) | (in >> 2);
965 }
966 else
967 in = *((uint32_t *)out) & 0xff; // 32
968 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
969 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
970 }
971 if (1 == pixelShift)
972 {
973 result >>= 3;
974 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
975 }
976 else
977 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
978 }
979 out += (1 << pixelShift);
980 }
981 out += (kIOHibernateProgressSpacing << pixelShift);
982 }
983 }
984}
985
986
987static void
988ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
989{
990 uint32_t rowBytes, pixelShift;
991 uint32_t x, y;
992 int32_t blob, lastBlob;
993 uint32_t alpha, in, color, result;
994 uint8_t * out;
995 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
996
997 pixelShift = display->depth >> 4;
998 if (pixelShift < 1)
999 return;
1000
1001 rowBytes = display->rowBytes;
1002
1003 screen += ((display->width
1004 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1005 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1006
1007 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
1008
1009 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
1010
1011 for (y = 0; y < kIOHibernateProgressHeight; y++)
1012 {
1013 out = screen + y * rowBytes;
1014 for (blob = firstBlob; blob <= lastBlob; blob++)
1015 {
1016 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
1017 for (x = 0; x < kIOHibernateProgressWidth; x++)
1018 {
1019 alpha = gIOHibernateProgressAlpha[y][x];
1020 result = color;
1021 if (alpha)
1022 {
1023 if (0xff != alpha)
1024 {
1025 in = display->progressSaveUnder[blob][saveindex[blob]++];
1026 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
1027 }
1028 if (1 == pixelShift)
1029 {
1030 result >>= 3;
1031 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1032 }
1033 else
1034 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1035 }
1036 out += (1 << pixelShift);
1037 }
1038 out += (kIOHibernateProgressSpacing << pixelShift);
1039 }
1040 }
1041}
1042
1043/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1044
316670eb
A		 1045IOReturn
1046IOHibernateIOKitSleep(void)
1047{
1048 IOReturn ret = kIOReturnSuccess;
1049 IOLockLock(gFSLock);
1050 if (kFSOpening == gFSState)
1051 {
1052 gFSState = kFSTimedOut;
1053 HIBLOG("hibernate file open timed out\n");
1054 ret = kIOReturnTimeout;
1055 }
1056 IOLockUnlock(gFSLock);
1057 return (ret);
1058}
1059
1060/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1061
3a60a9f5
A		 1062IOReturn
1063IOHibernateSystemHasSlept(void)
1064{
316670eb 1065 IOReturn ret = kIOReturnSuccess;
3a60a9f5 1066 IOHibernateVars * vars = &gIOHibernateVars;
316670eb 1067 OSObject * obj = 0;
2d21ac55
A		 1068 OSData * data;
1069
316670eb
A		 1070 IOLockLock(gFSLock);
1071 if ((kFSOpened != gFSState) && gIOHibernateMode)
1072 {
1073 ret = kIOReturnTimeout;
1074 }
1075 IOLockUnlock(gFSLock);
1076 if (kIOReturnSuccess != ret) return (ret);
1077
1078 if (gIOHibernateMode) obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
2d21ac55
A		 1079 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1080 if (obj && !vars->previewBuffer)
1081 obj->release();
3a60a9f5 1082
2d21ac55
A		 1083 vars->consoleMapping = NULL;
1084 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
3a60a9f5 1085 {
2d21ac55
A		 1086 vars->previewBuffer->release();
1087 vars->previewBuffer = 0;
1088 }
3a60a9f5 1089
0b4c1975
A		 1090 if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1091 && vars->previewBuffer
1092 && (data = OSDynamicCast(OSData,
2d21ac55
A		 1093 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
1094 {
1095 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
b0d623f7 1096 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
2d21ac55
A		 1097
1098 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1099
1100 if (kIOHibernatePreviewUpdates & flags)
1101 {
1102 PE_Video consoleInfo;
1103 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1104
1105 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1106
1107 graphicsInfo->width = consoleInfo.v_width;
1108 graphicsInfo->height = consoleInfo.v_height;
1109 graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
1110 graphicsInfo->depth = consoleInfo.v_depth;
1111 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1112
1113 HIBPRINT("video %p %d %d %d\n",
0b4c1975
A		 1114 vars->consoleMapping, graphicsInfo->depth,
1115 graphicsInfo->width, graphicsInfo->height);
2d21ac55
A		 1116 if (vars->consoleMapping)
1117 ProgressInit(graphicsInfo, vars->consoleMapping,
1118 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1119 }
3a60a9f5 1120 }
2d21ac55 1121
3a60a9f5
A		 1122 if (gIOOptionsEntry)
1123 gIOOptionsEntry->sync();
1124
316670eb 1125 return (ret);
3a60a9f5
A		 1126}
1127
1128/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1129
6d2010ae
A		 1130static DeviceTreeNode *
1131MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry)
1132{
1133 DeviceTreeNodeProperty * prop;
1134 DeviceTreeNode * child;
1135 IORegistryEntry * childRegEntry;
1136 const char * nameProp;
1137 unsigned int propLen, idx;
1138
1139 prop = (DeviceTreeNodeProperty *) (entry + 1);
1140 for (idx = 0; idx < entry->nProperties; idx++)
1141 {
1142 if (regEntry && (0 != strcmp("name", prop->name)))
1143 {
1144 regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
1145// HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
1146 }
1147 prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
1148 }
1149
1150 child = (DeviceTreeNode *) prop;
1151 for (idx = 0; idx < entry->nChildren; idx++)
1152 {
1153 if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen))
1154 panic("no name");
1155 childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
1156// HIBPRINT("%s == %p\n", nameProp, childRegEntry);
1157 child = MergeDeviceTree(child, childRegEntry);
1158 }
1159 return (child);
1160}
1161
3a60a9f5
A		 1162IOReturn
1163IOHibernateSystemWake(void)
1164{
316670eb
A		 1165 if (kFSOpened == gFSState)
1166 {
3e170ce0 1167 IOPolledFilePollersClose(gIOHibernateVars.fileVars, kIOPolledPostflightState);
316670eb
A		 1168 IOHibernateDone(&gIOHibernateVars);
1169 }
1170 else
1171 {
1172 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1173 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1174 }
1175 return (kIOReturnSuccess);
1176}
3a60a9f5 1177
316670eb
A		 1178static IOReturn
1179IOHibernateDone(IOHibernateVars * vars)
1180{
99c3a104 1181 hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
3a60a9f5
A		 1182
1183 if (vars->videoMapping)
1184 {
1185 if (vars->videoMapSize)
1186 // remove mappings
1187 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1188 if (vars->videoAllocSize)
1189 // dealloc range
b0d623f7 1190 kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
3a60a9f5
A		 1191 }
1192
1193 if (vars->previewBuffer)
1194 {
1195 vars->previewBuffer->release();
1196 vars->previewBuffer = 0;
1197 }
1198
0b4c1975
A		 1199 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1200 {
1201 IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
1202 gIOHibernateCurrentHeader->options, 32);
1203 }
1204 else
1205 {
1206 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1207 }
1208
1209 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1210 && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus))
1211 {
1212 IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
1213 &gIOHibernateGraphicsInfo->gfxStatus,
1214 sizeof(gIOHibernateGraphicsInfo->gfxStatus));
1215 }
1216 else
1217 {
1218 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1219 }
1220
3a60a9f5
A		 1221 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1222
b0d623f7 1223#if defined(__i386__) || defined(__x86_64__)
060df5ea 1224 IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
6d2010ae 1225 IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
2d21ac55
A		 1226
1227 /*
1228 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1229 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1230 */
1231 if (gIOOptionsEntry) {
2d21ac55 1232
060df5ea
A		 1233 if (gIOHibernateRTCVariablesKey) {
1234 if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
1235 gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
1236 }
1237 }
1238
1239 if (gIOHibernateBootNextKey)
1240 {
1241 if (gIOHibernateBootNextSave)
1242 {
1243 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
1244 gIOHibernateBootNextSave->release();
1245 gIOHibernateBootNextSave = NULL;
2d21ac55 1246 }
060df5ea
A		 1247 else
1248 gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
1249 }
39236c6e 1250 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) gIOOptionsEntry->sync();
2d21ac55 1251 }
0c530ab8 1252#endif
3a60a9f5 1253
3e170ce0 1254 if (vars->srcBuffer) vars->srcBuffer->release();
6d2010ae 1255 bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
99c3a104 1256 if (vars->handoffBuffer)
6d2010ae 1257 {
99c3a104 1258 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
6d2010ae 1259 {
99c3a104
A		 1260 IOHibernateHandoff * handoff;
1261 bool done = false;
1262 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
1263 !done;
1264 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
6d2010ae 1265 {
99c3a104
A		 1266 HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
1267 uint8_t * data = &handoff->data[0];
1268 switch (handoff->type)
1269 {
1270 case kIOHibernateHandoffTypeEnd:
1271 done = true;
1272 break;
6d2010ae 1273
99c3a104
A		 1274 case kIOHibernateHandoffTypeDeviceTree:
1275 MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
1276 break;
1277
1278 case kIOHibernateHandoffTypeKeyStore:
6d2010ae 1279#if defined(__i386__) || defined(__x86_64__)
6d2010ae 1280 {
99c3a104
A		 1281 IOBufferMemoryDescriptor *
1282 md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
1283 if (md)
1284 {
1285 IOSetKeyStoreData(md);
1286 }
6d2010ae 1287 }
6d2010ae 1288#endif
99c3a104
A		 1289 break;
1290
1291 default:
1292 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
1293 break;
1294 }
1295 }
6d2010ae
A		 1296 }
1297 vars->handoffBuffer->release();
1298 }
3a60a9f5
A		 1299
1300 bzero(vars, sizeof(*vars));
1301
1302// gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1303
1304 return (kIOReturnSuccess);
1305}
1306
5ba3f43e 1307static void
813fb2f6 1308IOHibernateSystemPostWakeTrim(void * p1, void * p2)
3a60a9f5 1309{
813fb2f6
A		 1310 // invalidate & close the image file
1311 if (p1) IOLockLock(gFSLock);
1312 if (kFSTrimDelay == gFSState)
3a60a9f5 1313 {
3e170ce0
A		 1314 IOPolledFileIOVars * vars = &gFileVars;
1315 IOPolledFileClose(&vars,
39236c6e 1316#if DISABLE_TRIM
813fb2f6 1317 0, NULL, 0, 0, 0);
39236c6e 1318#else
813fb2f6
A		 1319 0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader),
1320 sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize);
39236c6e 1321#endif
813fb2f6 1322 gFSState = kFSIdle;
fe8ab488 1323 }
813fb2f6
A		 1324 if (p1) IOLockUnlock(gFSLock);
1325}
fe8ab488 1326
813fb2f6 1327IOReturn
5ba3f43e 1328IOHibernateSystemPostWake(bool now)
813fb2f6
A		 1329{
1330 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1331 IOLockLock(gFSLock);
5c9f4661
A		 1332 if (kFSTrimDelay == gFSState)
1333 {
1334 thread_call_cancel(gIOHibernateTrimCalloutEntry);
1335 IOHibernateSystemPostWakeTrim(NULL, NULL);
1336 }
813fb2f6
A		 1337 else if (kFSOpened != gFSState) gFSState = kFSIdle;
1338 else
1339 {
813fb2f6 1340 gFSState = kFSTrimDelay;
5c9f4661
A		 1341 if (now)
1342 {
1343 thread_call_cancel(gIOHibernateTrimCalloutEntry);
1344 IOHibernateSystemPostWakeTrim(NULL, NULL);
1345 }
5ba3f43e
A		 1346 else
1347 {
1348 AbsoluteTime deadline;
1349 clock_interval_to_deadline(TRIM_DELAY, kMillisecondScale, &deadline );
1350 thread_call_enter1_delayed(gIOHibernateTrimCalloutEntry, NULL, deadline);
1351 }
813fb2f6
A		 1352 }
1353 IOLockUnlock(gFSLock);
fe8ab488 1354
3a60a9f5
A		 1355 return (kIOReturnSuccess);
1356}
1357
d190cdc3 1358uint32_t IOHibernateWasScreenLocked(void)
316670eb 1359{
d190cdc3 1360 uint32_t ret = 0;
316670eb
A		 1361 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry)
1362 {
1363 OSData *
1364 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
d190cdc3 1365 if (data)
316670eb 1366 {
d190cdc3
A		 1367 ret = ((uint32_t *)data->getBytesNoCopy())[0];
1368 gIOChosenEntry->setProperty(kIOBooterScreenLockStateKey, data);
1369 }
316670eb 1370 }
d190cdc3
A		 1371 else gIOChosenEntry->removeProperty(kIOBooterScreenLockStateKey);
1372
316670eb
A		 1373 return (ret);
1374}
1375
3a60a9f5
A		 1376/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1377
2d21ac55 1378SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
6d2010ae 1379 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2d21ac55
A		 1380 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
1381SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
6d2010ae 1382 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2d21ac55
A		 1383 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
1384SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
6d2010ae 1385 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2d21ac55 1386 &gIOHibernateMode, 0, "");
39236c6e
A		 1387SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
1388 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
39037602 1389 &_hibernateStats, hibernate_statistics_t, "");
39236c6e
A		 1390
1391SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
1392 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
39037602 1393 &_hibernateStats.graphicsReadyTime, 0, "");
39236c6e
A		 1394SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
1395 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
39037602 1396 &_hibernateStats.wakeNotificationTime, 0, "");
39236c6e
A		 1397SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
1398 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
39037602 1399 &_hibernateStats.lockScreenReadyTime, 0, "");
39236c6e
A		 1400SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
1401 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
39037602 1402 &_hibernateStats.hidReadyTime, 0, "");
39236c6e 1403
2d21ac55 1404
3a60a9f5
A		 1405void
1406IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1407{
3e170ce0
A		 1408 gIOHibernateBootImageKey = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1409
1410#if defined(__i386__) || defined(__x86_64__)
1411 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1412 gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
1413 gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
1414 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1415#endif /* defined(__i386__) || defined(__x86_64__) */
1416
3a60a9f5
A		 1417 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
1418 if (data)
1419 {
1420 rootDomain->setProperty(kIOHibernateStateKey, data);
1421 data->release();
1422 }
1423
2d21ac55 1424 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))
3a60a9f5
A		 1425 gIOHibernateMode = kIOHibernateModeOn;
1426 else
1427 gIOHibernateFilename[0] = 0;
1428
3a60a9f5 1429 sysctl_register_oid(&sysctl__kern_hibernatefile);
3a60a9f5 1430 sysctl_register_oid(&sysctl__kern_bootsignature);
3a60a9f5 1431 sysctl_register_oid(&sysctl__kern_hibernatemode);
39236c6e
A		 1432 sysctl_register_oid(&sysctl__kern_hibernatestatistics);
1433 sysctl_register_oid(&sysctl__kern_hibernategraphicsready);
1434 sysctl_register_oid(&sysctl__kern_hibernatewakenotification);
1435 sysctl_register_oid(&sysctl__kern_hibernatelockscreenready);
1436 sysctl_register_oid(&sysctl__kern_hibernatehidready);
316670eb 1437
3e170ce0
A		 1438 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1439
316670eb 1440 gFSLock = IOLockAlloc();
3a60a9f5
A		 1441}
1442
3a60a9f5
A		 1443/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1444
3e170ce0
A		 1445static IOReturn
1446IOHibernatePolledFileWrite(IOPolledFileIOVars * vars,
1447 const uint8_t * bytes, IOByteCount size,
1448 IOPolledFileCryptVars * cryptvars)
6d2010ae 1449{
3e170ce0 1450 IOReturn err;
6d2010ae 1451
3e170ce0
A		 1452 err = IOPolledFileWrite(vars, bytes, size, cryptvars);
1453 if ((kIOReturnSuccess == err) && hibernate_should_abort()) err = kIOReturnAborted;
6d2010ae 1454
3e170ce0 1455 return (err);
6d2010ae
A		 1456}
1457
3e170ce0 1458/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
6d2010ae 1459
2d21ac55 1460extern "C" uint32_t
3a60a9f5
A		 1461hibernate_write_image(void)
1462{
1463 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
1464 IOHibernateVars * vars = &gIOHibernateVars;
1465 IOPolledFileExtent * fileExtents;
1466
39037602 1467 _static_assert_1_arg(sizeof(IOHibernateImageHeader) == 512);
0c530ab8 1468
3a60a9f5
A		 1469 uint32_t pageCount, pagesDone;
1470 IOReturn err;
0b4c1975
A		 1471 vm_offset_t ppnum, page;
1472 IOItemCount count;
3a60a9f5
A		 1473 uint8_t * src;
1474 uint8_t * data;
39236c6e
A		 1475 uint8_t * compressed;
1476 uint8_t * scratch;
3e170ce0 1477 IOByteCount pageCompressedSize;
3a60a9f5
A		 1478 uint64_t compressedSize, uncompressedSize;
1479 uint64_t image1Size = 0;
1480 uint32_t bitmap_size;
0b4c1975 1481 bool iterDone, pollerOpen, needEncrypt;
3a60a9f5 1482 uint32_t restore1Sum, sum, sum1, sum2;
39236c6e 1483 int wkresult;
3a60a9f5
A		 1484 uint32_t tag;
1485 uint32_t pageType;
1486 uint32_t pageAndCount[2];
6d2010ae
A		 1487 addr64_t phys64;
1488 IOByteCount segLen;
3e170ce0
A		 1489 uintptr_t hibernateBase;
1490 uintptr_t hibernateEnd;
3a60a9f5
A		 1491
1492 AbsoluteTime startTime, endTime;
6d2010ae 1493 AbsoluteTime allTime, compTime;
0b4c1975 1494 uint64_t compBytes;
3a60a9f5
A		 1495 uint64_t nsec;
1496 uint32_t lastProgressStamp = 0;
1497 uint32_t progressStamp;
2d21ac55 1498 uint32_t blob, lastBlob = (uint32_t) -1L;
3a60a9f5 1499
39236c6e
A		 1500 uint32_t wiredPagesEncrypted;
1501 uint32_t dirtyPagesEncrypted;
1502 uint32_t wiredPagesClear;
3e170ce0
A		 1503 uint32_t svPageCount;
1504 uint32_t zvPageCount;
39236c6e 1505
3e170ce0
A		 1506 IOPolledFileCryptVars _cryptvars;
1507 IOPolledFileCryptVars * cryptvars = 0;
3a60a9f5 1508
39236c6e
A		 1509 wiredPagesEncrypted = 0;
1510 dirtyPagesEncrypted = 0;
1511 wiredPagesClear = 0;
3e170ce0
A		 1512 svPageCount = 0;
1513 zvPageCount = 0;
0b4c1975 1514
39037602
A		 1515 if (!vars->fileVars
1516 || !vars->fileVars->pollers
1517 || !(kIOHibernateModeOn & gIOHibernateMode)) return (kIOHibernatePostWriteSleep);
3a60a9f5 1518
0b4c1975
A		 1519 if (kIOHibernateModeSleep & gIOHibernateMode)
1520 kdebug_enable = save_kdebug_enable;
1521
5ba3f43e 1522 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START);
6d2010ae 1523 IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
0b4c1975 1524
3a60a9f5
A		 1525 restore1Sum = sum1 = sum2 = 0;
1526
2d21ac55 1527#if CRYPTO
3a60a9f5
A		 1528 // encryption data. "iv" is the "initial vector".
1529 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1530 {
1531 static const unsigned char first_iv[AES_BLOCK_SIZE]
1532 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
1533 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
1534
1535 cryptvars = &gIOHibernateCryptWakeContext;
3e170ce0 1536 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
3a60a9f5
A		 1537 aes_encrypt_key(vars->cryptKey,
1538 kIOHibernateAESKeySize,
1539 &cryptvars->ctx.encrypt);
1540 aes_decrypt_key(vars->cryptKey,
1541 kIOHibernateAESKeySize,
1542 &cryptvars->ctx.decrypt);
1543
1544 cryptvars = &_cryptvars;
3e170ce0 1545 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
6d2010ae
A		 1546 for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++)
1547 vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
1548 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
3a60a9f5
A		 1549 aes_encrypt_key(vars->wiredCryptKey,
1550 kIOHibernateAESKeySize,
1551 &cryptvars->ctx.encrypt);
1552
1553 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1554 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1555 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
3a60a9f5 1556 }
2d21ac55 1557#endif /* CRYPTO */
3a60a9f5 1558
3a60a9f5
A		 1559 hibernate_page_list_setall(vars->page_list,
1560 vars->page_list_wired,
db609669
A		 1561 vars->page_list_pal,
1562 false /* !preflight */,
39236c6e
A		 1563 /* discard_all */
1564 ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
1565 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
3a60a9f5
A		 1566 &pageCount);
1567
1568 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
1569
3e170ce0 1570 fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy();
3a60a9f5
A		 1571
1572#if 0
1573 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
1574 for (page = 0; page < count; page++)
1575 {
1576 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
1577 fileExtents[page].start, fileExtents[page].length,
1578 fileExtents[page].start + fileExtents[page].length);
1579 }
1580#endif
1581
0b4c1975 1582 needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
3a60a9f5 1583 AbsoluteTime_to_scalar(&compTime) = 0;
6d2010ae 1584 compBytes = 0;
3a60a9f5
A		 1585
1586 clock_get_uptime(&allTime);
b0d623f7
A		 1587 IOService::getPMRootDomain()->pmStatsRecordEvent(
1588 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
3a60a9f5
A		 1589 do
1590 {
1591 compressedSize = 0;
1592 uncompressedSize = 0;
3e170ce0
A		 1593 svPageCount = 0;
1594 zvPageCount = 0;
3a60a9f5 1595
39236c6e 1596 IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
3a60a9f5
A		 1597
1598 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
1599 ml_get_interrupts_enabled());
00867663
A		 1600 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState,
1601 // abortable if not low battery
1602 !IOService::getPMRootDomain()->mustHibernate());
3a60a9f5
A		 1603 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
1604 pollerOpen = (kIOReturnSuccess == err);
1605 if (!pollerOpen)
1606 break;
1607
1608 // copy file block extent list if larger than header
1609
3e170ce0 1610 count = vars->fileVars->fileExtents->getLength();
3a60a9f5
A		 1611 if (count > sizeof(header->fileExtentMap))
1612 {
1613 count -= sizeof(header->fileExtentMap);
3e170ce0 1614 err = IOHibernatePolledFileWrite(vars->fileVars,
3a60a9f5
A		 1615 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
1616 if (kIOReturnSuccess != err)
1617 break;
1618 }
1619
6d2010ae 1620 hibernateBase = HIB_BASE; /* Defined in PAL headers */
316670eb 1621 hibernateEnd = (segHIBB + segSizeHIB);
6d2010ae 1622
3a60a9f5 1623 // copy out restore1 code
6d2010ae
A		 1624
1625 for (count = 0;
1626 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1627 count += segLen)
1628 {
1629 for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++)
1630 {
1631 gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
1632 }
1633 }
1634
1635 page = atop_32(kvtophys(hibernateBase));
1636 count = atop_32(round_page(hibernateEnd) - hibernateBase);
1637 header->restore1CodePhysPage = page;
1638 header->restore1CodeVirt = hibernateBase;
3a60a9f5 1639 header->restore1PageCount = count;
b0d623f7
A		 1640 header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
1641 header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
3a60a9f5 1642
316670eb 1643 // sum __HIB seg, with zeros for the stack
b0d623f7 1644 src = (uint8_t *) trunc_page(hibernateBase);
3a60a9f5
A		 1645 for (page = 0; page < count; page++)
1646 {
1647 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
6d2010ae 1648 restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
3a60a9f5 1649 else
0b4c1975 1650 restore1Sum += 0x00000000;
3a60a9f5
A		 1651 src += page_size;
1652 }
1653 sum1 = restore1Sum;
1654
316670eb 1655 // write the __HIB seg, with zeros for the stack
3a60a9f5 1656
b0d623f7
A		 1657 src = (uint8_t *) trunc_page(hibernateBase);
1658 count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
3a60a9f5
A		 1659 if (count)
1660 {
3e170ce0 1661 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
3a60a9f5
A		 1662 if (kIOReturnSuccess != err)
1663 break;
1664 }
3e170ce0 1665 err = IOHibernatePolledFileWrite(vars->fileVars,
3a60a9f5
A		 1666 (uint8_t *) 0,
1667 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
1668 cryptvars);
1669 if (kIOReturnSuccess != err)
1670 break;
1671 src = &gIOHibernateRestoreStackEnd[0];
b0d623f7 1672 count = round_page(hibernateEnd) - ((uintptr_t) src);
3a60a9f5
A		 1673 if (count)
1674 {
3e170ce0 1675 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
3a60a9f5
A		 1676 if (kIOReturnSuccess != err)
1677 break;
1678 }
1679
39236c6e
A		 1680 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1681 {
1682 vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
1683 vars->fileVars->encryptEnd = UINT64_MAX;
1684 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1685 }
316670eb 1686
3a60a9f5
A		 1687 // write the preview buffer
1688
2d21ac55 1689 if (vars->previewBuffer)
3a60a9f5
A		 1690 {
1691 ppnum = 0;
1692 count = 0;
1693 do
1694 {
b0d623f7 1695 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
3a60a9f5
A		 1696 pageAndCount[0] = atop_64(phys64);
1697 pageAndCount[1] = atop_32(segLen);
3e170ce0 1698 err = IOHibernatePolledFileWrite(vars->fileVars,
3a60a9f5
A		 1699 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1700 cryptvars);
1701 if (kIOReturnSuccess != err)
1702 break;
1703 count += segLen;
1704 ppnum += sizeof(pageAndCount);
1705 }
1706 while (phys64);
1707 if (kIOReturnSuccess != err)
1708 break;
1709
b0d623f7 1710 src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
316670eb
A		 1711
1712 ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
1713
2d21ac55 1714 count = vars->previewBuffer->getLength();
3a60a9f5
A		 1715
1716 header->previewPageListSize = ppnum;
1717 header->previewSize = count + ppnum;
1718
1719 for (page = 0; page < count; page += page_size)
0b4c1975
A		 1720 {
1721 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
1722 sum1 += hibernate_sum_page(src + page, atop_64(phys64));
1723 }
3e170ce0 1724 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
3a60a9f5
A		 1725 if (kIOReturnSuccess != err)
1726 break;
1727 }
1728
1729 // mark areas for no save
3e170ce0
A		 1730 IOMemoryDescriptor * ioBuffer;
1731 ioBuffer = IOPolledFileGetIOBuffer(vars->fileVars);
3a60a9f5 1732 for (count = 0;
3e170ce0 1733 (phys64 = ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
3a60a9f5
A		 1734 count += segLen)
1735 {
1736 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1737 atop_64(phys64), atop_32(segLen),
1738 kIOHibernatePageStateFree);
1739 pageCount -= atop_32(segLen);
1740 }
1741
1742 for (count = 0;
b0d623f7 1743 (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
3a60a9f5
A		 1744 count += segLen)
1745 {
1746 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1747 atop_64(phys64), atop_32(segLen),
1748 kIOHibernatePageStateFree);
1749 pageCount -= atop_32(segLen);
1750 }
1751
1752 // copy out bitmap of pages available for trashing during restore
1753
1754 bitmap_size = vars->page_list_wired->list_size;
1755 src = (uint8_t *) vars->page_list_wired;
3e170ce0 1756 err = IOHibernatePolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
3a60a9f5
A		 1757 if (kIOReturnSuccess != err)
1758 break;
1759
1760 // mark more areas for no save, but these are not available
1761 // for trashing during restore
0c530ab8
A		 1762
1763 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
3a60a9f5 1764
6d2010ae
A		 1765
1766 page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
1767 count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
3a60a9f5
A		 1768 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1769 page, count,
1770 kIOHibernatePageStateFree);
1771 pageCount -= count;
3a60a9f5
A		 1772
1773 if (vars->previewBuffer) for (count = 0;
b0d623f7 1774 (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
3a60a9f5
A		 1775 count += segLen)
1776 {
1777 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1778 atop_64(phys64), atop_32(segLen),
1779 kIOHibernatePageStateFree);
1780 pageCount -= atop_32(segLen);
1781 }
1782
6d2010ae
A		 1783 for (count = 0;
1784 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1785 count += segLen)
1786 {
1787 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1788 atop_64(phys64), atop_32(segLen),
1789 kIOHibernatePageStateFree);
1790 pageCount -= atop_32(segLen);
1791 }
1792
5ba3f43e
A		 1793#if KASAN
1794 vm_size_t shadow_pages_free = atop_64(shadow_ptop) - atop_64(shadow_pnext);
1795
1796 /* no need to save unused shadow pages */
1797 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1798 atop_64(shadow_pnext),
1799 shadow_pages_free,
1800 kIOHibernatePageStateFree);
1801#endif
1802
3a60a9f5 1803 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
39236c6e
A		 1804 compressed = src + page_size;
1805 scratch = compressed + page_size;
1806
6d2010ae
A		 1807 pagesDone = 0;
1808 lastBlob = 0;
3a60a9f5 1809
db609669
A		 1810 HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
1811 bitmap_size, header->previewSize,
1812 pageCount, vars->fileVars->position);
3a60a9f5 1813
0b4c1975
A		 1814 enum
1815 // pageType
1816 {
1817 kWired = 0x02,
1818 kEncrypt = 0x01,
1819 kWiredEncrypt = kWired | kEncrypt,
1820 kWiredClear = kWired,
1821 kUnwiredEncrypt = kEncrypt
1822 };
1823
39037602
A		 1824 bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features()));
1825#define _pmap_is_noencrypt(x) (cpuAES ? false : pmap_is_noencrypt((x)))
1826
0b4c1975 1827 for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)
3a60a9f5 1828 {
316670eb
A		 1829 if (kUnwiredEncrypt == pageType)
1830 {
1831 // start unwired image
39236c6e
A		 1832 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1833 {
1834 vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
1835 vars->fileVars->encryptEnd = UINT64_MAX;
1836 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1837 }
316670eb
A		 1838 bcopy(&cryptvars->aes_iv[0],
1839 &gIOHibernateCryptWakeContext.aes_iv[0],
1840 sizeof(cryptvars->aes_iv));
1841 cryptvars = &gIOHibernateCryptWakeContext;
0b4c1975
A		 1842 }
1843 for (iterDone = false, ppnum = 0; !iterDone; )
1844 {
1845 count = hibernate_page_list_iterate((kWired & pageType)
1846 ? vars->page_list_wired : vars->page_list,
1847 &ppnum);
1848// kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
1849 iterDone = !count;
39037602 1850
0b4c1975 1851 if (count && (kWired & pageType) && needEncrypt)
3a60a9f5 1852 {
0b4c1975
A		 1853 uint32_t checkIndex;
1854 for (checkIndex = 0;
1855 (checkIndex < count)
39037602 1856 && (((kEncrypt & pageType) == 0) == _pmap_is_noencrypt(ppnum + checkIndex));
0b4c1975
A		 1857 checkIndex++)
1858 {}
1859 if (!checkIndex)
1860 {
1861 ppnum++;
1862 continue;
1863 }
1864 count = checkIndex;
3a60a9f5
A		 1865 }
1866
0b4c1975
A		 1867 switch (pageType)
1868 {
39236c6e
A		 1869 case kWiredEncrypt: wiredPagesEncrypted += count; break;
1870 case kWiredClear: wiredPagesClear += count; break;
1871 case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
0b4c1975
A		 1872 }
1873
1874 if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */}
3a60a9f5 1875 else
3a60a9f5 1876 {
0b4c1975
A		 1877 pageAndCount[0] = ppnum;
1878 pageAndCount[1] = count;
3e170ce0 1879 err = IOHibernatePolledFileWrite(vars->fileVars,
0b4c1975
A		 1880 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1881 cryptvars);
1882 if (kIOReturnSuccess != err)
1883 break;
3a60a9f5 1884 }
0b4c1975
A		 1885
1886 for (page = ppnum; page < (ppnum + count); page++)
3a60a9f5 1887 {
0b4c1975
A		 1888 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
1889 if (err)
1890 {
1891 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
1892 break;
1893 }
1894
1895 sum = hibernate_sum_page(src, page);
1896 if (kWired & pageType)
1897 sum1 += sum;
1898 else
1899 sum2 += sum;
1900
1901 clock_get_uptime(&startTime);
3e170ce0 1902 wkresult = WKdm_compress_new((const WK_word*) src,
39236c6e
A		 1903 (WK_word*) compressed,
1904 (WK_word*) scratch,
1905 page_size - 4);
0b4c1975 1906
3a60a9f5 1907 clock_get_uptime(&endTime);
0b4c1975
A		 1908 ADD_ABSOLUTETIME(&compTime, &endTime);
1909 SUB_ABSOLUTETIME(&compTime, &startTime);
39236c6e 1910
0b4c1975 1911 compBytes += page_size;
39236c6e
A		 1912 pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
1913
3e170ce0 1914 if (pageCompressedSize == 0)
39236c6e 1915 {
3e170ce0 1916 pageCompressedSize = 4;
0b4c1975 1917 data = src;
3e170ce0
A		 1918
1919 if (*(uint32_t *)src)
1920 svPageCount++;
1921 else
1922 zvPageCount++;
1923 }
1924 else
1925 {
1926 if (pageCompressedSize != page_size)
1927 data = compressed;
1928 else
1929 data = src;
1930 }
0b4c1975
A		 1931
1932 tag = pageCompressedSize | kIOHibernateTagSignature;
3e170ce0 1933 err = IOHibernatePolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
0b4c1975
A		 1934 if (kIOReturnSuccess != err)
1935 break;
1936
3e170ce0 1937 err = IOHibernatePolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
0b4c1975
A		 1938 if (kIOReturnSuccess != err)
1939 break;
1940
1941 compressedSize += pageCompressedSize;
39236c6e 1942 uncompressedSize += page_size;
0b4c1975
A		 1943 pagesDone++;
1944
1945 if (vars->consoleMapping && (0 == (1023 & pagesDone)))
1946 {
1947 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
1948 if (blob != lastBlob)
1949 {
1950 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
1951 lastBlob = blob;
1952 }
1953 }
1954 if (0 == (8191 & pagesDone))
1955 {
1956 clock_get_uptime(&endTime);
1957 SUB_ABSOLUTETIME(&endTime, &allTime);
1958 absolutetime_to_nanoseconds(endTime, &nsec);
1959 progressStamp = nsec / 750000000ULL;
1960 if (progressStamp != lastProgressStamp)
1961 {
1962 lastProgressStamp = progressStamp;
1963 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
1964 }
3a60a9f5
A		 1965 }
1966 }
0b4c1975
A		 1967 if (kIOReturnSuccess != err)
1968 break;
1969 ppnum = page;
3a60a9f5 1970 }
0b4c1975 1971
3a60a9f5
A		 1972 if (kIOReturnSuccess != err)
1973 break;
0b4c1975 1974
39236c6e 1975 if ((kEncrypt & pageType) && vars->fileVars->encryptStart)
3a60a9f5 1976 {
7ddcb079 1977 vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
0b4c1975
A		 1978 HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
1979 }
1980
1981 if (kWiredEncrypt != pageType)
1982 {
1983 // end of image1/2 - fill to next block
3e170ce0 1984 err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
3a60a9f5
A		 1985 if (kIOReturnSuccess != err)
1986 break;
0b4c1975
A		 1987 }
1988 if (kWiredClear == pageType)
1989 {
7ddcb079 1990 // enlarge wired image for test
3e170ce0 1991// err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
7ddcb079 1992
0b4c1975
A		 1993 // end wired image
1994 header->encryptStart = vars->fileVars->encryptStart;
1995 header->encryptEnd = vars->fileVars->encryptEnd;
3a60a9f5 1996 image1Size = vars->fileVars->position;
7ddcb079 1997 HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
0b4c1975 1998 image1Size, header->encryptStart, header->encryptEnd);
3a60a9f5
A		 1999 }
2000 }
3a60a9f5 2001 if (kIOReturnSuccess != err)
8a3053a0
A		 2002 {
2003 if (kIOReturnOverrun == err)
2004 {
3e170ce0 2005 // update actual compression ratio on not enough space (for retry)
8a3053a0
A		 2006 gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
2007 }
3e170ce0
A		 2008
2009 // update partial amount written (for IOPolledFileClose cleanup/unmap)
2010 header->imageSize = vars->fileVars->position;
3a60a9f5 2011 break;
8a3053a0 2012 }
3a60a9f5
A		 2013
2014 // Header:
2015
2016 header->imageSize = vars->fileVars->position;
2017 header->image1Size = image1Size;
2018 header->bitmapSize = bitmap_size;
2019 header->pageCount = pageCount;
3a60a9f5
A		 2020
2021 header->restore1Sum = restore1Sum;
2022 header->image1Sum = sum1;
2023 header->image2Sum = sum2;
316670eb 2024 header->sleepTime = gIOLastSleepTime.tv_sec;
db609669
A		 2025
2026 header->compression = (compressedSize << 8) / uncompressedSize;
2027 gIOHibernateCompression = header->compression;
3a60a9f5 2028
3e170ce0 2029 count = vars->fileVars->fileExtents->getLength();
3a60a9f5
A		 2030 if (count > sizeof(header->fileExtentMap))
2031 {
2032 header->fileExtentMapSize = count;
2033 count = sizeof(header->fileExtentMap);
2034 }
2035 else
2036 header->fileExtentMapSize = sizeof(header->fileExtentMap);
2037 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
0b4c1975 2038
39236c6e
A		 2039 header->deviceBase = vars->fileVars->block0;
2040 header->deviceBlockSize = vars->fileVars->blockSize;
6d2010ae 2041
3a60a9f5 2042 IOPolledFileSeek(vars->fileVars, 0);
3e170ce0 2043 err = IOHibernatePolledFileWrite(vars->fileVars,
3a60a9f5
A		 2044 (uint8_t *) header, sizeof(IOHibernateImageHeader),
2045 cryptvars);
2046 if (kIOReturnSuccess != err)
2047 break;
3e170ce0 2048 err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
3a60a9f5
A		 2049 }
2050 while (false);
2051
2052 clock_get_uptime(&endTime);
b0d623f7
A		 2053
2054 IOService::getPMRootDomain()->pmStatsRecordEvent(
2055 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
2056
3a60a9f5
A		 2057 SUB_ABSOLUTETIME(&endTime, &allTime);
2058 absolutetime_to_nanoseconds(endTime, &nsec);
39236c6e 2059 HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
3a60a9f5
A		 2060
2061 absolutetime_to_nanoseconds(compTime, &nsec);
6d2010ae
A		 2062 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2063 compBytes,
2064 nsec / 1000000ULL,
2065 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
3a60a9f5 2066
6d2010ae
A		 2067 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2068 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
2069 vars->fileVars->cryptBytes,
2070 nsec / 1000000ULL,
2071 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
3a60a9f5 2072
db609669
A		 2073 HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
2074 header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
3a60a9f5 2075 uncompressedSize, atop_32(uncompressedSize), compressedSize,
8f6c56a5 2076 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
3a60a9f5
A		 2077 sum1, sum2);
2078
3e170ce0
A		 2079 HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
2080 svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
c910b4d9 2081
3a60a9f5 2082 if (pollerOpen)
ecc0ceb4 2083 IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted );
3a60a9f5 2084
2d21ac55
A		 2085 if (vars->consoleMapping)
2086 ProgressUpdate(gIOHibernateGraphicsInfo,
2087 vars->consoleMapping, 0, kIOHibernateProgressCount);
2088
3a60a9f5
A		 2089 HIBLOG("hibernate_write_image done(%x)\n", err);
2090
2091 // should we come back via regular wake, set the state in memory.
2092 gIOHibernateState = kIOHibernateStateInactive;
2093
5ba3f43e
A		 2094 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, wiredPagesEncrypted,
2095 wiredPagesClear, dirtyPagesEncrypted);
0b4c1975 2096
2d21ac55
A		 2097 if (kIOReturnSuccess == err)
2098 {
2099 if (kIOHibernateModeSleep & gIOHibernateMode)
2100 {
2101 return (kIOHibernatePostWriteSleep);
2102 }
2103 else if(kIOHibernateModeRestart & gIOHibernateMode)
2104 {
2105 return (kIOHibernatePostWriteRestart);
2106 }
2107 else
2108 {
2109 /* by default, power down */
2110 return (kIOHibernatePostWriteHalt);
2111 }
2112 }
2113 else if (kIOReturnAborted == err)
2114 {
2115 return (kIOHibernatePostWriteWake);
2116 }
3a60a9f5 2117 else
3a60a9f5 2118 {
2d21ac55
A		 2119 /* on error, sleep */
2120 return (kIOHibernatePostWriteSleep);
3a60a9f5
A		 2121 }
2122}
2123
2124/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2125
2126extern "C" void
2127hibernate_machine_init(void)
2128{
2129 IOReturn err;
2130 uint32_t sum;
2131 uint32_t pagesDone;
0b4c1975 2132 uint32_t pagesRead = 0;
6d2010ae 2133 AbsoluteTime startTime, compTime;
3a60a9f5 2134 AbsoluteTime allTime, endTime;
39236c6e
A		 2135 AbsoluteTime startIOTime, endIOTime;
2136 uint64_t nsec, nsecIO;
6d2010ae 2137 uint64_t compBytes;
3a60a9f5
A		 2138 uint32_t lastProgressStamp = 0;
2139 uint32_t progressStamp;
3e170ce0 2140 IOPolledFileCryptVars * cryptvars = 0;
3a60a9f5
A		 2141
2142 IOHibernateVars * vars = &gIOHibernateVars;
39236c6e 2143 bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
3a60a9f5 2144
3e170ce0 2145 if (!vars->fileVars || !vars->fileVars->pollers)
3a60a9f5
A		 2146 return;
2147
3a60a9f5
A		 2148 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2149 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2150
3a60a9f5
A		 2151 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
2152 {
2153 HIBLOG("regular wake\n");
2154 return;
2155 }
2156
2157 HIBPRINT("diag %x %x %x %x\n",
2158 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
6d2010ae 2159 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
3a60a9f5 2160
39236c6e
A		 2161#define t40ms(x) (tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)
2162#define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
2163 tStat(booterStart, booterStart);
39037602 2164 gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart;
39236c6e
A		 2165 tStat(booterDuration0, booterTime0);
2166 tStat(booterDuration1, booterTime1);
2167 tStat(booterDuration2, booterTime2);
2168 tStat(booterDuration, booterTime);
2169 tStat(booterConnectDisplayDuration, connectDisplayTime);
2170 tStat(booterSplashDuration, splashTime);
2171 tStat(trampolineDuration, trampolineTime);
2172
2173 gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size;
2174 gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize;
2175 gIOHibernateStats->image1Pages = pagesDone;
2176
5ba3f43e
A		 2177 /* HIBERNATE_stats */
2178 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 14), gIOHibernateStats->smcStart,
2179 gIOHibernateStats->booterStart, gIOHibernateStats->booterDuration,
2180 gIOHibernateStats->trampolineDuration);
2181
39236c6e
A		 2182 HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
2183 gIOHibernateStats->booterStart,
2184 gIOHibernateStats->smcStart,
2185 gIOHibernateStats->booterDuration0,
2186 gIOHibernateStats->booterDuration1,
2187 gIOHibernateStats->booterDuration2,
2188 gIOHibernateStats->booterDuration,
2189 gIOHibernateStats->booterConnectDisplayDuration,
2190 gIOHibernateStats->booterSplashDuration,
2191 gIOHibernateStats->trampolineDuration);
2192
2193 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
2194 gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
2195 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
bd504ef0 2196
39236c6e
A		 2197 if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
2198 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)))
2199 {
0c530ab8 2200 hibernate_page_list_discard(vars->page_list);
39236c6e 2201 }
0c530ab8 2202
6d2010ae
A		 2203 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
2204
2205 if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount)
2206 panic("handoff overflow");
2207
2208 IOHibernateHandoff * handoff;
2209 bool done = false;
2210 bool foundCryptData = false;
2211
2212 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
2213 !done;
2214 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
2215 {
2216// HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
2217 uint8_t * data = &handoff->data[0];
2218 switch (handoff->type)
2219 {
2220 case kIOHibernateHandoffTypeEnd:
2221 done = true;
2222 break;
2223
2224 case kIOHibernateHandoffTypeGraphicsInfo:
39037602
A		 2225 if (handoff->bytecount == sizeof(*gIOHibernateGraphicsInfo))
2226 {
2227 bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
2228 }
6d2010ae
A		 2229 break;
2230
2231 case kIOHibernateHandoffTypeCryptVars:
2232 if (cryptvars)
2233 {
2234 hibernate_cryptwakevars_t *
2235 wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
2236 bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
2237 }
2238 foundCryptData = true;
2239 bzero(data, handoff->bytecount);
2240 break;
2241
2242 case kIOHibernateHandoffTypeMemoryMap:
bd504ef0
A		 2243
2244 clock_get_uptime(&allTime);
2245
6d2010ae
A		 2246 hibernate_newruntime_map(data, handoff->bytecount,
2247 gIOHibernateCurrentHeader->systemTableOffset);
bd504ef0
A		 2248
2249 clock_get_uptime(&endTime);
2250
2251 SUB_ABSOLUTETIME(&endTime, &allTime);
2252 absolutetime_to_nanoseconds(endTime, &nsec);
2253
2254 HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
2255
6d2010ae
A		 2256 break;
2257
2258 case kIOHibernateHandoffTypeDeviceTree:
2259 {
2260// DTEntry chosen = NULL;
2261// HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen));
2262 }
2263 break;
2264
2265 default:
2266 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
2267 break;
2268 }
2269 }
2270 if (cryptvars && !foundCryptData)
2271 panic("hibernate handoff");
2272
39037602 2273 HIBPRINT("video 0x%llx %d %d %d status %x\n",
316670eb
A		 2274 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2275 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
2276
2277 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)
3a60a9f5
A		 2278 {
2279 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2280 * gIOHibernateGraphicsInfo->rowBytes);
22ba694c
A		 2281 if (vars->videoMapSize > vars->videoAllocSize) vars->videoMapSize = 0;
2282 else
2283 {
2284 IOMapPages(kernel_map,
2285 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2286 vars->videoMapSize, kIOMapInhibitCache );
2287 }
3a60a9f5
A		 2288 }
2289
316670eb
A		 2290 if (vars->videoMapSize)
2291 ProgressUpdate(gIOHibernateGraphicsInfo,
2292 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2293
0c530ab8 2294 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
39236c6e
A		 2295 uint8_t * compressed = src + page_size;
2296 uint8_t * scratch = compressed + page_size;
2297 uint32_t decoOffset;
3a60a9f5
A		 2298
2299 clock_get_uptime(&allTime);
6d2010ae
A		 2300 AbsoluteTime_to_scalar(&compTime) = 0;
2301 compBytes = 0;
3a60a9f5 2302
3e170ce0
A		 2303 HIBLOG("IOPolledFilePollersOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2304 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledAfterSleepState, false);
39236c6e
A		 2305 clock_get_uptime(&startIOTime);
2306 endTime = startIOTime;
2307 SUB_ABSOLUTETIME(&endTime, &allTime);
2308 absolutetime_to_nanoseconds(endTime, &nsec);
3e170ce0 2309 HIBLOG("IOPolledFilePollersOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
3a60a9f5 2310
3a60a9f5
A		 2311 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2312
3a60a9f5 2313 // kick off the read ahead
3a60a9f5
A		 2314 vars->fileVars->bufferHalf = 0;
2315 vars->fileVars->bufferLimit = 0;
2316 vars->fileVars->lastRead = 0;
6d2010ae 2317 vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
3a60a9f5 2318 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
6d2010ae
A		 2319 vars->fileVars->cryptBytes = 0;
2320 AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
3a60a9f5 2321
6d2010ae 2322 err = IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
3a60a9f5
A		 2323 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2324 // --
2325
2326 HIBLOG("hibernate_machine_init reading\n");
2327
2328 uint32_t * header = (uint32_t *) src;
2329 sum = 0;
2330
6d2010ae 2331 while (kIOReturnSuccess == err)
3a60a9f5
A		 2332 {
2333 unsigned int count;
2334 unsigned int page;
2335 uint32_t tag;
2336 vm_offset_t ppnum, compressedSize;
2337
36401178
A		 2338 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2339 if (kIOReturnSuccess != err)
2340 break;
3a60a9f5
A		 2341
2342 ppnum = header[0];
2343 count = header[1];
2344
2345// HIBPRINT("(%x, %x)\n", ppnum, count);
2346
2347 if (!count)
2348 break;
2349
2350 for (page = 0; page < count; page++)
2351 {
36401178
A		 2352 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2353 if (kIOReturnSuccess != err)
2354 break;
3a60a9f5
A		 2355
2356 compressedSize = kIOHibernateTagLength & tag;
36401178
A		 2357 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength))
2358 {
2359 err = kIOReturnIPCError;
2360 break;
2361 }
2362
3e170ce0
A		 2363 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2364 if (kIOReturnSuccess != err) break;
2365
2366 if (compressedSize < page_size)
36401178 2367 {
3e170ce0
A		 2368 decoOffset = page_size;
2369 clock_get_uptime(&startTime);
2370
2371 if (compressedSize == 4) {
2372 int i;
2373 uint32_t *s, *d;
2374
2375 s = (uint32_t *)src;
2376 d = (uint32_t *)(uintptr_t)compressed;
2377
2378 for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++)
2379 *d++ = *s;
39236c6e 2380 }
3e170ce0
A		 2381 else
2382 WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, compressedSize);
2383 clock_get_uptime(&endTime);
2384 ADD_ABSOLUTETIME(&compTime, &endTime);
2385 SUB_ABSOLUTETIME(&compTime, &startTime);
2386 compBytes += page_size;
2387 }
2388 else decoOffset = 0;
39236c6e 2389
3e170ce0
A		 2390 sum += hibernate_sum_page((src + decoOffset), ppnum);
2391 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2392 if (err)
2393 {
39236c6e
A		 2394 HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
2395 break;
36401178 2396 }
3a60a9f5
A		 2397
2398 ppnum++;
2399 pagesDone++;
0b4c1975 2400 pagesRead++;
3a60a9f5 2401
3a60a9f5
A		 2402 if (0 == (8191 & pagesDone))
2403 {
2404 clock_get_uptime(&endTime);
2405 SUB_ABSOLUTETIME(&endTime, &allTime);
2406 absolutetime_to_nanoseconds(endTime, &nsec);
2407 progressStamp = nsec / 750000000ULL;
2408 if (progressStamp != lastProgressStamp)
2409 {
2410 lastProgressStamp = progressStamp;
2411 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2412 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2413 }
2414 }
2415 }
2416 }
7ddcb079
A		 2417 if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages))
2418 err = kIOReturnLockedRead;
3a60a9f5 2419
36401178
A		 2420 if (kIOReturnSuccess != err)
2421 panic("Hibernate restore error %x", err);
2422
3a60a9f5 2423 gIOHibernateCurrentHeader->actualImage2Sum = sum;
db609669 2424 gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
3a60a9f5 2425
39236c6e
A		 2426 clock_get_uptime(&endIOTime);
2427
3e170ce0 2428 err = IOPolledFilePollersClose(vars->fileVars, kIOPolledAfterSleepState);
3a60a9f5 2429
3a60a9f5 2430 clock_get_uptime(&endTime);
b0d623f7
A		 2431
2432 IOService::getPMRootDomain()->pmStatsRecordEvent(
2433 kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
2434 IOService::getPMRootDomain()->pmStatsRecordEvent(
2435 kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
2436
3a60a9f5
A		 2437 SUB_ABSOLUTETIME(&endTime, &allTime);
2438 absolutetime_to_nanoseconds(endTime, &nsec);
2439
39236c6e
A		 2440 SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
2441 absolutetime_to_nanoseconds(endIOTime, &nsecIO);
2442
2443 gIOHibernateStats->kernelImageReadDuration = nsec / 1000000ULL;
2444 gIOHibernateStats->imagePages = pagesDone;
2445
2446 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
2447 pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
2448 nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
6d2010ae
A		 2449
2450 absolutetime_to_nanoseconds(compTime, &nsec);
2451 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2452 compBytes,
2453 nsec / 1000000ULL,
2454 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2455
2456 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2457 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
2458 vars->fileVars->cryptBytes,
2459 nsec / 1000000ULL,
2460 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2461
5ba3f43e 2462 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 2), pagesRead, pagesDone);
3a60a9f5
A		 2463}
2464
2465/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
99c3a104 2466
39236c6e
A		 2467void IOHibernateSetWakeCapabilities(uint32_t capability)
2468{
2469 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
2470 {
2471 gIOHibernateStats->wakeCapability = capability;
8a3053a0
A		 2472
2473 if (kIOPMSystemCapabilityGraphics & capability)
2474 {
2475 vm_compressor_do_warmup();
2476 }
39236c6e
A		 2477 }
2478}
2479
2480/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2481
99c3a104
A		 2482void IOHibernateSystemRestart(void)
2483{
2484 static uint8_t noteStore[32] __attribute__((aligned(32)));
2485 IORegistryEntry * regEntry;
2486 const OSSymbol * sym;
2487 OSData * noteProp;
2488 OSData * data;
2489 uintptr_t * smcVars;
2490 uint8_t * smcBytes;
2491 size_t len;
2492 addr64_t element;
2493
2494 data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
2495 if (!data) return;
2496
2497 smcVars = (typeof(smcVars)) data->getBytesNoCopy();
2498 smcBytes = (typeof(smcBytes)) smcVars[1];
2499 len = smcVars[0];
2500 if (len > sizeof(noteStore)) len = sizeof(noteStore);
2501 noteProp = OSData::withCapacity(3 * sizeof(element));
2502 if (!noteProp) return;
2503 element = len;
2504 noteProp->appendBytes(&element, sizeof(element));
2505 element = crc32(0, smcBytes, len);
2506 noteProp->appendBytes(&element, sizeof(element));
2507
2508 bcopy(smcBytes, noteStore, len);
2509 element = (addr64_t) &noteStore[0];
2510 element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
2511 noteProp->appendBytes(&element, sizeof(element));
2512
2513 if (!gIOOptionsEntry)
2514 {
2515 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
2516 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
2517 if (regEntry && !gIOOptionsEntry)
2518 regEntry->release();
2519 }
2520
2521 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
2522 if (gIOOptionsEntry && sym) gIOOptionsEntry->setProperty(sym, noteProp);
2523 if (noteProp) noteProp->release();
2524 if (sym) sym->release();
2525}
2526
2527
2528
mirror of XNU