]> git.saurik.com Git - apple/xnu.git/blob - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blob
? search:


3303016d05a404c9336861e8e49e04d742a1a8ae
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
1 /*
2 * Copyright (c) 2004-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30 *
31 * Sleep:
32 *
33 * - PMRootDomain calls IOHibernateSystemSleep() before system sleep
34 * (devices awake, normal execution context)
35 * - IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
36 * grabs its extents and searches for a polling driver willing to work with that IOMedia.
37 * The BSD code makes an ioctl to the storage driver to get the partition base offset to
38 * the disk, and other ioctls to get the transfer constraints
39 * If successful, the file is written to make sure its initially not bootable (in case of
40 * later failure) and nvram set to point to the first block of the file. (Has to be done
41 * here so blocking is possible in nvram support).
42 * hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
43 * page out any pages it wants to (currently zero, but probably some percentage of memory).
44 * Its assumed just allocating pages will cause the VM system to naturally select the best
45 * pages for eviction. It also copies processor flags needed for the restore path and sets
46 * a flag in the boot processor proc info.
47 * gIOHibernateState = kIOHibernateStateHibernating.
48 * - Regular sleep progresses - some drivers may inspect the root domain property
49 * kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
50 * as usual but leaves motherboard I/O on.
51 * - Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
52 * at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
53 * all ppc RC bits out of the hash table and caches into the mapping structures.
54 * - hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
55 * hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
56 * All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
57 * by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
58 * The image header and block list are written. The header includes the second file extent so
59 * only the header block is needed to read the file, regardless of filesystem.
60 * The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
61 * (only) is used to decompress the image during wake/boot.
62 * Some additional pages are removed from the bitmaps - the buffers used for hibernation.
63 * The bitmaps are written to the image.
64 * More areas are removed from the bitmaps (after they have been written to the image) - the
65 * segment "__HIB" pages and interrupt stack.
66 * Each wired page is compressed and written and then each non-wired page. Compression and
67 * disk writes are in parallel.
68 * The image header is written to the start of the file and the polling driver closed.
69 * The machine powers down (or sleeps).
70 *
71 * Boot/Wake:
72 *
73 * - BootX sees the boot-image nvram variable containing the device and block number of the image,
74 * reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
75 * - BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
76 * in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
77 * that is in the kernel's __HIB section.
78 * - BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
79 * hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
80 * only code & data in that section is safe to call since all the other wired pages are still
81 * compressed in the image.
82 * - hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
83 * It uses the bitmaps to work out which pages can be uncompressed from the image to their final
84 * location directly, and copies those that can't to interim free pages. When the image has been
85 * completed, the copies are uncompressed, overwriting the wired image pages.
86 * hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
87 * is used to get pages into place for 64bit.
88 * - the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
89 * changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
90 * are removed from the software strutures, and the hash table is reinitialized.
91 * - After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
92 * of memory, using the polled mode driver, before other threads can run or any devices are turned on.
93 * This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
94 * for the remaining non wired pages.
95 * - The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
96 * (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
97 * is closed via bsd.
98 *
99 * Polled Mode I/O:
100 *
101 * IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
102 * registry, specifying an object of calls IOPolledInterface.
103 *
104 * Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
105 * partition) that the image is going to live, looking for polled interface properties. If it finds
106 * one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
107 * interfaces found are kept in an ordered list.
108 *
109 * There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
110 * few different contexts things happen in:
111 *
112 * - there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
113 * up and running) and after wake - this is safe to allocate memory and do anything. The device
114 * ignores sleep requests from that point since its a waste of time if it goes to sleep and
115 * immediately wakes back up for the image write.
116 *
117 * - there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
118 * immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
119 * from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
120 * used to flush and set the disk to sleep.
121 *
122 * - there is an Open/Close (AfterSleep) pair made around the image read operations that happen
123 * immediately after sleep. These can't block or allocate memory. This is happening after the platform
124 * expert has woken the low level bits of the system, but most of the I/O system has not. There is only
125 * one thread running.
126 *
127 * For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
128 * (prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
129 * the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
130 * Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
131 * that is called for the hardware to check for events, and complete the I/O via the callback.
132 * The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
133 * to restrict I/O ops.
134 */
135
136 #include <sys/systm.h>
137
138 #include <IOKit/IOWorkLoop.h>
139 #include <IOKit/IOCommandGate.h>
140 #include <IOKit/IOTimerEventSource.h>
141 #include <IOKit/IOPlatformExpert.h>
142 #include <IOKit/IOKitDebug.h>
143 #include <IOKit/IOTimeStamp.h>
144 #include <IOKit/pwr_mgt/RootDomain.h>
145 #include <IOKit/pwr_mgt/IOPMPrivate.h>
146 #include <IOKit/IOMessage.h>
147 #include <IOKit/IODeviceTreeSupport.h>
148 #include <IOKit/IOBSD.h>
149 #include <IOKit/IOKitKeysPrivate.h>
150 #include "RootDomainUserClient.h"
151 #include <IOKit/pwr_mgt/IOPowerConnection.h>
152 #include "IOPMPowerStateQueue.h"
153 #include <IOKit/IOBufferMemoryDescriptor.h>
154 #include <IOKit/AppleKeyStoreInterface.h>
155 #include <libkern/crypto/aes.h>
156
157 #include <sys/uio.h>
158 #include <sys/conf.h>
159 #include <sys/stat.h>
160 #include <sys/fcntl.h> // (FWRITE, ...)
161 #include <sys/sysctl.h>
162 #include <sys/kdebug.h>
163 #include <stdint.h>
164
165 #include <IOKit/IOHibernatePrivate.h>
166 #include <IOKit/IOPolledInterface.h>
167 #include <IOKit/IONVRAM.h>
168 #include "IOHibernateInternal.h"
169 #include <vm/vm_protos.h>
170 #include "IOKitKernelInternal.h"
171 #include <pexpert/device_tree.h>
172
173 #include <machine/pal_routines.h>
174 #include <machine/pal_hibernate.h>
175 #if defined(__i386__) || defined(__x86_64__)
176 #include <i386/tsc.h>
177 #include <i386/cpuid.h>
178 #include <vm/WKdm_new.h>
179 #elif defined(__arm64__)
180 #include <arm64/amcc_rorgn.h>
181 #endif /* defined(__i386__) || defined(__x86_64__) */
182 #include <san/kasan.h>
183
184
185 extern "C" addr64_t kvtophys(vm_offset_t va);
186 extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
187
188 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
189
190 #define DISABLE_TRIM 0
191 #define TRIM_DELAY 25000
192
193 extern unsigned int save_kdebug_enable;
194 extern uint32_t gIOHibernateState;
195 uint32_t gIOHibernateMode;
196 static char gIOHibernateBootSignature[256 + 1];
197 static char gIOHibernateFilename[MAXPATHLEN + 1];
198 uint32_t gIOHibernateCount;
199
200 static uuid_string_t gIOHibernateBridgeBootSessionUUIDString;
201
202 static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
203 uint32_t gIOHibernateFreeTime = 0 * 1000; // max time to spend freeing pages (ms)
204
205 enum {
206 HIB_COMPR_RATIO_ARM64 = (0xa5), // compression ~65%. Since we don't support retries we start higher.
207 HIB_COMPR_RATIO_INTEL = (0x80) // compression 50%
208 };
209
210 #if defined(__arm64__)
211 static uint64_t gIOHibernateCompression = HIB_COMPR_RATIO_ARM64;
212 #else
213 static uint64_t gIOHibernateCompression = HIB_COMPR_RATIO_INTEL;
214 #endif /* __arm64__ */
215 boolean_t gIOHibernateStandbyDisabled;
216
217 static IODTNVRAM * gIOOptionsEntry;
218 static IORegistryEntry * gIOChosenEntry;
219
220 static const OSSymbol * gIOHibernateBootImageKey;
221 static const OSSymbol * gIOHibernateBootSignatureKey;
222 static const OSSymbol * gIOBridgeBootSessionUUIDKey;
223
224 #if defined(__i386__) || defined(__x86_64__)
225
226 static const OSSymbol * gIOHibernateRTCVariablesKey;
227 static const OSSymbol * gIOHibernateBoot0082Key;
228 static const OSSymbol * gIOHibernateBootNextKey;
229 static OSData * gIOHibernateBoot0082Data;
230 static OSData * gIOHibernateBootNextData;
231 static OSObject * gIOHibernateBootNextSave;
232
233 #endif /* defined(__i386__) || defined(__x86_64__) */
234
235 static IOLock * gFSLock;
236 uint32_t gFSState;
237 static thread_call_t gIOHibernateTrimCalloutEntry;
238 static IOPolledFileIOVars gFileVars;
239 static IOHibernateVars gIOHibernateVars;
240 static IOPolledFileCryptVars gIOHibernateCryptWakeContext;
241 static hibernate_graphics_t _hibernateGraphics;
242 static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
243 static hibernate_statistics_t _hibernateStats;
244 static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats;
245
246 enum{
247 kFSIdle = 0,
248 kFSOpening = 2,
249 kFSOpened = 3,
250 kFSTimedOut = 4,
251 kFSTrimDelay = 5
252 };
253
254 static IOReturn IOHibernateDone(IOHibernateVars * vars);
255 static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature);
256 static void IOSetBootImageNVRAM(OSData * data);
257 static void IOHibernateSystemPostWakeTrim(void * p1, void * p2);
258
259 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
260
261 enum { kDefaultIOSize = 128 * 1024 };
262 enum { kVideoMapSize = 80 * 1024 * 1024 };
263
264 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
265
266 // copy from phys addr to MD
267
268 static IOReturn
269 IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
270 IOByteCount offset, addr64_t bytes, IOByteCount length)
271 {
272 addr64_t srcAddr = bytes;
273 IOByteCount remaining;
274
275 remaining = length = min(length, md->getLength() - offset);
276 while (remaining) { // (process another target segment?)
277 addr64_t dstAddr64;
278 IOByteCount dstLen;
279
280 dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
281 if (!dstAddr64) {
282 break;
283 }
284
285 // Clip segment length to remaining
286 if (dstLen > remaining) {
287 dstLen = remaining;
288 }
289
290 #if 1
291 bcopy_phys(srcAddr, dstAddr64, dstLen);
292 #else
293 copypv(srcAddr, dstAddr64, dstLen,
294 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
295 #endif
296 srcAddr += dstLen;
297 offset += dstLen;
298 remaining -= dstLen;
299 }
300
301 assert(!remaining);
302
303 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
304 }
305
306 // copy from MD to phys addr
307
308 static IOReturn
309 IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
310 IOByteCount offset, addr64_t bytes, IOByteCount length)
311 {
312 addr64_t dstAddr = bytes;
313 IOByteCount remaining;
314
315 remaining = length = min(length, md->getLength() - offset);
316 while (remaining) { // (process another target segment?)
317 addr64_t srcAddr64;
318 IOByteCount dstLen;
319
320 srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
321 if (!srcAddr64) {
322 break;
323 }
324
325 // Clip segment length to remaining
326 if (dstLen > remaining) {
327 dstLen = remaining;
328 }
329
330 #if 1
331 bcopy_phys(srcAddr64, dstAddr, dstLen);
332 #else
333 copypv(srcAddr, dstAddr64, dstLen,
334 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
335 #endif
336 dstAddr += dstLen;
337 offset += dstLen;
338 remaining -= dstLen;
339 }
340
341 assert(!remaining);
342
343 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
344 }
345
346 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
347
348 void
349 hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
350 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
351 {
352 count += ppnum;
353
354 if (count > UINT_MAX) {
355 panic("hibernate_set_page_state ppnum");
356 }
357
358 switch (kind) {
359 case kIOHibernatePageStateUnwiredSave:
360 // unwired save
361 for (; ppnum < count; ppnum++) {
362 hibernate_page_bitset(page_list, FALSE, (uint32_t) ppnum);
363 hibernate_page_bitset(page_list_wired, TRUE, (uint32_t) ppnum);
364 }
365 break;
366 case kIOHibernatePageStateWiredSave:
367 // wired save
368 for (; ppnum < count; ppnum++) {
369 hibernate_page_bitset(page_list, FALSE, (uint32_t) ppnum);
370 hibernate_page_bitset(page_list_wired, FALSE, (uint32_t) ppnum);
371 }
372 break;
373 case kIOHibernatePageStateFree:
374 // free page
375 for (; ppnum < count; ppnum++) {
376 hibernate_page_bitset(page_list, TRUE, (uint32_t) ppnum);
377 hibernate_page_bitset(page_list_wired, TRUE, (uint32_t) ppnum);
378 }
379 break;
380 default:
381 panic("hibernate_set_page_state");
382 }
383 }
384
385 static void
386 hibernate_set_descriptor_page_state(IOHibernateVars *vars,
387 IOMemoryDescriptor *descriptor,
388 uint32_t kind,
389 uint32_t *pageCount)
390 {
391 IOItemCount count;
392 addr64_t phys64;
393 IOByteCount segLen;
394 if (descriptor) {
395 for (count = 0;
396 (phys64 = descriptor->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
397 count += segLen) {
398 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
399 atop_64(phys64), atop_32(segLen),
400 kind);
401 *pageCount -= atop_32(segLen);
402 }
403 }
404 }
405
406 static vm_offset_t
407 hibernate_page_list_iterate(hibernate_page_list_t * list, ppnum_t * pPage)
408 {
409 uint32_t page = ((typeof(page)) * pPage);
410 uint32_t count;
411 hibernate_bitmap_t * bitmap;
412
413 while ((bitmap = hibernate_page_bitmap_pin(list, &page))) {
414 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
415 if (!count) {
416 break;
417 }
418 page += count;
419 if (page <= bitmap->last_page) {
420 break;
421 }
422 }
423
424 *pPage = page;
425 if (bitmap) {
426 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
427 } else {
428 count = 0;
429 }
430
431 return count;
432 }
433
434 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
435
436 IOReturn
437 IOHibernateSystemSleep(void)
438 {
439 IOReturn err;
440 OSData * nvramData;
441 OSObject * obj;
442 OSString * str;
443 OSNumber * num;
444 bool dsSSD, vmflush, swapPinned;
445 IOHibernateVars * vars;
446 uint64_t setFileSize = 0;
447
448 gIOHibernateState = kIOHibernateStateInactive;
449
450 gIOHibernateDebugFlags = 0;
451 if (kIOLogHibernate & gIOKitDebug) {
452 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
453 }
454
455 if (IOService::getPMRootDomain()->getHibernateSettings(
456 &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime)) {
457 if (kIOHibernateModeSleep & gIOHibernateMode) {
458 // default to discard clean for safe sleep
459 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
460 | kIOHibernateModeDiscardCleanActive);
461 }
462 }
463
464 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey))) {
465 if ((str = OSDynamicCast(OSString, obj))) {
466 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
467 sizeof(gIOHibernateFilename));
468 }
469 obj->release();
470 }
471
472 if (!gIOHibernateMode || !gIOHibernateFilename[0]) {
473 return kIOReturnUnsupported;
474 }
475
476 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
477
478 vars = IONew(IOHibernateVars, 1);
479 if (!vars) {
480 return kIOReturnNoMemory;
481 }
482 bzero(vars, sizeof(*vars));
483
484 IOLockLock(gFSLock);
485 if (!gIOHibernateTrimCalloutEntry) {
486 gIOHibernateTrimCalloutEntry = thread_call_allocate(&IOHibernateSystemPostWakeTrim, &gFSLock);
487 }
488 IOHibernateSystemPostWakeTrim(NULL, NULL);
489 thread_call_cancel(gIOHibernateTrimCalloutEntry);
490 if (kFSIdle != gFSState) {
491 HIBLOG("hibernate file busy\n");
492 IOLockUnlock(gFSLock);
493 IODelete(vars, IOHibernateVars, 1);
494 return kIOReturnBusy;
495 }
496 gFSState = kFSOpening;
497 IOLockUnlock(gFSLock);
498
499 swapPinned = false;
500 do{
501 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
502 HIBERNATION_SRC_BUFFER_SIZE, page_size);
503
504 vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
505 ptoa_64(gIOHibernateHandoffPageCount), page_size);
506
507 if (!vars->srcBuffer || !vars->handoffBuffer) {
508 err = kIOReturnNoMemory;
509 break;
510 }
511
512 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey))) {
513 if ((num = OSDynamicCast(OSNumber, obj))) {
514 vars->fileMinSize = num->unsigned64BitValue();
515 }
516 obj->release();
517 }
518 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey))) {
519 if ((num = OSDynamicCast(OSNumber, obj))) {
520 vars->fileMaxSize = num->unsigned64BitValue();
521 }
522 obj->release();
523 }
524
525 boolean_t encryptedswap = true;
526 uint32_t pageCount;
527 AbsoluteTime startTime, endTime;
528 uint64_t nsec;
529
530 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
531 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
532 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
533
534 vmflush = ((kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
535 err = hibernate_alloc_page_lists(&vars->page_list,
536 &vars->page_list_wired,
537 &vars->page_list_pal);
538 if (KERN_SUCCESS != err) {
539 HIBLOG("%s err, hibernate_alloc_page_lists return 0x%x\n", __FUNCTION__, err);
540 break;
541 }
542
543 err = hibernate_pin_swap(TRUE);
544 if (KERN_SUCCESS != err) {
545 HIBLOG("%s error, hibernate_pin_swap return 0x%x\n", __FUNCTION__, err);
546 break;
547 }
548 swapPinned = true;
549
550 if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode)) {
551 hibernate_page_list_setall(vars->page_list,
552 vars->page_list_wired,
553 vars->page_list_pal,
554 true /* preflight */,
555 vmflush /* discard */,
556 &pageCount);
557 PE_Video consoleInfo;
558 bzero(&consoleInfo, sizeof(consoleInfo));
559 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
560
561 // estimate: 6% increase in pages compressed
562 // screen preview 2 images compressed 0%
563 setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
564 + vars->page_list->list_size
565 + (consoleInfo.v_width * consoleInfo.v_height * 8);
566 enum { setFileRound = 1024 * 1024ULL };
567 setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
568
569 HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
570 pageCount, (100ULL * gIOHibernateCompression) >> 8,
571 setFileSize, vars->fileMinSize);
572
573 if (!(kIOHibernateModeFileResize & gIOHibernateMode)
574 && (setFileSize < vars->fileMinSize)) {
575 setFileSize = vars->fileMinSize;
576 }
577 }
578
579 vars->volumeCryptKeySize = sizeof(vars->volumeCryptKey);
580 err = IOPolledFileOpen(gIOHibernateFilename,
581 (kIOPolledFileCreate | kIOPolledFileHibernate),
582 setFileSize, 0,
583 gIOHibernateCurrentHeader, sizeof(gIOHibernateCurrentHeader),
584 &vars->fileVars, &nvramData,
585 &vars->volumeCryptKey[0], &vars->volumeCryptKeySize);
586
587 if (KERN_SUCCESS != err) {
588 IOLockLock(gFSLock);
589 if (kFSOpening != gFSState) {
590 err = kIOReturnTimeout;
591 }
592 IOLockUnlock(gFSLock);
593 }
594
595 if (KERN_SUCCESS != err) {
596 HIBLOG("IOPolledFileOpen(%x)\n", err);
597 break;
598 }
599
600 // write extents for debug data usage in EFI
601 IOWriteExtentsToFile(vars->fileVars, kIOHibernateHeaderOpenSignature);
602
603 err = IOPolledFilePollersSetup(vars->fileVars, kIOPolledPreflightState);
604 if (KERN_SUCCESS != err) {
605 break;
606 }
607
608 clock_get_uptime(&startTime);
609 err = hibernate_setup(gIOHibernateCurrentHeader,
610 vmflush,
611 vars->page_list, vars->page_list_wired, vars->page_list_pal);
612 clock_get_uptime(&endTime);
613 SUB_ABSOLUTETIME(&endTime, &startTime);
614 absolutetime_to_nanoseconds(endTime, &nsec);
615
616 boolean_t haveSwapPin, hibFileSSD;
617 haveSwapPin = vm_swap_files_pinned();
618
619 hibFileSSD = (kIOPolledFileSSD & vars->fileVars->flags);
620
621 HIBLOG("hibernate_setup(%d) took %qd ms, swapPin(%d) ssd(%d)\n",
622 err, nsec / 1000000ULL,
623 haveSwapPin, hibFileSSD);
624 if (KERN_SUCCESS != err) {
625 break;
626 }
627
628 gIOHibernateStandbyDisabled = ((!haveSwapPin || !hibFileSSD));
629
630 dsSSD = ((0 != (kIOPolledFileSSD & vars->fileVars->flags))
631 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
632
633 if (dsSSD) {
634 gIOHibernateCurrentHeader->options |= kIOHibernateOptionSSD | kIOHibernateOptionColor;
635 } else {
636 gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
637 }
638
639
640 #if defined(__i386__) || defined(__x86_64__)
641 if (vars->volumeCryptKeySize &&
642 (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey))) {
643 uintptr_t smcVars[2];
644 smcVars[0] = vars->volumeCryptKeySize;
645 smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
646
647 IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
648 bzero(smcVars, sizeof(smcVars));
649 }
650 #endif
651
652
653 if (encryptedswap || vars->volumeCryptKeySize) {
654 gIOHibernateMode ^= kIOHibernateModeEncrypt;
655 }
656
657 if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options) {
658 vars->videoAllocSize = kVideoMapSize;
659 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize, VM_KERN_MEMORY_IOKIT)) {
660 vars->videoMapping = 0;
661 }
662 }
663
664 // generate crypt keys
665 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++) {
666 vars->wiredCryptKey[i] = ((uint8_t) random());
667 }
668 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++) {
669 vars->cryptKey[i] = ((uint8_t) random());
670 }
671
672 // set nvram
673
674 IOSetBootImageNVRAM(nvramData);
675 nvramData->release();
676
677 #if defined(__i386__) || defined(__x86_64__)
678 {
679 struct AppleRTCHibernateVars {
680 uint8_t signature[4];
681 uint32_t revision;
682 uint8_t booterSignature[20];
683 uint8_t wiredCryptKey[16];
684 };
685 AppleRTCHibernateVars rtcVars;
686 OSData * data;
687
688 rtcVars.signature[0] = 'A';
689 rtcVars.signature[1] = 'A';
690 rtcVars.signature[2] = 'P';
691 rtcVars.signature[3] = 'L';
692 rtcVars.revision = 1;
693 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
694
695 if (gIOChosenEntry
696 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOHibernateBootSignatureKey)))
697 && (sizeof(rtcVars.booterSignature) <= data->getLength())) {
698 bcopy(data->getBytesNoCopy(), &rtcVars.booterSignature[0], sizeof(rtcVars.booterSignature));
699 } else if (gIOHibernateBootSignature[0]) {
700 char c;
701 uint8_t value = 0;
702 uint32_t in, out, digits;
703 for (in = out = digits = 0;
704 (c = gIOHibernateBootSignature[in]) && (in < sizeof(gIOHibernateBootSignature));
705 in++) {
706 if ((c >= 'a') && (c <= 'f')) {
707 c -= 'a' - 10;
708 } else if ((c >= 'A') && (c <= 'F')) {
709 c -= 'A' - 10;
710 } else if ((c >= '0') && (c <= '9')) {
711 c -= '0';
712 } else {
713 if (c == '=') {
714 out = digits = value = 0;
715 }
716 continue;
717 }
718 value = ((uint8_t) ((value << 4) | c));
719 if (digits & 1) {
720 rtcVars.booterSignature[out++] = value;
721 if (out >= sizeof(rtcVars.booterSignature)) {
722 break;
723 }
724 }
725 digits++;
726 }
727 }
728 #if DEBUG || DEVELOPMENT
729 if (kIOLogHibernate & gIOKitDebug) {
730 IOKitKernelLogBuffer("H> rtc:",
731 &rtcVars, sizeof(rtcVars), &kprintf);
732 }
733 #endif /* DEBUG || DEVELOPMENT */
734
735 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
736 if (data) {
737 if (gIOHibernateRTCVariablesKey) {
738 IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
739 }
740 data->release();
741 }
742 if (gIOChosenEntry && gIOOptionsEntry) {
743 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
744 if (data) {
745 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
746 }
747 // set BootNext
748 if (!gIOHibernateBoot0082Data) {
749 OSData * fileData = NULL;
750 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
751 if (data && data->getLength() >= 4) {
752 fileData = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-file-path"));
753 }
754 if (data && (data->getLength() <= UINT16_MAX)) {
755 // AppleNVRAM_EFI_LOAD_OPTION
756 struct {
757 uint32_t Attributes;
758 uint16_t FilePathLength;
759 uint16_t Desc;
760 } loadOptionHeader;
761 loadOptionHeader.Attributes = 1;
762 loadOptionHeader.FilePathLength = ((uint16_t) data->getLength());
763 loadOptionHeader.Desc = 0;
764 if (fileData) {
765 loadOptionHeader.FilePathLength -= 4;
766 loadOptionHeader.FilePathLength += fileData->getLength();
767 }
768 gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
769 if (gIOHibernateBoot0082Data) {
770 gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
771 if (fileData) {
772 gIOHibernateBoot0082Data->appendBytes(data->getBytesNoCopy(), data->getLength() - 4);
773 gIOHibernateBoot0082Data->appendBytes(fileData);
774 } else {
775 gIOHibernateBoot0082Data->appendBytes(data);
776 }
777 }
778 }
779 }
780 if (!gIOHibernateBootNextData) {
781 uint16_t bits = 0x0082;
782 gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
783 }
784
785 #if DEBUG || DEVELOPMENT
786 if (kIOLogHibernate & gIOKitDebug) {
787 IOKitKernelLogBuffer("H> bootnext:",
788 gIOHibernateBoot0082Data->getBytesNoCopy(), gIOHibernateBoot0082Data->getLength(), &kprintf);
789 }
790 #endif /* DEBUG || DEVELOPMENT */
791 if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData) {
792 gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
793 gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
794 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
795 }
796 // BootNext
797 }
798 }
799 #endif /* !i386 && !x86_64 */
800 }while (false);
801
802 if (swapPinned) {
803 hibernate_pin_swap(FALSE);
804 }
805
806 IOLockLock(gFSLock);
807 if ((kIOReturnSuccess == err) && (kFSOpening != gFSState)) {
808 HIBLOG("hibernate file close due timeout\n");
809 err = kIOReturnTimeout;
810 }
811 if (kIOReturnSuccess == err) {
812 gFSState = kFSOpened;
813 gIOHibernateVars = *vars;
814 gFileVars = *vars->fileVars;
815 gFileVars.allocated = false;
816 gIOHibernateVars.fileVars = &gFileVars;
817 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
818 gIOHibernateCurrentHeader->kernVirtSlide = vm_kernel_slide;
819 gIOHibernateState = kIOHibernateStateHibernating;
820
821 #if DEBUG || DEVELOPMENT
822 if (kIOLogHibernate & gIOKitDebug) {
823 OSData * data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
824 if (data) {
825 uintptr_t * smcVars = (typeof(smcVars))data->getBytesNoCopy();
826 IOKitKernelLogBuffer("H> smc:",
827 (const void *)smcVars[1], smcVars[0], &kprintf);
828 }
829 }
830 #endif /* DEBUG || DEVELOPMENT */
831 } else {
832 IOPolledFileIOVars * fileVars = vars->fileVars;
833 IOHibernateDone(vars);
834 IOPolledFileClose(&fileVars,
835 #if DISABLE_TRIM
836 0, NULL, 0, 0, 0);
837 #else
838 0, NULL, 0, sizeof(IOHibernateImageHeader), setFileSize);
839 #endif
840 gFSState = kFSIdle;
841 }
842 IOLockUnlock(gFSLock);
843
844 if (vars->fileVars) {
845 IODelete(vars->fileVars, IOPolledFileIOVars, 1);
846 }
847 IODelete(vars, IOHibernateVars, 1);
848
849 return err;
850 }
851
852 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
853
854 static void
855 IOSetBootImageNVRAM(OSData * data)
856 {
857 IORegistryEntry * regEntry;
858
859 if (!gIOOptionsEntry) {
860 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
861 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
862 if (regEntry && !gIOOptionsEntry) {
863 regEntry->release();
864 }
865 }
866 if (gIOOptionsEntry && gIOHibernateBootImageKey) {
867 if (data) {
868 gIOOptionsEntry->setProperty(gIOHibernateBootImageKey, data);
869 #if DEBUG || DEVELOPMENT
870 if (kIOLogHibernate & gIOKitDebug) {
871 IOKitKernelLogBuffer("H> boot-image:",
872 data->getBytesNoCopy(), data->getLength(), &kprintf);
873 }
874 #endif /* DEBUG || DEVELOPMENT */
875 } else {
876 gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
877 #if __x86_64__
878 gIOOptionsEntry->sync();
879 #else
880 if (gIOHibernateState == kIOHibernateStateWakingFromHibernate) {
881 // if we woke from hibernation, the booter may have changed the state of NVRAM, so force a sync
882 gIOOptionsEntry->sync();
883 }
884 #endif
885 }
886 }
887 }
888
889 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
890 /*
891 * Writes header to disk with signature, block size and file extents data.
892 * If there are more than 2 extents, then they are written on second block.
893 */
894 static IOReturn
895 IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature)
896 {
897 IOHibernateImageHeader hdr;
898 IOItemCount count;
899 IOReturn err = kIOReturnSuccess;
900 int rc;
901 IOPolledFileExtent * fileExtents;
902
903 fileExtents = (typeof(fileExtents))vars->fileExtents->getBytesNoCopy();
904
905 memset(&hdr, 0, sizeof(IOHibernateImageHeader));
906 count = vars->fileExtents->getLength();
907 if (count > sizeof(hdr.fileExtentMap)) {
908 hdr.fileExtentMapSize = count;
909 count = sizeof(hdr.fileExtentMap);
910 } else {
911 hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap);
912 }
913
914 bcopy(fileExtents, &hdr.fileExtentMap[0], count);
915
916 // copy file block extent list if larger than header
917 if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap)) {
918 count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap);
919 rc = kern_write_file(vars->fileRef, vars->blockSize,
920 (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)),
921 count, IO_SKIP_ENCRYPTION);
922 if (rc != 0) {
923 HIBLOG("kern_write_file returned %d\n", rc);
924 err = kIOReturnIOError;
925 goto exit;
926 }
927 }
928 hdr.signature = signature;
929 hdr.deviceBlockSize = vars->blockSize;
930
931 rc = kern_write_file(vars->fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION);
932 if (rc != 0) {
933 HIBLOG("kern_write_file returned %d\n", rc);
934 err = kIOReturnIOError;
935 goto exit;
936 }
937
938 exit:
939 return err;
940 }
941
942 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
943
944 DECLARE_IOHIBERNATEPROGRESSALPHA
945
946 static void
947 ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
948 {
949 uint32_t rowBytes, pixelShift;
950 uint32_t x, y;
951 int32_t blob;
952 uint32_t alpha, color, result;
953 uint8_t * out, in;
954 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
955
956 rowBytes = display->rowBytes;
957 pixelShift = display->depth >> 4;
958 if (pixelShift < 1) {
959 return;
960 }
961
962 screen += ((display->width
963 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
964 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
965
966 for (y = 0; y < kIOHibernateProgressHeight; y++) {
967 out = screen + y * rowBytes;
968 for (blob = 0; blob < kIOHibernateProgressCount; blob++) {
969 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
970 for (x = 0; x < kIOHibernateProgressWidth; x++) {
971 alpha = gIOHibernateProgressAlpha[y][x];
972 result = color;
973 if (alpha) {
974 if (0xff != alpha) {
975 if (1 == pixelShift) {
976 in = *((uint16_t *)out) & 0x1f; // 16
977 in = ((uint8_t)(in << 3)) | ((uint8_t)(in >> 2));
978 } else {
979 in = *((uint32_t *)out) & 0xff; // 32
980 }
981 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
982 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
983 }
984 if (1 == pixelShift) {
985 result >>= 3;
986 *((uint16_t *)out) = ((uint16_t)((result << 10) | (result << 5) | result)); // 16
987 } else {
988 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
989 }
990 }
991 out += (1 << pixelShift);
992 }
993 out += (kIOHibernateProgressSpacing << pixelShift);
994 }
995 }
996 }
997
998
999 static void
1000 ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
1001 {
1002 uint32_t rowBytes, pixelShift;
1003 uint32_t x, y;
1004 int32_t blob, lastBlob;
1005 uint32_t alpha, in, color, result;
1006 uint8_t * out;
1007 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1008
1009 pixelShift = display->depth >> 4;
1010 if (pixelShift < 1) {
1011 return;
1012 }
1013
1014 rowBytes = display->rowBytes;
1015
1016 screen += ((display->width
1017 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1018 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1019
1020 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
1021
1022 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
1023
1024 for (y = 0; y < kIOHibernateProgressHeight; y++) {
1025 out = screen + y * rowBytes;
1026 for (blob = firstBlob; blob <= lastBlob; blob++) {
1027 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
1028 for (x = 0; x < kIOHibernateProgressWidth; x++) {
1029 alpha = gIOHibernateProgressAlpha[y][x];
1030 result = color;
1031 if (alpha) {
1032 if (0xff != alpha) {
1033 in = display->progressSaveUnder[blob][saveindex[blob]++];
1034 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
1035 }
1036 if (1 == pixelShift) {
1037 result >>= 3;
1038 *((uint16_t *)out) = ((uint16_t)((result << 10) | (result << 5) | result)); // 16
1039 } else {
1040 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1041 }
1042 }
1043 out += (1 << pixelShift);
1044 }
1045 out += (kIOHibernateProgressSpacing << pixelShift);
1046 }
1047 }
1048 }
1049
1050 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1051
1052 IOReturn
1053 IOHibernateIOKitSleep(void)
1054 {
1055 IOReturn ret = kIOReturnSuccess;
1056 IOLockLock(gFSLock);
1057 if (kFSOpening == gFSState) {
1058 gFSState = kFSTimedOut;
1059 HIBLOG("hibernate file open timed out\n");
1060 ret = kIOReturnTimeout;
1061 }
1062 IOLockUnlock(gFSLock);
1063 return ret;
1064 }
1065
1066 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1067
1068 IOReturn
1069 IOHibernateSystemHasSlept(void)
1070 {
1071 IOReturn ret = kIOReturnSuccess;
1072 IOHibernateVars * vars = &gIOHibernateVars;
1073 OSObject * obj = NULL;
1074 OSData * data;
1075
1076 IOLockLock(gFSLock);
1077 if ((kFSOpened != gFSState) && gIOHibernateMode) {
1078 ret = kIOReturnTimeout;
1079 }
1080 IOLockUnlock(gFSLock);
1081 if (kIOReturnSuccess != ret) {
1082 return ret;
1083 }
1084
1085 if (gIOHibernateMode) {
1086 obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
1087 }
1088 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1089 if (obj && !vars->previewBuffer) {
1090 obj->release();
1091 }
1092 if (vars->previewBuffer && (vars->previewBuffer->getLength() > UINT_MAX)) {
1093 OSSafeReleaseNULL(vars->previewBuffer);
1094 }
1095
1096 vars->consoleMapping = NULL;
1097 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare())) {
1098 vars->previewBuffer->release();
1099 vars->previewBuffer = NULL;
1100 }
1101
1102 if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1103 && vars->previewBuffer
1104 && (data = OSDynamicCast(OSData,
1105 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey)))) {
1106 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
1107 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
1108
1109 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1110
1111 if (kIOHibernatePreviewUpdates & flags) {
1112 PE_Video consoleInfo;
1113 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1114
1115 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1116
1117 graphicsInfo->width = (uint32_t) consoleInfo.v_width;
1118 graphicsInfo->height = (uint32_t) consoleInfo.v_height;
1119 graphicsInfo->rowBytes = (uint32_t) consoleInfo.v_rowBytes;
1120 graphicsInfo->depth = (uint32_t) consoleInfo.v_depth;
1121 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1122
1123 HIBPRINT("video %p %d %d %d\n",
1124 vars->consoleMapping, graphicsInfo->depth,
1125 graphicsInfo->width, graphicsInfo->height);
1126 if (vars->consoleMapping) {
1127 ProgressInit(graphicsInfo, vars->consoleMapping,
1128 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1129 }
1130 }
1131 }
1132
1133 if (gIOOptionsEntry) {
1134 #if __x86_64__
1135 gIOOptionsEntry->sync();
1136 #else
1137 if (gIOHibernateMode) {
1138 gIOOptionsEntry->sync();
1139 }
1140 #endif
1141 }
1142
1143 return ret;
1144 }
1145
1146 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1147
1148 static const DeviceTreeNode *
1149 MergeDeviceTree(const DeviceTreeNode * entry, IORegistryEntry * regEntry, OSSet * entriesToUpdate, vm_offset_t region_start, vm_size_t region_size)
1150 {
1151 DeviceTreeNodeProperty * prop;
1152 const DeviceTreeNode * child;
1153 IORegistryEntry * childRegEntry;
1154 const char * nameProp;
1155 unsigned int propLen, idx;
1156
1157 bool updateEntry = true;
1158 if (!regEntry) {
1159 updateEntry = false;
1160 } else if (entriesToUpdate && !entriesToUpdate->containsObject(regEntry)) {
1161 updateEntry = false;
1162 }
1163
1164 prop = (DeviceTreeNodeProperty *) (entry + 1);
1165 for (idx = 0; idx < entry->nProperties; idx++) {
1166 if (updateEntry && (0 != strcmp("name", prop->name))) {
1167 regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
1168 // HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
1169 }
1170 prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
1171 }
1172
1173 if (entriesToUpdate) {
1174 entriesToUpdate->removeObject(regEntry);
1175 if (entriesToUpdate->getCount() == 0) {
1176 // we've updated all the entries we care about so we can stop
1177 return NULL;
1178 }
1179 }
1180
1181 child = (const DeviceTreeNode *) prop;
1182 for (idx = 0; idx < entry->nChildren; idx++) {
1183 if (kSuccess != SecureDTGetPropertyRegion(child, "name", (void const **) &nameProp, &propLen,
1184 region_start, region_size)) {
1185 panic("no name");
1186 }
1187 childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
1188 // HIBPRINT("%s == %p\n", nameProp, childRegEntry);
1189 child = MergeDeviceTree(child, childRegEntry, entriesToUpdate, region_start, region_size);
1190 if (!child) {
1191 // the recursive call updated the last entry we cared about, so we can stop
1192 break;
1193 }
1194 }
1195 return child;
1196 }
1197
1198 IOReturn
1199 IOHibernateSystemWake(void)
1200 {
1201 if (kFSOpened == gFSState) {
1202 IOPolledFilePollersClose(gIOHibernateVars.fileVars, kIOPolledPostflightState);
1203 IOHibernateDone(&gIOHibernateVars);
1204 } else {
1205 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1206 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1207 }
1208
1209 if (gIOOptionsEntry && gIOHibernateBootImageKey) {
1210 // if we got this far, clear boot-image
1211 // we don't need to sync immediately; the booter should have already removed this entry
1212 // we just want to make sure that if anyone syncs nvram after this point, we don't re-write
1213 // a stale boot-image value
1214 gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
1215 }
1216
1217 return kIOReturnSuccess;
1218 }
1219
1220 static IOReturn
1221 IOHibernateDone(IOHibernateVars * vars)
1222 {
1223 IOReturn err;
1224 OSData * data;
1225
1226 hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
1227
1228 if (vars->videoMapping) {
1229 if (vars->videoMapSize) {
1230 // remove mappings
1231 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1232 }
1233 if (vars->videoAllocSize) {
1234 // dealloc range
1235 kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
1236 }
1237 }
1238
1239 if (vars->previewBuffer) {
1240 vars->previewBuffer->release();
1241 vars->previewBuffer = NULL;
1242 }
1243
1244 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
1245 IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
1246 gIOHibernateCurrentHeader->options, 32);
1247 } else {
1248 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1249 }
1250
1251 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1252 && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus)) {
1253 IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
1254 &gIOHibernateGraphicsInfo->gfxStatus,
1255 sizeof(gIOHibernateGraphicsInfo->gfxStatus));
1256 } else {
1257 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1258 }
1259
1260 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1261
1262 #if defined(__i386__) || defined(__x86_64__)
1263 IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
1264 IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
1265
1266 /*
1267 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1268 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1269 */
1270 if (gIOOptionsEntry) {
1271 if (gIOHibernateRTCVariablesKey) {
1272 if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
1273 gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
1274 }
1275 }
1276
1277 if (gIOHibernateBootNextKey) {
1278 if (gIOHibernateBootNextSave) {
1279 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
1280 gIOHibernateBootNextSave->release();
1281 gIOHibernateBootNextSave = NULL;
1282 } else {
1283 gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
1284 }
1285 }
1286 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) {
1287 gIOOptionsEntry->sync();
1288 }
1289 }
1290 #endif
1291
1292 if (vars->srcBuffer) {
1293 vars->srcBuffer->release();
1294 }
1295
1296
1297 bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
1298 if (vars->handoffBuffer) {
1299 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
1300 IOHibernateHandoff * handoff;
1301 bool done = false;
1302 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
1303 !done;
1304 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount]) {
1305 HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
1306 uint8_t * __unused data = &handoff->data[0];
1307 switch (handoff->type) {
1308 case kIOHibernateHandoffTypeEnd:
1309 done = true;
1310 break;
1311
1312 case kIOHibernateHandoffTypeDeviceTree:
1313 {
1314 #if defined(__i386__) || defined(__x86_64__)
1315 // On Intel, process the entirety of the passed in device tree
1316 OSSet * entriesToUpdate = NULL;
1317 #elif defined(__arm64__)
1318 // On ARM, only allow hibernation to update specific entries
1319 const char *mergePaths[] = {
1320 kIODeviceTreePlane ":/chosen/boot-object-manifests",
1321 kIODeviceTreePlane ":/chosen/secure-boot-hashes",
1322 };
1323 const size_t mergePathCount = sizeof(mergePaths) / sizeof(mergePaths[0]);
1324 OSSet * entriesToUpdate = OSSet::withCapacity(mergePathCount);
1325 for (size_t i = 0; i < mergePathCount; i++) {
1326 IORegistryEntry *entry = IORegistryEntry::fromPath(mergePaths[i]);
1327 if (!entry) {
1328 panic("failed to find %s in IORegistry", mergePaths[i]);
1329 }
1330 entriesToUpdate->setObject(entry);
1331 OSSafeReleaseNULL(entry);
1332 }
1333 #endif
1334 MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot(), entriesToUpdate,
1335 (vm_offset_t)data, (vm_size_t)handoff->bytecount);
1336 OSSafeReleaseNULL(entriesToUpdate);
1337 break;
1338 }
1339
1340 case kIOHibernateHandoffTypeKeyStore:
1341 #if defined(__i386__) || defined(__x86_64__)
1342 {
1343 IOBufferMemoryDescriptor *
1344 md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
1345 if (md) {
1346 IOSetKeyStoreData(md);
1347 }
1348 }
1349 #endif
1350 break;
1351
1352 default:
1353 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
1354 break;
1355 }
1356 }
1357 #if defined(__i386__) || defined(__x86_64__)
1358 if (vars->volumeCryptKeySize) {
1359 IOBufferMemoryDescriptor *
1360 bmd = IOBufferMemoryDescriptor::withBytes(&vars->volumeCryptKey[0],
1361 vars->volumeCryptKeySize, kIODirectionOutIn);
1362 if (!bmd) {
1363 panic("IOBufferMemoryDescriptor");
1364 }
1365 IOSetAPFSKeyStoreData(bmd);
1366 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
1367 }
1368 #endif
1369 }
1370 vars->handoffBuffer->release();
1371 }
1372
1373 if (gIOChosenEntry
1374 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey)))
1375 && (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength())) {
1376 bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0],
1377 sizeof(gIOHibernateBridgeBootSessionUUIDString));
1378 }
1379
1380 if (vars->hwEncrypt) {
1381 err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, NULL, 0);
1382 HIBLOG("IOPolledFilePollersSetEncryptionKey(0,%x)\n", err);
1383 }
1384
1385 bzero(vars, sizeof(*vars));
1386
1387 // gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1388 gIOHibernateCount++;
1389
1390 return kIOReturnSuccess;
1391 }
1392
1393 static void
1394 IOHibernateSystemPostWakeTrim(void * p1, void * p2)
1395 {
1396 // invalidate & close the image file
1397 if (p1) {
1398 IOLockLock(gFSLock);
1399 }
1400 if (kFSTrimDelay == gFSState) {
1401 IOPolledFileIOVars * vars = &gFileVars;
1402 IOPolledFileClose(&vars,
1403 #if DISABLE_TRIM
1404 0, NULL, 0, 0, 0);
1405 #else
1406 0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader),
1407 sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize);
1408 #endif
1409 gFSState = kFSIdle;
1410 }
1411 if (p1) {
1412 IOLockUnlock(gFSLock);
1413 }
1414 }
1415
1416 IOReturn
1417 IOHibernateSystemPostWake(bool now)
1418 {
1419 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1420 IOSetBootImageNVRAM(NULL);
1421
1422 IOLockLock(gFSLock);
1423 if (kFSTrimDelay == gFSState) {
1424 thread_call_cancel(gIOHibernateTrimCalloutEntry);
1425 IOHibernateSystemPostWakeTrim(NULL, NULL);
1426 } else if (kFSOpened != gFSState) {
1427 gFSState = kFSIdle;
1428 } else {
1429 gFSState = kFSTrimDelay;
1430 if (now) {
1431 thread_call_cancel(gIOHibernateTrimCalloutEntry);
1432 IOHibernateSystemPostWakeTrim(NULL, NULL);
1433 } else {
1434 AbsoluteTime deadline;
1435 clock_interval_to_deadline(TRIM_DELAY, kMillisecondScale, &deadline );
1436 thread_call_enter1_delayed(gIOHibernateTrimCalloutEntry, NULL, deadline);
1437 }
1438 }
1439 IOLockUnlock(gFSLock);
1440
1441 return kIOReturnSuccess;
1442 }
1443
1444 uint32_t
1445 IOHibernateWasScreenLocked(void)
1446 {
1447 uint32_t ret = 0;
1448 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry) {
1449 OSData *
1450 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
1451 if (data) {
1452 ret = ((uint32_t *)data->getBytesNoCopy())[0];
1453 gIOChosenEntry->setProperty(kIOBooterScreenLockStateKey, data);
1454 }
1455 } else {
1456 gIOChosenEntry->removeProperty(kIOBooterScreenLockStateKey);
1457 }
1458
1459 return ret;
1460 }
1461
1462 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1463
1464 SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
1465 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_LOCKED,
1466 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
1467 SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
1468 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_LOCKED,
1469 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
1470 SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
1471 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_LOCKED,
1472 &gIOHibernateMode, 0, "");
1473 SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
1474 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED,
1475 &_hibernateStats, hibernate_statistics_t, "");
1476 SYSCTL_OID_MANUAL(_kern_bridge, OID_AUTO, bootsessionuuid,
1477 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1478 gIOHibernateBridgeBootSessionUUIDString, sizeof(gIOHibernateBridgeBootSessionUUIDString),
1479 sysctl_handle_string, "A", "");
1480
1481 SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
1482 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1483 &_hibernateStats.graphicsReadyTime, 0, "");
1484 SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
1485 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1486 &_hibernateStats.wakeNotificationTime, 0, "");
1487 SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
1488 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1489 &_hibernateStats.lockScreenReadyTime, 0, "");
1490 SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
1491 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1492 &_hibernateStats.hidReadyTime, 0, "");
1493
1494 SYSCTL_UINT(_kern, OID_AUTO, hibernatecount,
1495 CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1496 &gIOHibernateCount, 0, "");
1497
1498 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1499
1500 static int
1501 hibernate_set_preview SYSCTL_HANDLER_ARGS
1502 {
1503 #pragma unused(oidp, arg1, arg2)
1504
1505 if (!IOTaskHasEntitlement(current_task(), kIOHibernateSetPreviewEntitlementKey)) {
1506 return EPERM;
1507 }
1508
1509 if ((req->newptr == USER_ADDR_NULL) || (!req->newlen)) {
1510 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewBufferKey);
1511 return 0;
1512 }
1513
1514 size_t rounded_size = round_page(req->newlen);
1515 IOBufferMemoryDescriptor *md = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, rounded_size, page_size);
1516 if (!md) {
1517 return ENOMEM;
1518 }
1519
1520 uint8_t *bytes = (uint8_t *)md->getBytesNoCopy();
1521 int error = SYSCTL_IN(req, bytes, req->newlen);
1522 if (error) {
1523 md->release();
1524 return error;
1525 }
1526
1527 IOService::getPMRootDomain()->setProperty(kIOHibernatePreviewBufferKey, md);
1528 md->release();
1529
1530 return 0;
1531 }
1532
1533 SYSCTL_PROC(_kern, OID_AUTO, hibernatepreview,
1534 CTLTYPE_OPAQUE | CTLFLAG_WR | CTLFLAG_LOCKED | CTLFLAG_ANYBODY, NULL, 0,
1535 hibernate_set_preview, "S", "");
1536
1537 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1538
1539 void
1540 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1541 {
1542 gIOHibernateBootImageKey = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1543 gIOHibernateBootSignatureKey = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
1544 gIOBridgeBootSessionUUIDKey = OSSymbol::withCStringNoCopy(kIOBridgeBootSessionUUIDKey);
1545
1546 #if defined(__i386__) || defined(__x86_64__)
1547 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1548 gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
1549 gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
1550 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1551 #endif /* defined(__i386__) || defined(__x86_64__) */
1552
1553 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
1554 if (data) {
1555 rootDomain->setProperty(kIOHibernateStateKey, data);
1556 data->release();
1557 }
1558
1559 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename))) {
1560 gIOHibernateMode = kIOHibernateModeOn;
1561 } else {
1562 gIOHibernateFilename[0] = 0;
1563 }
1564
1565 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1566
1567 if (gIOChosenEntry
1568 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey)))
1569 && (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength())) {
1570 sysctl_register_oid(&sysctl__kern_bridge_bootsessionuuid);
1571 bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0], sizeof(gIOHibernateBridgeBootSessionUUIDString));
1572 }
1573
1574 gFSLock = IOLockAlloc();
1575 gIOHibernateCount = 0;
1576 }
1577
1578 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1579
1580 static IOReturn
1581 IOHibernatePolledFileWrite(IOHibernateVars * vars,
1582 const uint8_t * bytes, IOByteCount size,
1583 IOPolledFileCryptVars * cryptvars)
1584 {
1585 IOReturn err;
1586
1587
1588 err = IOPolledFileWrite(vars->fileVars, bytes, size, cryptvars);
1589 if ((kIOReturnSuccess == err) && hibernate_should_abort()) {
1590 err = kIOReturnAborted;
1591 }
1592
1593
1594 return err;
1595 }
1596
1597 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1598
1599 extern "C" uint32_t
1600 hibernate_write_image(void)
1601 {
1602 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
1603 IOHibernateVars * vars = &gIOHibernateVars;
1604 IOPolledFileExtent * fileExtents;
1605
1606 #if !defined(__arm64__)
1607 _static_assert_1_arg(sizeof(IOHibernateImageHeader) == 512);
1608 #endif /* !defined(__arm64__) */
1609
1610 uint32_t pageCount, pagesDone;
1611 IOReturn err;
1612 ppnum_t ppnum, page;
1613 vm_offset_t count;
1614 uint8_t * src;
1615 uint8_t * data;
1616 uint8_t * compressed;
1617 uint8_t * scratch;
1618 IOByteCount pageCompressedSize;
1619 uint64_t compressedSize, uncompressedSize;
1620 uint64_t image1Size = 0;
1621 uint32_t bitmap_size;
1622 bool iterDone, pollerOpen, needEncrypt;
1623 int wkresult;
1624 uint32_t tag;
1625 uint32_t pageType;
1626 uint32_t pageAndCount[2];
1627 addr64_t phys64;
1628 IOByteCount segLen;
1629 uint32_t restore1Sum = 0, sum = 0, sum1 = 0, sum2 = 0;
1630 uintptr_t hibernateBase;
1631 uintptr_t hibernateEnd;
1632
1633 AbsoluteTime startTime, endTime;
1634 AbsoluteTime allTime, compTime;
1635 uint64_t compBytes;
1636 uint64_t nsec;
1637 uint64_t lastProgressStamp = 0;
1638 uint64_t progressStamp;
1639 uint32_t blob, lastBlob = (uint32_t) -1L;
1640
1641 uint32_t wiredPagesEncrypted;
1642 uint32_t dirtyPagesEncrypted;
1643 uint32_t wiredPagesClear;
1644 uint32_t svPageCount;
1645 uint32_t zvPageCount;
1646
1647 IOPolledFileCryptVars _cryptvars;
1648 IOPolledFileCryptVars * cryptvars = NULL;
1649
1650 wiredPagesEncrypted = 0;
1651 dirtyPagesEncrypted = 0;
1652 wiredPagesClear = 0;
1653 svPageCount = 0;
1654 zvPageCount = 0;
1655
1656 if (!vars->fileVars
1657 || !vars->fileVars->pollers
1658 || !(kIOHibernateModeOn & gIOHibernateMode)) {
1659 return kIOHibernatePostWriteSleep;
1660 }
1661
1662
1663 #if !defined(__arm64__)
1664 if (kIOHibernateModeSleep & gIOHibernateMode) {
1665 kdebug_enable = save_kdebug_enable;
1666 }
1667 #endif /* !defined(__arm64__) */
1668
1669 pal_hib_write_hook();
1670
1671 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START);
1672 IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
1673
1674 #if CRYPTO
1675 // encryption data. "iv" is the "initial vector".
1676 if (kIOHibernateModeEncrypt & gIOHibernateMode) {
1677 static const unsigned char first_iv[AES_BLOCK_SIZE]
1678 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
1679 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
1680
1681 cryptvars = &gIOHibernateCryptWakeContext;
1682 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1683 aes_encrypt_key(vars->cryptKey,
1684 kIOHibernateAESKeySize,
1685 &cryptvars->ctx.encrypt);
1686 aes_decrypt_key(vars->cryptKey,
1687 kIOHibernateAESKeySize,
1688 &cryptvars->ctx.decrypt);
1689
1690 cryptvars = &_cryptvars;
1691 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1692 for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++) {
1693 vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
1694 }
1695 aes_encrypt_key(vars->wiredCryptKey,
1696 kIOHibernateAESKeySize,
1697 &cryptvars->ctx.encrypt);
1698
1699 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1700 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1701 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
1702 }
1703 #endif /* CRYPTO */
1704
1705 hibernate_page_list_setall(vars->page_list,
1706 vars->page_list_wired,
1707 vars->page_list_pal,
1708 false /* !preflight */,
1709 /* discard_all */
1710 ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
1711 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
1712 &pageCount);
1713
1714 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
1715
1716 fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy();
1717
1718 #if 0
1719 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
1720 for (page = 0; page < count; page++) {
1721 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
1722 fileExtents[page].start, fileExtents[page].length,
1723 fileExtents[page].start + fileExtents[page].length);
1724 }
1725 #endif
1726
1727 needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
1728 AbsoluteTime_to_scalar(&compTime) = 0;
1729 compBytes = 0;
1730
1731 clock_get_uptime(&allTime);
1732 IOService::getPMRootDomain()->pmStatsRecordEvent(
1733 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
1734 do{
1735 compressedSize = 0;
1736 uncompressedSize = 0;
1737 svPageCount = 0;
1738 zvPageCount = 0;
1739
1740 IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
1741
1742 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
1743 ml_get_interrupts_enabled());
1744 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState,
1745 // abortable if not low battery
1746 !IOService::getPMRootDomain()->mustHibernate());
1747 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
1748 pollerOpen = (kIOReturnSuccess == err);
1749 if (!pollerOpen) {
1750 break;
1751 }
1752
1753 if (vars->volumeCryptKeySize) {
1754 err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
1755 HIBLOG("IOPolledFilePollersSetEncryptionKey(%x)\n", err);
1756 vars->hwEncrypt = (kIOReturnSuccess == err);
1757 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
1758 if (vars->hwEncrypt) {
1759 header->options |= kIOHibernateOptionHWEncrypt;
1760 }
1761 }
1762
1763 // copy file block extent list if larger than header
1764
1765 count = vars->fileVars->fileExtents->getLength();
1766 if (count > sizeof(header->fileExtentMap)) {
1767 count -= sizeof(header->fileExtentMap);
1768 err = IOHibernatePolledFileWrite(vars,
1769 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
1770 if (kIOReturnSuccess != err) {
1771 break;
1772 }
1773 }
1774
1775 // copy out restore1 code
1776
1777 for (count = 0;
1778 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1779 count += segLen) {
1780 for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++) {
1781 gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64_ppnum(phys64) + pagesDone;
1782 }
1783 }
1784
1785 hibernateBase = HIB_BASE; /* Defined in PAL headers */
1786 hibernateEnd = (segHIBB + segSizeHIB);
1787
1788 page = atop_32(kvtophys(hibernateBase));
1789 count = atop_32(round_page(hibernateEnd) - hibernateBase);
1790 uintptr_t entrypoint = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
1791 uintptr_t stack = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
1792 if ((count > UINT_MAX) || (entrypoint > UINT_MAX) || (stack > UINT_MAX)) {
1793 panic("malformed kernel layout");
1794 }
1795 header->restore1CodePhysPage = (ppnum_t) page;
1796 header->restore1CodeVirt = hibernateBase;
1797 header->restore1PageCount = (uint32_t) count;
1798 header->restore1CodeOffset = (uint32_t) entrypoint;
1799 header->restore1StackOffset = (uint32_t) stack;
1800
1801 if (uuid_parse(&gIOHibernateBridgeBootSessionUUIDString[0], &header->bridgeBootSessionUUID[0])) {
1802 bzero(&header->bridgeBootSessionUUID[0], sizeof(header->bridgeBootSessionUUID));
1803 }
1804
1805 // sum __HIB seg, with zeros for the stack
1806 src = (uint8_t *) trunc_page(hibernateBase);
1807 for (page = 0; page < count; page++) {
1808 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0])) {
1809 restore1Sum += hibernate_sum_page(src, (uint32_t) (header->restore1CodeVirt + page));
1810 } else {
1811 restore1Sum += 0x00000000;
1812 }
1813 src += page_size;
1814 }
1815 sum1 = restore1Sum;
1816
1817 // write the __HIB seg, with zeros for the stack
1818
1819 src = (uint8_t *) trunc_page(hibernateBase);
1820 count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
1821 if (count) {
1822 err = IOHibernatePolledFileWrite(vars, src, count, cryptvars);
1823 if (kIOReturnSuccess != err) {
1824 break;
1825 }
1826 }
1827 err = IOHibernatePolledFileWrite(vars,
1828 (uint8_t *) NULL,
1829 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
1830 cryptvars);
1831 if (kIOReturnSuccess != err) {
1832 break;
1833 }
1834 src = &gIOHibernateRestoreStackEnd[0];
1835 count = round_page(hibernateEnd) - ((uintptr_t) src);
1836 if (count) {
1837 err = IOHibernatePolledFileWrite(vars, src, count, cryptvars);
1838 if (kIOReturnSuccess != err) {
1839 break;
1840 }
1841 }
1842
1843 if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) {
1844 vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
1845 vars->fileVars->encryptEnd = UINT64_MAX;
1846 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1847 }
1848
1849 // write the preview buffer
1850
1851 if (vars->previewBuffer) {
1852 ppnum = 0;
1853 count = 0;
1854 do{
1855 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
1856 pageAndCount[0] = atop_64_ppnum(phys64);
1857 pageAndCount[1] = atop_64_ppnum(segLen);
1858 err = IOHibernatePolledFileWrite(vars,
1859 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1860 cryptvars);
1861 if (kIOReturnSuccess != err) {
1862 break;
1863 }
1864 count += segLen;
1865 ppnum += sizeof(pageAndCount);
1866 }while (phys64);
1867 if (kIOReturnSuccess != err) {
1868 break;
1869 }
1870
1871 src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
1872
1873 ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
1874
1875 count = (uint32_t) vars->previewBuffer->getLength();
1876
1877 header->previewPageListSize = ((uint32_t) ppnum);
1878 header->previewSize = ((uint32_t) (count + ppnum));
1879
1880 for (page = 0; page < count; page += page_size) {
1881 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
1882 sum1 += hibernate_sum_page(src + page, atop_64_ppnum(phys64));
1883 }
1884 if (kIOReturnSuccess != err) {
1885 break;
1886 }
1887 err = IOHibernatePolledFileWrite(vars, src, count, cryptvars);
1888 if (kIOReturnSuccess != err) {
1889 break;
1890 }
1891 }
1892
1893 // mark areas for no save
1894 hibernate_set_descriptor_page_state(vars, IOPolledFileGetIOBuffer(vars->fileVars),
1895 kIOHibernatePageStateFree, &pageCount);
1896 hibernate_set_descriptor_page_state(vars, vars->srcBuffer,
1897 kIOHibernatePageStateFree, &pageCount);
1898
1899 // copy out bitmap of pages available for trashing during restore
1900
1901 bitmap_size = vars->page_list_wired->list_size;
1902 src = (uint8_t *) vars->page_list_wired;
1903 err = IOHibernatePolledFileWrite(vars, src, bitmap_size, cryptvars);
1904 if (kIOReturnSuccess != err) {
1905 break;
1906 }
1907
1908 // mark more areas for no save, but these are not available
1909 // for trashing during restore
1910
1911 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
1912
1913 #if defined(__i386__) || defined(__x86_64__)
1914 // __HIB is explicitly saved above so we don't have to save it again
1915 page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
1916 count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
1917 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1918 page, count,
1919 kIOHibernatePageStateFree);
1920 pageCount -= count;
1921 #elif defined(__arm64__)
1922 // the segments described in IOHibernateHibSegInfo are stored directly in the
1923 // hibernation file, so they don't need to be saved again
1924 extern unsigned long gPhysBase, gPhysSize;
1925 for (size_t i = 0; i < NUM_HIBSEGINFO_SEGMENTS; i++) {
1926 page = segInfo->segments[i].physPage;
1927 count = segInfo->segments[i].pageCount;
1928 uint64_t physAddr = ptoa_64(page);
1929 uint64_t size = ptoa_64(count);
1930 if (size &&
1931 (physAddr >= gPhysBase) &&
1932 (physAddr + size <= gPhysBase + gPhysSize)) {
1933 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1934 page, count,
1935 kIOHibernatePageStateFree);
1936 pageCount -= count;
1937 }
1938 }
1939 #else
1940 #error unimplemented
1941 #endif
1942
1943 hibernate_set_descriptor_page_state(vars, vars->previewBuffer,
1944 kIOHibernatePageStateFree, &pageCount);
1945 hibernate_set_descriptor_page_state(vars, vars->handoffBuffer,
1946 kIOHibernatePageStateFree, &pageCount);
1947
1948 #if KASAN
1949 vm_size_t shadow_pages_free = atop_64(shadow_ptop) - atop_64(shadow_pnext);
1950
1951 /* no need to save unused shadow pages */
1952 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1953 atop_64(shadow_pnext),
1954 shadow_pages_free,
1955 kIOHibernatePageStateFree);
1956 #endif
1957
1958 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
1959 compressed = src + page_size;
1960 scratch = compressed + page_size;
1961
1962 pagesDone = 0;
1963 lastBlob = 0;
1964
1965 HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
1966 bitmap_size, header->previewSize,
1967 pageCount, vars->fileVars->position);
1968
1969
1970 enum
1971 // pageType
1972 {
1973 kWired = 0x02,
1974 kEncrypt = 0x01,
1975 kWiredEncrypt = kWired | kEncrypt,
1976 kWiredClear = kWired,
1977 kUnwiredEncrypt = kEncrypt
1978 };
1979
1980 #if defined(__i386__) || defined(__x86_64__)
1981 bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features()));
1982 #else /* defined(__i386__) || defined(__x86_64__) */
1983 static const bool cpuAES = true;
1984 #endif /* defined(__i386__) || defined(__x86_64__) */
1985
1986 for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--) {
1987 if (kUnwiredEncrypt == pageType) {
1988 // start unwired image
1989 if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) {
1990 vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
1991 vars->fileVars->encryptEnd = UINT64_MAX;
1992 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1993 }
1994 bcopy(&cryptvars->aes_iv[0],
1995 &gIOHibernateCryptWakeContext.aes_iv[0],
1996 sizeof(cryptvars->aes_iv));
1997 cryptvars = &gIOHibernateCryptWakeContext;
1998 }
1999 for (iterDone = false, ppnum = 0; !iterDone;) {
2000 if (cpuAES && (pageType == kWiredClear)) {
2001 count = 0;
2002 } else {
2003 count = hibernate_page_list_iterate((kWired & pageType) ? vars->page_list_wired : vars->page_list,
2004 &ppnum);
2005 if (count > UINT_MAX) {
2006 count = UINT_MAX;
2007 }
2008 }
2009 // kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
2010 iterDone = !count;
2011
2012 if (!cpuAES) {
2013 if (count && (kWired & pageType) && needEncrypt) {
2014 uint32_t checkIndex;
2015 for (checkIndex = 0;
2016 (checkIndex < count)
2017 && (((kEncrypt & pageType) == 0) == pmap_is_noencrypt(((ppnum_t)(ppnum + checkIndex))));
2018 checkIndex++) {
2019 }
2020 if (!checkIndex) {
2021 ppnum++;
2022 continue;
2023 }
2024 count = checkIndex;
2025 }
2026 }
2027
2028 switch (pageType) {
2029 case kWiredEncrypt: wiredPagesEncrypted += count; break;
2030 case kWiredClear: wiredPagesClear += count; break;
2031 case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
2032 }
2033
2034 if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */
2035 } else {
2036 pageAndCount[0] = (uint32_t) ppnum;
2037 pageAndCount[1] = (uint32_t) count;
2038 err = IOHibernatePolledFileWrite(vars,
2039 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
2040 cryptvars);
2041 if (kIOReturnSuccess != err) {
2042 break;
2043 }
2044 }
2045
2046 for (page = ppnum; page < (ppnum + count); page++) {
2047 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
2048 if (err) {
2049 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
2050 break;
2051 }
2052
2053 sum = hibernate_sum_page(src, (uint32_t) page);
2054 if (kWired & pageType) {
2055 sum1 += sum;
2056 } else {
2057 sum2 += sum;
2058 }
2059
2060 clock_get_uptime(&startTime);
2061 wkresult = WKdm_compress_new((const WK_word*) src,
2062 (WK_word*) compressed,
2063 (WK_word*) scratch,
2064 (uint32_t) (page_size - 4));
2065
2066 clock_get_uptime(&endTime);
2067 ADD_ABSOLUTETIME(&compTime, &endTime);
2068 SUB_ABSOLUTETIME(&compTime, &startTime);
2069
2070 compBytes += page_size;
2071 pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
2072
2073 if (pageCompressedSize == 0) {
2074 pageCompressedSize = 4;
2075 data = src;
2076
2077 if (*(uint32_t *)src) {
2078 svPageCount++;
2079 } else {
2080 zvPageCount++;
2081 }
2082 } else {
2083 if (pageCompressedSize != page_size) {
2084 data = compressed;
2085 } else {
2086 data = src;
2087 }
2088 }
2089
2090 assert(pageCompressedSize <= page_size);
2091 tag = ((uint32_t) pageCompressedSize) | kIOHibernateTagSignature;
2092 err = IOHibernatePolledFileWrite(vars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
2093 if (kIOReturnSuccess != err) {
2094 break;
2095 }
2096
2097 err = IOHibernatePolledFileWrite(vars, data, (pageCompressedSize + 3) & ~3, cryptvars);
2098 if (kIOReturnSuccess != err) {
2099 break;
2100 }
2101
2102 compressedSize += pageCompressedSize;
2103 uncompressedSize += page_size;
2104 pagesDone++;
2105
2106 if (vars->consoleMapping && (0 == (1023 & pagesDone))) {
2107 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
2108 if (blob != lastBlob) {
2109 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
2110 lastBlob = blob;
2111 }
2112 }
2113 if (0 == (8191 & pagesDone)) {
2114 clock_get_uptime(&endTime);
2115 SUB_ABSOLUTETIME(&endTime, &allTime);
2116 absolutetime_to_nanoseconds(endTime, &nsec);
2117 progressStamp = nsec / 750000000ULL;
2118 if (progressStamp != lastProgressStamp) {
2119 lastProgressStamp = progressStamp;
2120 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
2121 }
2122 }
2123 }
2124 if (kIOReturnSuccess != err) {
2125 break;
2126 }
2127 ppnum = page;
2128 }
2129
2130 if (kIOReturnSuccess != err) {
2131 break;
2132 }
2133
2134 if ((kEncrypt & pageType) && vars->fileVars->encryptStart) {
2135 vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
2136 HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
2137 }
2138
2139 if (kWiredEncrypt != pageType) {
2140 // end of image1/2 - fill to next block
2141 err = IOHibernatePolledFileWrite(vars, NULL, 0, cryptvars);
2142 if (kIOReturnSuccess != err) {
2143 break;
2144 }
2145 }
2146 if (kWiredClear == pageType) {
2147 // enlarge wired image for test
2148 // err = IOHibernatePolledFileWrite(vars, 0, 0x60000000, cryptvars);
2149
2150 // end wired image
2151 header->encryptStart = vars->fileVars->encryptStart;
2152 header->encryptEnd = vars->fileVars->encryptEnd;
2153 image1Size = vars->fileVars->position;
2154 HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
2155 image1Size, header->encryptStart, header->encryptEnd);
2156 }
2157 }
2158 if (kIOReturnSuccess != err) {
2159 if (kIOReturnOverrun == err) {
2160 // update actual compression ratio on not enough space (for retry)
2161 gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
2162 }
2163
2164 // update partial amount written (for IOPolledFileClose cleanup/unmap)
2165 header->imageSize = vars->fileVars->position;
2166 break;
2167 }
2168
2169
2170 // Header:
2171
2172 header->imageSize = vars->fileVars->position;
2173 header->image1Size = image1Size;
2174 header->bitmapSize = bitmap_size;
2175 header->pageCount = pageCount;
2176
2177 header->restore1Sum = restore1Sum;
2178 header->image1Sum = sum1;
2179 header->image2Sum = sum2;
2180 header->sleepTime = gIOLastSleepTime.tv_sec;
2181
2182 header->compression = ((uint32_t)((compressedSize << 8) / uncompressedSize));
2183 #if defined(__arm64__)
2184 /*
2185 * We don't support retry on hibernation failure and so
2186 * we don't want to set this value to anything smaller
2187 * just because we may have been lucky this time around.
2188 * Though we'll let it go higher.
2189 */
2190 if (header->compression < HIB_COMPR_RATIO_ARM64) {
2191 header->compression = HIB_COMPR_RATIO_ARM64;
2192 }
2193 #endif /* __arm64__ */
2194
2195 gIOHibernateCompression = header->compression;
2196
2197 count = vars->fileVars->fileExtents->getLength();
2198 if (count > sizeof(header->fileExtentMap)) {
2199 header->fileExtentMapSize = ((uint32_t) count);
2200 count = sizeof(header->fileExtentMap);
2201 } else {
2202 header->fileExtentMapSize = sizeof(header->fileExtentMap);
2203 }
2204 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
2205
2206 header->deviceBase = vars->fileVars->block0;
2207 header->deviceBlockSize = vars->fileVars->blockSize;
2208 header->lastHibAbsTime = mach_absolute_time();
2209 header->lastHibContTime = mach_continuous_time();
2210
2211
2212 IOPolledFileSeek(vars->fileVars, 0);
2213 err = IOHibernatePolledFileWrite(vars,
2214 (uint8_t *) header, sizeof(IOHibernateImageHeader),
2215 cryptvars);
2216 if (kIOReturnSuccess != err) {
2217 break;
2218 }
2219
2220 err = IOHibernatePolledFileWrite(vars, NULL, 0, cryptvars);
2221 }while (false);
2222
2223 clock_get_uptime(&endTime);
2224
2225 IOService::getPMRootDomain()->pmStatsRecordEvent(
2226 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
2227
2228 SUB_ABSOLUTETIME(&endTime, &allTime);
2229 absolutetime_to_nanoseconds(endTime, &nsec);
2230 HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
2231
2232 absolutetime_to_nanoseconds(compTime, &nsec);
2233 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2234 compBytes,
2235 nsec / 1000000ULL,
2236 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2237
2238 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2239 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
2240 vars->fileVars->cryptBytes,
2241 nsec / 1000000ULL,
2242 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2243
2244 HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%)\n",
2245 header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
2246 uncompressedSize, atop_32(uncompressedSize), compressedSize,
2247 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0);
2248
2249 HIBLOG("\nsum1 %x, sum2 %x\n", sum1, sum2);
2250
2251 HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
2252 svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
2253
2254 if (pollerOpen) {
2255 IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted );
2256 }
2257
2258 if (vars->consoleMapping) {
2259 ProgressUpdate(gIOHibernateGraphicsInfo,
2260 vars->consoleMapping, 0, kIOHibernateProgressCount);
2261 }
2262
2263 HIBLOG("hibernate_write_image done(%x)\n", err);
2264
2265 // should we come back via regular wake, set the state in memory.
2266 gIOHibernateState = kIOHibernateStateInactive;
2267
2268 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, wiredPagesEncrypted,
2269 wiredPagesClear, dirtyPagesEncrypted);
2270
2271 #if defined(__arm64__)
2272 if (kIOReturnSuccess == err) {
2273 return kIOHibernatePostWriteHalt;
2274 } else {
2275 // on ARM, once ApplePMGR decides we're hibernating, we can't turn back
2276 // see: <rdar://problem/63848862> Tonga ApplePMGR diff quiesce path support
2277 panic("hibernate_write_image encountered error 0x%x", err);
2278 }
2279 #else
2280 if (kIOReturnSuccess == err) {
2281 if (kIOHibernateModeSleep & gIOHibernateMode) {
2282 return kIOHibernatePostWriteSleep;
2283 } else if (kIOHibernateModeRestart & gIOHibernateMode) {
2284 return kIOHibernatePostWriteRestart;
2285 } else {
2286 /* by default, power down */
2287 return kIOHibernatePostWriteHalt;
2288 }
2289 } else if (kIOReturnAborted == err) {
2290 return kIOHibernatePostWriteWake;
2291 } else {
2292 /* on error, sleep */
2293 return kIOHibernatePostWriteSleep;
2294 }
2295 #endif
2296 }
2297
2298 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2299
2300 extern "C" void
2301 hibernate_machine_init(void)
2302 {
2303 IOReturn err;
2304 uint32_t sum;
2305 uint32_t pagesDone;
2306 uint32_t pagesRead = 0;
2307 AbsoluteTime startTime, compTime;
2308 AbsoluteTime allTime, endTime;
2309 AbsoluteTime startIOTime, endIOTime;
2310 uint64_t nsec, nsecIO;
2311 uint64_t compBytes;
2312 uint64_t lastProgressStamp = 0;
2313 uint64_t progressStamp;
2314 IOPolledFileCryptVars * cryptvars = NULL;
2315
2316 IOHibernateVars * vars = &gIOHibernateVars;
2317 bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
2318
2319 if (!vars->fileVars || !vars->fileVars->pollers) {
2320 return;
2321 }
2322
2323 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2324 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2325
2326 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) {
2327 HIBLOG("regular wake\n");
2328 return;
2329 }
2330
2331 HIBPRINT("diag %x %x %x %x\n",
2332 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
2333 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
2334
2335 #if defined(__i386__) || defined(__x86_64__)
2336 #define t40ms(x) ((uint32_t)((tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)))
2337 #else /* defined(__i386__) || defined(__x86_64__) */
2338 #define t40ms(x) x
2339 #endif /* defined(__i386__) || defined(__x86_64__) */
2340 #define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
2341 tStat(booterStart, booterStart);
2342 gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart;
2343 tStat(booterDuration0, booterTime0);
2344 tStat(booterDuration1, booterTime1);
2345 tStat(booterDuration2, booterTime2);
2346 tStat(booterDuration, booterTime);
2347 tStat(booterConnectDisplayDuration, connectDisplayTime);
2348 tStat(booterSplashDuration, splashTime);
2349 tStat(trampolineDuration, trampolineTime);
2350
2351 gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size;
2352 gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize;
2353 gIOHibernateStats->image1Pages = pagesDone;
2354
2355 /* HIBERNATE_stats */
2356 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 14), gIOHibernateStats->smcStart,
2357 gIOHibernateStats->booterStart, gIOHibernateStats->booterDuration,
2358 gIOHibernateStats->trampolineDuration);
2359
2360 HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
2361 gIOHibernateStats->booterStart,
2362 gIOHibernateStats->smcStart,
2363 gIOHibernateStats->booterDuration0,
2364 gIOHibernateStats->booterDuration1,
2365 gIOHibernateStats->booterDuration2,
2366 gIOHibernateStats->booterDuration,
2367 gIOHibernateStats->booterConnectDisplayDuration,
2368 gIOHibernateStats->booterSplashDuration,
2369 gIOHibernateStats->trampolineDuration);
2370
2371 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
2372 gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
2373 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
2374
2375 if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
2376 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))) {
2377 hibernate_page_list_discard(vars->page_list);
2378 }
2379
2380 if (vars->hwEncrypt) {
2381 // if vars->hwEncrypt is true, we don't need cryptvars since we supply the
2382 // decryption key via IOPolledFilePollersSetEncryptionKey
2383 cryptvars = NULL;
2384 } else {
2385 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : NULL;
2386 }
2387
2388 if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount) {
2389 panic("handoff overflow");
2390 }
2391
2392 IOHibernateHandoff * handoff;
2393 bool done = false;
2394 bool foundCryptData = false;
2395 bool foundVolumeEncryptData = false;
2396 const uint8_t * handoffStart = (const uint8_t*)vars->handoffBuffer->getBytesNoCopy();
2397 const uint8_t * handoffEnd = handoffStart + vars->handoffBuffer->getLength();
2398
2399 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
2400 !done;
2401 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount]) {
2402 if (((uint8_t*)handoff < handoffStart) ||
2403 (&handoff->data[handoff->bytecount] > handoffEnd)) {
2404 panic("handoff out of range");
2405 }
2406 // HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
2407 uint8_t * data = &handoff->data[0];
2408 switch (handoff->type) {
2409 case kIOHibernateHandoffTypeEnd:
2410 done = true;
2411 break;
2412
2413 case kIOHibernateHandoffTypeGraphicsInfo:
2414 if (handoff->bytecount == sizeof(*gIOHibernateGraphicsInfo)) {
2415 bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
2416 }
2417 break;
2418
2419 case kIOHibernateHandoffTypeCryptVars:
2420 if (cryptvars) {
2421 hibernate_cryptwakevars_t *
2422 wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
2423 if (handoff->bytecount == sizeof(*wakevars)) {
2424 bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
2425 } else {
2426 panic("kIOHibernateHandoffTypeCryptVars(%d)", handoff->bytecount);
2427 }
2428 }
2429 foundCryptData = true;
2430 bzero(data, handoff->bytecount);
2431 break;
2432
2433 case kIOHibernateHandoffTypeVolumeCryptKey:
2434 if (handoff->bytecount == vars->volumeCryptKeySize) {
2435 bcopy(data, &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
2436 foundVolumeEncryptData = true;
2437 } else {
2438 panic("kIOHibernateHandoffTypeVolumeCryptKey(%d)", handoff->bytecount);
2439 }
2440 break;
2441
2442 #if defined(__i386__) || defined(__x86_64__)
2443 case kIOHibernateHandoffTypeMemoryMap:
2444
2445 clock_get_uptime(&allTime);
2446
2447 hibernate_newruntime_map(data, handoff->bytecount,
2448 gIOHibernateCurrentHeader->systemTableOffset);
2449
2450 clock_get_uptime(&endTime);
2451
2452 SUB_ABSOLUTETIME(&endTime, &allTime);
2453 absolutetime_to_nanoseconds(endTime, &nsec);
2454
2455 HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
2456
2457 break;
2458
2459 case kIOHibernateHandoffTypeDeviceTree:
2460 {
2461 // DTEntry chosen = NULL;
2462 // HIBPRINT("SecureDTLookupEntry %d\n", SecureDTLookupEntry((const DTEntry) data, "/chosen", &chosen));
2463 }
2464 break;
2465 #endif /* defined(__i386__) || defined(__x86_64__) */
2466
2467 default:
2468 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
2469 break;
2470 }
2471 }
2472
2473 if (vars->hwEncrypt && !foundVolumeEncryptData) {
2474 panic("no volumeCryptKey");
2475 } else if (cryptvars && !foundCryptData) {
2476 panic("hibernate handoff");
2477 }
2478
2479 HIBPRINT("video 0x%llx %d %d %d status %x\n",
2480 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2481 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
2482
2483 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress) {
2484 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2485 * gIOHibernateGraphicsInfo->rowBytes);
2486 if (vars->videoMapSize > vars->videoAllocSize) {
2487 vars->videoMapSize = 0;
2488 } else {
2489 IOMapPages(kernel_map,
2490 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2491 vars->videoMapSize, kIOMapInhibitCache );
2492 }
2493 }
2494
2495 if (vars->videoMapSize) {
2496 ProgressUpdate(gIOHibernateGraphicsInfo,
2497 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2498 }
2499
2500 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2501 uint8_t * compressed = src + page_size;
2502 uint8_t * scratch = compressed + page_size;
2503 uint32_t decoOffset;
2504
2505 clock_get_uptime(&allTime);
2506 AbsoluteTime_to_scalar(&compTime) = 0;
2507 compBytes = 0;
2508
2509 HIBLOG("IOPolledFilePollersOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2510 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledAfterSleepState, false);
2511 clock_get_uptime(&startIOTime);
2512 endTime = startIOTime;
2513 SUB_ABSOLUTETIME(&endTime, &allTime);
2514 absolutetime_to_nanoseconds(endTime, &nsec);
2515 HIBLOG("IOPolledFilePollersOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
2516
2517 if (vars->hwEncrypt) {
2518 err = IOPolledFilePollersSetEncryptionKey(vars->fileVars,
2519 &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
2520 HIBLOG("IOPolledFilePollersSetEncryptionKey(%x) %ld\n", err, vars->volumeCryptKeySize);
2521 if (kIOReturnSuccess != err) {
2522 panic("IOPolledFilePollersSetEncryptionKey(0x%x)", err);
2523 }
2524 cryptvars = NULL;
2525 }
2526
2527 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2528
2529 // kick off the read ahead
2530 vars->fileVars->bufferHalf = 0;
2531 vars->fileVars->bufferLimit = 0;
2532 vars->fileVars->lastRead = 0;
2533 vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
2534 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2535 vars->fileVars->cryptBytes = 0;
2536 AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
2537
2538 err = IOPolledFileRead(vars->fileVars, NULL, 0, cryptvars);
2539 if (kIOReturnSuccess != err) {
2540 panic("Hibernate restore error %x", err);
2541 }
2542 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2543 // --
2544
2545 HIBLOG("hibernate_machine_init reading\n");
2546
2547
2548 uint32_t * header = (uint32_t *) src;
2549 sum = 0;
2550
2551 while (kIOReturnSuccess == err) {
2552 unsigned int count;
2553 unsigned int page;
2554 uint32_t tag;
2555 vm_offset_t compressedSize;
2556 ppnum_t ppnum;
2557
2558 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2559 if (kIOReturnSuccess != err) {
2560 panic("Hibernate restore error %x", err);
2561 }
2562
2563 ppnum = header[0];
2564 count = header[1];
2565
2566 // HIBPRINT("(%x, %x)\n", ppnum, count);
2567
2568 if (!count) {
2569 break;
2570 }
2571
2572 for (page = 0; page < count; page++) {
2573 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2574 if (kIOReturnSuccess != err) {
2575 panic("Hibernate restore error %x", err);
2576 }
2577
2578 compressedSize = kIOHibernateTagLength & tag;
2579 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength)) {
2580 err = kIOReturnIPCError;
2581 panic("Hibernate restore error %x", err);
2582 }
2583
2584 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2585 if (kIOReturnSuccess != err) {
2586 panic("Hibernate restore error %x", err);
2587 }
2588
2589 if (compressedSize < page_size) {
2590 decoOffset = ((uint32_t) page_size);
2591 clock_get_uptime(&startTime);
2592
2593 if (compressedSize == 4) {
2594 int i;
2595 uint32_t *s, *d;
2596
2597 s = (uint32_t *)src;
2598 d = (uint32_t *)(uintptr_t)compressed;
2599
2600 for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++) {
2601 *d++ = *s;
2602 }
2603 } else {
2604 pal_hib_decompress_page(src, compressed, scratch, ((unsigned int) compressedSize));
2605 }
2606 clock_get_uptime(&endTime);
2607 ADD_ABSOLUTETIME(&compTime, &endTime);
2608 SUB_ABSOLUTETIME(&compTime, &startTime);
2609 compBytes += page_size;
2610 } else {
2611 decoOffset = 0;
2612 }
2613
2614 sum += hibernate_sum_page((src + decoOffset), ((uint32_t) ppnum));
2615 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2616 if (err) {
2617 HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
2618 panic("Hibernate restore error %x", err);
2619 }
2620
2621
2622 ppnum++;
2623 pagesDone++;
2624 pagesRead++;
2625
2626 if (0 == (8191 & pagesDone)) {
2627 clock_get_uptime(&endTime);
2628 SUB_ABSOLUTETIME(&endTime, &allTime);
2629 absolutetime_to_nanoseconds(endTime, &nsec);
2630 progressStamp = nsec / 750000000ULL;
2631 if (progressStamp != lastProgressStamp) {
2632 lastProgressStamp = progressStamp;
2633 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2634 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2635 }
2636 }
2637 }
2638 }
2639 if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages)) {
2640 err = kIOReturnLockedRead;
2641 }
2642
2643 if (kIOReturnSuccess != err) {
2644 panic("Hibernate restore error %x", err);
2645 }
2646
2647
2648 gIOHibernateCurrentHeader->actualImage2Sum = sum;
2649 gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
2650
2651 clock_get_uptime(&endIOTime);
2652
2653 err = IOPolledFilePollersClose(vars->fileVars, kIOPolledAfterSleepState);
2654
2655 clock_get_uptime(&endTime);
2656
2657 IOService::getPMRootDomain()->pmStatsRecordEvent(
2658 kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
2659 IOService::getPMRootDomain()->pmStatsRecordEvent(
2660 kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
2661
2662 SUB_ABSOLUTETIME(&endTime, &allTime);
2663 absolutetime_to_nanoseconds(endTime, &nsec);
2664
2665 SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
2666 absolutetime_to_nanoseconds(endIOTime, &nsecIO);
2667
2668 gIOHibernateStats->kernelImageReadDuration = ((uint32_t) (nsec / 1000000ULL));
2669 gIOHibernateStats->imagePages = pagesDone;
2670
2671 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
2672 pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
2673 nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
2674
2675 absolutetime_to_nanoseconds(compTime, &nsec);
2676 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2677 compBytes,
2678 nsec / 1000000ULL,
2679 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2680
2681 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2682 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
2683 vars->fileVars->cryptBytes,
2684 nsec / 1000000ULL,
2685 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2686
2687 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 2), pagesRead, pagesDone);
2688 }
2689
2690 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2691
2692 void
2693 IOHibernateSetWakeCapabilities(uint32_t capability)
2694 {
2695 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
2696 gIOHibernateStats->wakeCapability = capability;
2697
2698 if (kIOPMSystemCapabilityGraphics & capability) {
2699 vm_compressor_do_warmup();
2700 }
2701 }
2702 }
2703
2704 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2705
2706 void
2707 IOHibernateSystemRestart(void)
2708 {
2709 static uint8_t noteStore[32] __attribute__((aligned(32)));
2710 IORegistryEntry * regEntry;
2711 const OSSymbol * sym;
2712 OSData * noteProp;
2713 OSData * data;
2714 uintptr_t * smcVars;
2715 uint8_t * smcBytes;
2716 size_t len;
2717 addr64_t element;
2718
2719 data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
2720 if (!data) {
2721 return;
2722 }
2723
2724 smcVars = (typeof(smcVars))data->getBytesNoCopy();
2725 smcBytes = (typeof(smcBytes))smcVars[1];
2726 len = smcVars[0];
2727 if (len > sizeof(noteStore)) {
2728 len = sizeof(noteStore);
2729 }
2730 noteProp = OSData::withCapacity(3 * sizeof(element));
2731 if (!noteProp) {
2732 return;
2733 }
2734 element = len;
2735 noteProp->appendBytes(&element, sizeof(element));
2736 element = crc32(0, smcBytes, len);
2737 noteProp->appendBytes(&element, sizeof(element));
2738
2739 bcopy(smcBytes, noteStore, len);
2740 element = (addr64_t) &noteStore[0];
2741 element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
2742 noteProp->appendBytes(&element, sizeof(element));
2743
2744 if (!gIOOptionsEntry) {
2745 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
2746 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
2747 if (regEntry && !gIOOptionsEntry) {
2748 regEntry->release();
2749 }
2750 }
2751
2752 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
2753 if (gIOOptionsEntry && sym) {
2754 gIOOptionsEntry->setProperty(sym, noteProp);
2755 }
2756 if (noteProp) {
2757 noteProp->release();
2758 }
2759 if (sym) {
2760 sym->release();
2761 }
2762 }
mirror of XNU