]> git.saurik.com Git - apple/xnu.git/blame - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-7195.101.1.tar.gz
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
CommitLineData
3a60a9f5 1/*
39037602 2 * Copyright (c) 2004-2016 Apple Inc. All rights reserved.
3a60a9f5 3 *
2d21ac55 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
39037602 5 *
2d21ac55
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
39037602 14 *
2d21ac55
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
39037602 17 *
2d21ac55
A		 18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5
A		 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55
A		 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
39037602 25 *
2d21ac55 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
3a60a9f5
A		 27 */
28
3a60a9f5 29/*
0a7de745
A		 30 *
31 * Sleep:
32 *
33 * - PMRootDomain calls IOHibernateSystemSleep() before system sleep
34 * (devices awake, normal execution context)
35 * - IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
36 * grabs its extents and searches for a polling driver willing to work with that IOMedia.
37 * The BSD code makes an ioctl to the storage driver to get the partition base offset to
38 * the disk, and other ioctls to get the transfer constraints
39 * If successful, the file is written to make sure its initially not bootable (in case of
40 * later failure) and nvram set to point to the first block of the file. (Has to be done
41 * here so blocking is possible in nvram support).
42 * hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
43 * page out any pages it wants to (currently zero, but probably some percentage of memory).
44 * Its assumed just allocating pages will cause the VM system to naturally select the best
45 * pages for eviction. It also copies processor flags needed for the restore path and sets
46 * a flag in the boot processor proc info.
47 * gIOHibernateState = kIOHibernateStateHibernating.
48 * - Regular sleep progresses - some drivers may inspect the root domain property
49 * kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
50 * as usual but leaves motherboard I/O on.
51 * - Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
52 * at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
53 * all ppc RC bits out of the hash table and caches into the mapping structures.
54 * - hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
55 * hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
56 * All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
57 * by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
58 * The image header and block list are written. The header includes the second file extent so
59 * only the header block is needed to read the file, regardless of filesystem.
60 * The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
61 * (only) is used to decompress the image during wake/boot.
62 * Some additional pages are removed from the bitmaps - the buffers used for hibernation.
63 * The bitmaps are written to the image.
64 * More areas are removed from the bitmaps (after they have been written to the image) - the
65 * segment "__HIB" pages and interrupt stack.
66 * Each wired page is compressed and written and then each non-wired page. Compression and
67 * disk writes are in parallel.
68 * The image header is written to the start of the file and the polling driver closed.
69 * The machine powers down (or sleeps).
70 *
71 * Boot/Wake:
72 *
73 * - BootX sees the boot-image nvram variable containing the device and block number of the image,
74 * reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
75 * - BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
76 * in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
77 * that is in the kernel's __HIB section.
78 * - BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
79 * hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
80 * only code & data in that section is safe to call since all the other wired pages are still
81 * compressed in the image.
82 * - hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
83 * It uses the bitmaps to work out which pages can be uncompressed from the image to their final
84 * location directly, and copies those that can't to interim free pages. When the image has been
85 * completed, the copies are uncompressed, overwriting the wired image pages.
86 * hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
87 * is used to get pages into place for 64bit.
88 * - the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
89 * changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
90 * are removed from the software strutures, and the hash table is reinitialized.
91 * - After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
92 * of memory, using the polled mode driver, before other threads can run or any devices are turned on.
93 * This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
94 * for the remaining non wired pages.
95 * - The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
96 * (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
97 * is closed via bsd.
98 *
99 * Polled Mode I/O:
100 *
101 * IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
102 * registry, specifying an object of calls IOPolledInterface.
103 *
104 * Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
105 * partition) that the image is going to live, looking for polled interface properties. If it finds
106 * one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
107 * interfaces found are kept in an ordered list.
108 *
109 * There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
110 * few different contexts things happen in:
111 *
112 * - there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
113 * up and running) and after wake - this is safe to allocate memory and do anything. The device
114 * ignores sleep requests from that point since its a waste of time if it goes to sleep and
115 * immediately wakes back up for the image write.
116 *
117 * - there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
118 * immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
119 * from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
120 * used to flush and set the disk to sleep.
121 *
122 * - there is an Open/Close (AfterSleep) pair made around the image read operations that happen
123 * immediately after sleep. These can't block or allocate memory. This is happening after the platform
124 * expert has woken the low level bits of the system, but most of the I/O system has not. There is only
125 * one thread running.
126 *
127 * For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
128 * (prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
129 * the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
130 * Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
131 * that is called for the hardware to check for events, and complete the I/O via the callback.
132 * The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
133 * to restrict I/O ops.
134 */
3a60a9f5
A		 135
136#include <sys/systm.h>
137
138#include <IOKit/IOWorkLoop.h>
139#include <IOKit/IOCommandGate.h>
140#include <IOKit/IOTimerEventSource.h>
141#include <IOKit/IOPlatformExpert.h>
142#include <IOKit/IOKitDebug.h>
143#include <IOKit/IOTimeStamp.h>
144#include <IOKit/pwr_mgt/RootDomain.h>
145#include <IOKit/pwr_mgt/IOPMPrivate.h>
146#include <IOKit/IOMessage.h>
147#include <IOKit/IODeviceTreeSupport.h>
148#include <IOKit/IOBSD.h>
a39ff7e2 149#include <IOKit/IOKitKeysPrivate.h>
3a60a9f5
A		 150#include "RootDomainUserClient.h"
151#include <IOKit/pwr_mgt/IOPowerConnection.h>
152#include "IOPMPowerStateQueue.h"
153#include <IOKit/IOBufferMemoryDescriptor.h>
6d2010ae 154#include <IOKit/AppleKeyStoreInterface.h>
316670eb 155#include <libkern/crypto/aes.h>
3a60a9f5
A		 156
157#include <sys/uio.h>
158#include <sys/conf.h>
159#include <sys/stat.h>
160#include <sys/fcntl.h> // (FWRITE, ...)
3a60a9f5 161#include <sys/sysctl.h>
6d2010ae 162#include <sys/kdebug.h>
fe8ab488 163#include <stdint.h>
3a60a9f5
A		 164
165#include <IOKit/IOHibernatePrivate.h>
166#include <IOKit/IOPolledInterface.h>
167#include <IOKit/IONVRAM.h>
168#include "IOHibernateInternal.h"
39037602 169#include <vm/vm_protos.h>
3a60a9f5 170#include "IOKitKernelInternal.h"
6d2010ae
A		 171#include <pexpert/device_tree.h>
172
173#include <machine/pal_routines.h>
174#include <machine/pal_hibernate.h>
f427ee49 175#if defined(__i386__) || defined(__x86_64__)
39236c6e 176#include <i386/tsc.h>
39037602 177#include <i386/cpuid.h>
f427ee49
A		 178#include <vm/WKdm_new.h>
179#elif defined(__arm64__)
180#include <arm64/amcc_rorgn.h>
181#endif /* defined(__i386__) || defined(__x86_64__) */
5ba3f43e 182#include <san/kasan.h>
6d2010ae 183
f427ee49 184
0a7de745
A		 185extern "C" addr64_t kvtophys(vm_offset_t va);
186extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
3a60a9f5
A		 187
188/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
189
0a7de745
A		 190#define DISABLE_TRIM 0
191#define TRIM_DELAY 25000
39236c6e 192
0a7de745
A		 193extern unsigned int save_kdebug_enable;
194extern uint32_t gIOHibernateState;
195uint32_t gIOHibernateMode;
196static char gIOHibernateBootSignature[256 + 1];
197static char gIOHibernateFilename[MAXPATHLEN + 1];
f427ee49 198uint32_t gIOHibernateCount;
a39ff7e2 199
0a7de745 200static uuid_string_t gIOHibernateBridgeBootSessionUUIDString;
a39ff7e2 201
0a7de745
A		 202static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
203uint32_t gIOHibernateFreeTime = 0 * 1000; // max time to spend freeing pages (ms)
a991bd8d
A		 204
205enum {
206 HIB_COMPR_RATIO_ARM64 = (0xa5), // compression ~65%. Since we don't support retries we start higher.
207 HIB_COMPR_RATIO_INTEL = (0x80) // compression 50%
208};
209
210#if defined(__arm64__)
211static uint64_t gIOHibernateCompression = HIB_COMPR_RATIO_ARM64;
212#else
213static uint64_t gIOHibernateCompression = HIB_COMPR_RATIO_INTEL;
214#endif /* __arm64__ */
39037602 215boolean_t gIOHibernateStandbyDisabled;
3a60a9f5 216
0a7de745
A		 217static IODTNVRAM * gIOOptionsEntry;
218static IORegistryEntry * gIOChosenEntry;
3e170ce0 219
0a7de745
A		 220static const OSSymbol * gIOHibernateBootImageKey;
221static const OSSymbol * gIOHibernateBootSignatureKey;
222static const OSSymbol * gIOBridgeBootSessionUUIDKey;
3e170ce0 223
b0d623f7 224#if defined(__i386__) || defined(__x86_64__)
3e170ce0 225
0a7de745 226static const OSSymbol * gIOHibernateRTCVariablesKey;
060df5ea
A		 227static const OSSymbol * gIOHibernateBoot0082Key;
228static const OSSymbol * gIOHibernateBootNextKey;
0a7de745
A		 229static OSData * gIOHibernateBoot0082Data;
230static OSData * gIOHibernateBootNextData;
231static OSObject * gIOHibernateBootNextSave;
3e170ce0 232
3e170ce0 233#endif /* defined(__i386__) || defined(__x86_64__) */
3a60a9f5 234
316670eb 235static IOLock * gFSLock;
0a7de745 236uint32_t gFSState;
813fb2f6 237static thread_call_t gIOHibernateTrimCalloutEntry;
0a7de745
A		 238static IOPolledFileIOVars gFileVars;
239static IOHibernateVars gIOHibernateVars;
240static IOPolledFileCryptVars gIOHibernateCryptWakeContext;
241static hibernate_graphics_t _hibernateGraphics;
242static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
243static hibernate_statistics_t _hibernateStats;
244static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats;
245
246enum{
247 kFSIdle = 0,
248 kFSOpening = 2,
249 kFSOpened = 3,
250 kFSTimedOut = 4,
251 kFSTrimDelay = 5
316670eb
A		 252};
253
254static IOReturn IOHibernateDone(IOHibernateVars * vars);
3e170ce0
A		 255static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature);
256static void IOSetBootImageNVRAM(OSData * data);
5ba3f43e 257static void IOHibernateSystemPostWakeTrim(void * p1, void * p2);
316670eb 258
3a60a9f5
A		 259/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
260
3a60a9f5 261enum { kDefaultIOSize = 128 * 1024 };
22ba694c 262enum { kVideoMapSize = 80 * 1024 * 1024 };
3a60a9f5 263
3a60a9f5
A		 264/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
265
266// copy from phys addr to MD
267
268static IOReturn
269IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
0a7de745 270 IOByteCount offset, addr64_t bytes, IOByteCount length)
3a60a9f5 271{
0a7de745
A		 272 addr64_t srcAddr = bytes;
273 IOByteCount remaining;
3a60a9f5 274
0a7de745
A		 275 remaining = length = min(length, md->getLength() - offset);
276 while (remaining) { // (process another target segment?)
277 addr64_t dstAddr64;
278 IOByteCount dstLen;
3a60a9f5 279
0a7de745
A		 280 dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
281 if (!dstAddr64) {
282 break;
283 }
3a60a9f5 284
0a7de745
A		 285 // Clip segment length to remaining
286 if (dstLen > remaining) {
287 dstLen = remaining;
288 }
3a60a9f5
A		 289
290#if 1
0a7de745 291 bcopy_phys(srcAddr, dstAddr64, dstLen);
3a60a9f5 292#else
0a7de745
A		 293 copypv(srcAddr, dstAddr64, dstLen,
294 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
3a60a9f5 295#endif
0a7de745
A		 296 srcAddr += dstLen;
297 offset += dstLen;
298 remaining -= dstLen;
299 }
3a60a9f5 300
0a7de745 301 assert(!remaining);
3a60a9f5 302
0a7de745 303 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
3a60a9f5
A		 304}
305
306// copy from MD to phys addr
307
308static IOReturn
309IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
0a7de745 310 IOByteCount offset, addr64_t bytes, IOByteCount length)
3a60a9f5 311{
0a7de745
A		 312 addr64_t dstAddr = bytes;
313 IOByteCount remaining;
3a60a9f5 314
0a7de745
A		 315 remaining = length = min(length, md->getLength() - offset);
316 while (remaining) { // (process another target segment?)
317 addr64_t srcAddr64;
318 IOByteCount dstLen;
3a60a9f5 319
0a7de745
A		 320 srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
321 if (!srcAddr64) {
322 break;
323 }
3a60a9f5 324
0a7de745
A		 325 // Clip segment length to remaining
326 if (dstLen > remaining) {
327 dstLen = remaining;
328 }
3a60a9f5
A		 329
330#if 1
0a7de745 331 bcopy_phys(srcAddr64, dstAddr, dstLen);
3a60a9f5 332#else
0a7de745
A		 333 copypv(srcAddr, dstAddr64, dstLen,
334 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
3a60a9f5 335#endif
0a7de745
A		 336 dstAddr += dstLen;
337 offset += dstLen;
338 remaining -= dstLen;
339 }
3a60a9f5 340
0a7de745 341 assert(!remaining);
3a60a9f5 342
0a7de745 343 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
3a60a9f5
A		 344}
345
346/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
347
348void
349hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
0a7de745 350 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
3a60a9f5 351{
0a7de745 352 count += ppnum;
f427ee49
A		 353
354 if (count > UINT_MAX) {
355 panic("hibernate_set_page_state ppnum");
356 }
357
0a7de745
A		 358 switch (kind) {
359 case kIOHibernatePageStateUnwiredSave:
360 // unwired save
361 for (; ppnum < count; ppnum++) {
f427ee49
A		 362 hibernate_page_bitset(page_list, FALSE, (uint32_t) ppnum);
363 hibernate_page_bitset(page_list_wired, TRUE, (uint32_t) ppnum);
0a7de745
A		 364 }
365 break;
366 case kIOHibernatePageStateWiredSave:
367 // wired save
368 for (; ppnum < count; ppnum++) {
f427ee49
A		 369 hibernate_page_bitset(page_list, FALSE, (uint32_t) ppnum);
370 hibernate_page_bitset(page_list_wired, FALSE, (uint32_t) ppnum);
0a7de745
A		 371 }
372 break;
373 case kIOHibernatePageStateFree:
374 // free page
375 for (; ppnum < count; ppnum++) {
f427ee49
A		 376 hibernate_page_bitset(page_list, TRUE, (uint32_t) ppnum);
377 hibernate_page_bitset(page_list_wired, TRUE, (uint32_t) ppnum);
0a7de745
A		 378 }
379 break;
380 default:
381 panic("hibernate_set_page_state");
382 }
3a60a9f5
A		 383}
384
f427ee49
A		 385static void
386hibernate_set_descriptor_page_state(IOHibernateVars *vars,
387 IOMemoryDescriptor *descriptor,
388 uint32_t kind,
389 uint32_t *pageCount)
390{
391 IOItemCount count;
392 addr64_t phys64;
393 IOByteCount segLen;
394 if (descriptor) {
395 for (count = 0;
396 (phys64 = descriptor->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
397 count += segLen) {
398 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
399 atop_64(phys64), atop_32(segLen),
400 kind);
401 *pageCount -= atop_32(segLen);
402 }
403 }
404}
405
3e170ce0 406static vm_offset_t
f427ee49 407hibernate_page_list_iterate(hibernate_page_list_t * list, ppnum_t * pPage)
3e170ce0 408{
f427ee49 409 uint32_t page = ((typeof(page)) * pPage);
0a7de745
A		 410 uint32_t count;
411 hibernate_bitmap_t * bitmap;
412
413 while ((bitmap = hibernate_page_bitmap_pin(list, &page))) {
414 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
415 if (!count) {
416 break;
417 }
418 page += count;
419 if (page <= bitmap->last_page) {
420 break;
421 }
422 }
423
424 *pPage = page;
425 if (bitmap) {
426 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
427 } else {
428 count = 0;
429 }
430
431 return count;
fe8ab488 432}
fe8ab488
A		 433
434/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
435
3a60a9f5
A		 436IOReturn
437IOHibernateSystemSleep(void)
438{
0a7de745
A		 439 IOReturn err;
440 OSData * nvramData;
441 OSObject * obj;
442 OSString * str;
443 OSNumber * num;
444 bool dsSSD, vmflush, swapPinned;
445 IOHibernateVars * vars;
446 uint64_t setFileSize = 0;
447
448 gIOHibernateState = kIOHibernateStateInactive;
449
450 gIOHibernateDebugFlags = 0;
451 if (kIOLogHibernate & gIOKitDebug) {
452 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
453 }
454
455 if (IOService::getPMRootDomain()->getHibernateSettings(
456 &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime)) {
457 if (kIOHibernateModeSleep & gIOHibernateMode) {
458 // default to discard clean for safe sleep
459 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
460 | kIOHibernateModeDiscardCleanActive);
461 }
462 }
463
464 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey))) {
465 if ((str = OSDynamicCast(OSString, obj))) {
466 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
2d21ac55 467 sizeof(gIOHibernateFilename));
0a7de745
A		 468 }
469 obj->release();
470 }
db609669 471
0a7de745
A		 472 if (!gIOHibernateMode || !gIOHibernateFilename[0]) {
473 return kIOReturnUnsupported;
474 }
3e170ce0 475
0a7de745 476 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
6d2010ae 477
0a7de745
A		 478 vars = IONew(IOHibernateVars, 1);
479 if (!vars) {
480 return kIOReturnNoMemory;
481 }
482 bzero(vars, sizeof(*vars));
6d2010ae 483
0a7de745
A		 484 IOLockLock(gFSLock);
485 if (!gIOHibernateTrimCalloutEntry) {
486 gIOHibernateTrimCalloutEntry = thread_call_allocate(&IOHibernateSystemPostWakeTrim, &gFSLock);
487 }
488 IOHibernateSystemPostWakeTrim(NULL, NULL);
489 thread_call_cancel(gIOHibernateTrimCalloutEntry);
490 if (kFSIdle != gFSState) {
491 HIBLOG("hibernate file busy\n");
492 IOLockUnlock(gFSLock);
493 IODelete(vars, IOHibernateVars, 1);
494 return kIOReturnBusy;
3e170ce0 495 }
0a7de745
A		 496 gFSState = kFSOpening;
497 IOLockUnlock(gFSLock);
498
499 swapPinned = false;
500 do{
501 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
f427ee49 502 HIBERNATION_SRC_BUFFER_SIZE, page_size);
0a7de745
A		 503
504 vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
505 ptoa_64(gIOHibernateHandoffPageCount), page_size);
506
507 if (!vars->srcBuffer || !vars->handoffBuffer) {
508 err = kIOReturnNoMemory;
509 break;
510 }
511
512 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey))) {
513 if ((num = OSDynamicCast(OSNumber, obj))) {
514 vars->fileMinSize = num->unsigned64BitValue();
515 }
516 obj->release();
517 }
518 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey))) {
519 if ((num = OSDynamicCast(OSNumber, obj))) {
520 vars->fileMaxSize = num->unsigned64BitValue();
521 }
522 obj->release();
523 }
524
525 boolean_t encryptedswap = true;
526 uint32_t pageCount;
527 AbsoluteTime startTime, endTime;
528 uint64_t nsec;
529
530 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
531 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
532 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
533
534 vmflush = ((kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
535 err = hibernate_alloc_page_lists(&vars->page_list,
536 &vars->page_list_wired,
537 &vars->page_list_pal);
538 if (KERN_SUCCESS != err) {
cb323159 539 HIBLOG("%s err, hibernate_alloc_page_lists return 0x%x\n", __FUNCTION__, err);
0a7de745
A		 540 break;
541 }
542
543 err = hibernate_pin_swap(TRUE);
544 if (KERN_SUCCESS != err) {
cb323159 545 HIBLOG("%s error, hibernate_pin_swap return 0x%x\n", __FUNCTION__, err);
0a7de745
A		 546 break;
547 }
548 swapPinned = true;
549
550 if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode)) {
551 hibernate_page_list_setall(vars->page_list,
552 vars->page_list_wired,
553 vars->page_list_pal,
554 true /* preflight */,
555 vmflush /* discard */,
556 &pageCount);
557 PE_Video consoleInfo;
558 bzero(&consoleInfo, sizeof(consoleInfo));
559 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
560
561 // estimate: 6% increase in pages compressed
562 // screen preview 2 images compressed 0%
563 setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
564 + vars->page_list->list_size
565 + (consoleInfo.v_width * consoleInfo.v_height * 8);
566 enum { setFileRound = 1024 * 1024ULL };
567 setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
568
569 HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
570 pageCount, (100ULL * gIOHibernateCompression) >> 8,
571 setFileSize, vars->fileMinSize);
572
573 if (!(kIOHibernateModeFileResize & gIOHibernateMode)
574 && (setFileSize < vars->fileMinSize)) {
575 setFileSize = vars->fileMinSize;
576 }
577 }
578
579 vars->volumeCryptKeySize = sizeof(vars->volumeCryptKey);
580 err = IOPolledFileOpen(gIOHibernateFilename,
581 (kIOPolledFileCreate | kIOPolledFileHibernate),
582 setFileSize, 0,
583 gIOHibernateCurrentHeader, sizeof(gIOHibernateCurrentHeader),
584 &vars->fileVars, &nvramData,
585 &vars->volumeCryptKey[0], &vars->volumeCryptKeySize);
586
587 if (KERN_SUCCESS != err) {
588 IOLockLock(gFSLock);
589 if (kFSOpening != gFSState) {
590 err = kIOReturnTimeout;
591 }
592 IOLockUnlock(gFSLock);
593 }
594
595 if (KERN_SUCCESS != err) {
596 HIBLOG("IOPolledFileOpen(%x)\n", err);
597 break;
598 }
599
600 // write extents for debug data usage in EFI
601 IOWriteExtentsToFile(vars->fileVars, kIOHibernateHeaderOpenSignature);
602
603 err = IOPolledFilePollersSetup(vars->fileVars, kIOPolledPreflightState);
604 if (KERN_SUCCESS != err) {
605 break;
606 }
607
608 clock_get_uptime(&startTime);
609 err = hibernate_setup(gIOHibernateCurrentHeader,
610 vmflush,
611 vars->page_list, vars->page_list_wired, vars->page_list_pal);
612 clock_get_uptime(&endTime);
613 SUB_ABSOLUTETIME(&endTime, &startTime);
614 absolutetime_to_nanoseconds(endTime, &nsec);
615
616 boolean_t haveSwapPin, hibFileSSD;
617 haveSwapPin = vm_swap_files_pinned();
618
619 hibFileSSD = (kIOPolledFileSSD & vars->fileVars->flags);
620
621 HIBLOG("hibernate_setup(%d) took %qd ms, swapPin(%d) ssd(%d)\n",
622 err, nsec / 1000000ULL,
623 haveSwapPin, hibFileSSD);
624 if (KERN_SUCCESS != err) {
625 break;
626 }
627
628 gIOHibernateStandbyDisabled = ((!haveSwapPin || !hibFileSSD));
629
630 dsSSD = ((0 != (kIOPolledFileSSD & vars->fileVars->flags))
631 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
632
633 if (dsSSD) {
634 gIOHibernateCurrentHeader->options |= kIOHibernateOptionSSD | kIOHibernateOptionColor;
635 } else {
636 gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
637 }
638
639
640#if defined(__i386__) || defined(__x86_64__)
641 if (vars->volumeCryptKeySize &&
642 (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey))) {
643 uintptr_t smcVars[2];
644 smcVars[0] = vars->volumeCryptKeySize;
645 smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
646
647 IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
648 bzero(smcVars, sizeof(smcVars));
649 }
6d2010ae 650#endif
0b4c1975 651
0b4c1975 652
0a7de745
A		 653 if (encryptedswap || vars->volumeCryptKeySize) {
654 gIOHibernateMode ^= kIOHibernateModeEncrypt;
655 }
3a60a9f5 656
0a7de745
A		 657 if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options) {
658 vars->videoAllocSize = kVideoMapSize;
659 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize, VM_KERN_MEMORY_IOKIT)) {
660 vars->videoMapping = 0;
661 }
662 }
3a60a9f5 663
0a7de745
A		 664 // generate crypt keys
665 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++) {
f427ee49 666 vars->wiredCryptKey[i] = ((uint8_t) random());
0a7de745
A		 667 }
668 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++) {
f427ee49 669 vars->cryptKey[i] = ((uint8_t) random());
0a7de745 670 }
3a60a9f5 671
0a7de745 672 // set nvram
3a60a9f5 673
0a7de745
A		 674 IOSetBootImageNVRAM(nvramData);
675 nvramData->release();
3a60a9f5 676
b0d623f7 677#if defined(__i386__) || defined(__x86_64__)
0c530ab8 678 {
0a7de745
A		 679 struct AppleRTCHibernateVars {
680 uint8_t signature[4];
681 uint32_t revision;
682 uint8_t booterSignature[20];
683 uint8_t wiredCryptKey[16];
684 };
685 AppleRTCHibernateVars rtcVars;
686 OSData * data;
687
688 rtcVars.signature[0] = 'A';
689 rtcVars.signature[1] = 'A';
690 rtcVars.signature[2] = 'P';
691 rtcVars.signature[3] = 'L';
692 rtcVars.revision = 1;
693 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
694
695 if (gIOChosenEntry
696 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOHibernateBootSignatureKey)))
697 && (sizeof(rtcVars.booterSignature) <= data->getLength())) {
698 bcopy(data->getBytesNoCopy(), &rtcVars.booterSignature[0], sizeof(rtcVars.booterSignature));
699 } else if (gIOHibernateBootSignature[0]) {
700 char c;
701 uint8_t value = 0;
702 uint32_t in, out, digits;
703 for (in = out = digits = 0;
704 (c = gIOHibernateBootSignature[in]) && (in < sizeof(gIOHibernateBootSignature));
705 in++) {
706 if ((c >= 'a') && (c <= 'f')) {
707 c -= 'a' - 10;
708 } else if ((c >= 'A') && (c <= 'F')) {
709 c -= 'A' - 10;
710 } else if ((c >= '0') && (c <= '9')) {
711 c -= '0';
712 } else {
713 if (c == '=') {
714 out = digits = value = 0;
715 }
716 continue;
717 }
f427ee49 718 value = ((uint8_t) ((value << 4) | c));
0a7de745
A		 719 if (digits & 1) {
720 rtcVars.booterSignature[out++] = value;
721 if (out >= sizeof(rtcVars.booterSignature)) {
722 break;
723 }
724 }
725 digits++;
726 }
727 }
d26ffc64 728#if DEBUG || DEVELOPMENT
0a7de745
A		 729 if (kIOLogHibernate & gIOKitDebug) {
730 IOKitKernelLogBuffer("H> rtc:",
731 &rtcVars, sizeof(rtcVars), &kprintf);
732 }
d26ffc64
A		 733#endif /* DEBUG || DEVELOPMENT */
734
0a7de745
A		 735 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
736 if (data) {
737 if (gIOHibernateRTCVariablesKey) {
738 IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
739 }
740 data->release();
060df5ea 741 }
0a7de745
A		 742 if (gIOChosenEntry && gIOOptionsEntry) {
743 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
744 if (data) {
745 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
746 }
747 // set BootNext
748 if (!gIOHibernateBoot0082Data) {
cb323159 749 OSData * fileData = NULL;
0a7de745
A		 750 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
751 if (data && data->getLength() >= 4) {
752 fileData = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-file-path"));
753 }
f427ee49 754 if (data && (data->getLength() <= UINT16_MAX)) {
0a7de745
A		 755 // AppleNVRAM_EFI_LOAD_OPTION
756 struct {
757 uint32_t Attributes;
758 uint16_t FilePathLength;
759 uint16_t Desc;
760 } loadOptionHeader;
761 loadOptionHeader.Attributes = 1;
f427ee49 762 loadOptionHeader.FilePathLength = ((uint16_t) data->getLength());
0a7de745
A		 763 loadOptionHeader.Desc = 0;
764 if (fileData) {
765 loadOptionHeader.FilePathLength -= 4;
766 loadOptionHeader.FilePathLength += fileData->getLength();
767 }
768 gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
769 if (gIOHibernateBoot0082Data) {
770 gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
771 if (fileData) {
772 gIOHibernateBoot0082Data->appendBytes(data->getBytesNoCopy(), data->getLength() - 4);
773 gIOHibernateBoot0082Data->appendBytes(fileData);
774 } else {
775 gIOHibernateBoot0082Data->appendBytes(data);
776 }
777 }
778 }
779 }
780 if (!gIOHibernateBootNextData) {
781 uint16_t bits = 0x0082;
782 gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
783 }
d26ffc64
A		 784
785#if DEBUG || DEVELOPMENT
0a7de745
A		 786 if (kIOLogHibernate & gIOKitDebug) {
787 IOKitKernelLogBuffer("H> bootnext:",
788 gIOHibernateBoot0082Data->getBytesNoCopy(), gIOHibernateBoot0082Data->getLength(), &kprintf);
789 }
d26ffc64 790#endif /* DEBUG || DEVELOPMENT */
0a7de745
A		 791 if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData) {
792 gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
793 gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
794 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
795 }
796 // BootNext
797 }
3e170ce0 798 }
3e170ce0 799#endif /* !i386 && !x86_64 */
0a7de745
A		 800 }while (false);
801
802 if (swapPinned) {
803 hibernate_pin_swap(FALSE);
804 }
805
806 IOLockLock(gFSLock);
807 if ((kIOReturnSuccess == err) && (kFSOpening != gFSState)) {
808 HIBLOG("hibernate file close due timeout\n");
809 err = kIOReturnTimeout;
810 }
811 if (kIOReturnSuccess == err) {
812 gFSState = kFSOpened;
813 gIOHibernateVars = *vars;
814 gFileVars = *vars->fileVars;
815 gFileVars.allocated = false;
816 gIOHibernateVars.fileVars = &gFileVars;
817 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
f427ee49 818 gIOHibernateCurrentHeader->kernVirtSlide = vm_kernel_slide;
0a7de745 819 gIOHibernateState = kIOHibernateStateHibernating;
d26ffc64
A		 820
821#if DEBUG || DEVELOPMENT
0a7de745
A		 822 if (kIOLogHibernate & gIOKitDebug) {
823 OSData * data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
824 if (data) {
825 uintptr_t * smcVars = (typeof(smcVars))data->getBytesNoCopy();
826 IOKitKernelLogBuffer("H> smc:",
827 (const void *)smcVars[1], smcVars[0], &kprintf);
828 }
829 }
d26ffc64 830#endif /* DEBUG || DEVELOPMENT */
0a7de745
A		 831 } else {
832 IOPolledFileIOVars * fileVars = vars->fileVars;
833 IOHibernateDone(vars);
834 IOPolledFileClose(&fileVars,
3e170ce0 835#if DISABLE_TRIM
0a7de745 836 0, NULL, 0, 0, 0);
3e170ce0 837#else
0a7de745 838 0, NULL, 0, sizeof(IOHibernateImageHeader), setFileSize);
3e170ce0 839#endif
0a7de745
A		 840 gFSState = kFSIdle;
841 }
842 IOLockUnlock(gFSLock);
316670eb 843
0a7de745
A		 844 if (vars->fileVars) {
845 IODelete(vars->fileVars, IOPolledFileIOVars, 1);
846 }
847 IODelete(vars, IOHibernateVars, 1);
3a60a9f5 848
0a7de745 849 return err;
3a60a9f5
A		 850}
851
852/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
853
3e170ce0
A		 854static void
855IOSetBootImageNVRAM(OSData * data)
856{
0a7de745
A		 857 IORegistryEntry * regEntry;
858
859 if (!gIOOptionsEntry) {
860 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
861 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
862 if (regEntry && !gIOOptionsEntry) {
863 regEntry->release();
864 }
865 }
866 if (gIOOptionsEntry && gIOHibernateBootImageKey) {
867 if (data) {
868 gIOOptionsEntry->setProperty(gIOHibernateBootImageKey, data);
d26ffc64 869#if DEBUG || DEVELOPMENT
0a7de745
A		 870 if (kIOLogHibernate & gIOKitDebug) {
871 IOKitKernelLogBuffer("H> boot-image:",
872 data->getBytesNoCopy(), data->getLength(), &kprintf);
873 }
d26ffc64 874#endif /* DEBUG || DEVELOPMENT */
0a7de745
A		 875 } else {
876 gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
f427ee49 877#if __x86_64__
0a7de745 878 gIOOptionsEntry->sync();
f427ee49
A		 879#else
880 if (gIOHibernateState == kIOHibernateStateWakingFromHibernate) {
881 // if we woke from hibernation, the booter may have changed the state of NVRAM, so force a sync
882 gIOOptionsEntry->sync();
883 }
884#endif
0a7de745 885 }
3e170ce0 886 }
3e170ce0
A		 887}
888
889/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
a39ff7e2 890/*
3e170ce0
A		 891 * Writes header to disk with signature, block size and file extents data.
892 * If there are more than 2 extents, then they are written on second block.
893 */
894static IOReturn
895IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature)
896{
0a7de745
A		 897 IOHibernateImageHeader hdr;
898 IOItemCount count;
899 IOReturn err = kIOReturnSuccess;
900 int rc;
901 IOPolledFileExtent * fileExtents;
902
903 fileExtents = (typeof(fileExtents))vars->fileExtents->getBytesNoCopy();
904
905 memset(&hdr, 0, sizeof(IOHibernateImageHeader));
906 count = vars->fileExtents->getLength();
907 if (count > sizeof(hdr.fileExtentMap)) {
908 hdr.fileExtentMapSize = count;
909 count = sizeof(hdr.fileExtentMap);
910 } else {
911 hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap);
912 }
913
914 bcopy(fileExtents, &hdr.fileExtentMap[0], count);
915
916 // copy file block extent list if larger than header
917 if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap)) {
918 count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap);
919 rc = kern_write_file(vars->fileRef, vars->blockSize,
920 (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)),
921 count, IO_SKIP_ENCRYPTION);
922 if (rc != 0) {
923 HIBLOG("kern_write_file returned %d\n", rc);
924 err = kIOReturnIOError;
925 goto exit;
926 }
927 }
928 hdr.signature = signature;
929 hdr.deviceBlockSize = vars->blockSize;
930
931 rc = kern_write_file(vars->fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION);
932 if (rc != 0) {
933 HIBLOG("kern_write_file returned %d\n", rc);
934 err = kIOReturnIOError;
935 goto exit;
936 }
3e170ce0
A		 937
938exit:
0a7de745 939 return err;
3e170ce0
A		 940}
941
3e170ce0
A		 942/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
943
2d21ac55
A		 944DECLARE_IOHIBERNATEPROGRESSALPHA
945
946static void
947ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
948{
0a7de745
A		 949 uint32_t rowBytes, pixelShift;
950 uint32_t x, y;
951 int32_t blob;
f427ee49
A		 952 uint32_t alpha, color, result;
953 uint8_t * out, in;
0a7de745
A		 954 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
955
956 rowBytes = display->rowBytes;
957 pixelShift = display->depth >> 4;
958 if (pixelShift < 1) {
959 return;
960 }
961
962 screen += ((display->width
963 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
964 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
965
966 for (y = 0; y < kIOHibernateProgressHeight; y++) {
967 out = screen + y * rowBytes;
968 for (blob = 0; blob < kIOHibernateProgressCount; blob++) {
969 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
970 for (x = 0; x < kIOHibernateProgressWidth; x++) {
971 alpha = gIOHibernateProgressAlpha[y][x];
972 result = color;
973 if (alpha) {
974 if (0xff != alpha) {
975 if (1 == pixelShift) {
976 in = *((uint16_t *)out) & 0x1f; // 16
f427ee49 977 in = ((uint8_t)(in << 3)) | ((uint8_t)(in >> 2));
0a7de745
A		 978 } else {
979 in = *((uint32_t *)out) & 0xff; // 32
980 }
981 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
982 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
983 }
984 if (1 == pixelShift) {
985 result >>= 3;
f427ee49 986 *((uint16_t *)out) = ((uint16_t)((result << 10) | (result << 5) | result)); // 16
0a7de745
A		 987 } else {
988 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
989 }
990 }
991 out += (1 << pixelShift);
992 }
993 out += (kIOHibernateProgressSpacing << pixelShift);
994 }
995 }
2d21ac55
A		 996}
997
998
999static void
1000ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
1001{
0a7de745
A		 1002 uint32_t rowBytes, pixelShift;
1003 uint32_t x, y;
1004 int32_t blob, lastBlob;
1005 uint32_t alpha, in, color, result;
1006 uint8_t * out;
1007 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1008
1009 pixelShift = display->depth >> 4;
1010 if (pixelShift < 1) {
1011 return;
1012 }
1013
1014 rowBytes = display->rowBytes;
1015
1016 screen += ((display->width
1017 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1018 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1019
1020 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
1021
1022 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
1023
1024 for (y = 0; y < kIOHibernateProgressHeight; y++) {
1025 out = screen + y * rowBytes;
1026 for (blob = firstBlob; blob <= lastBlob; blob++) {
1027 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
1028 for (x = 0; x < kIOHibernateProgressWidth; x++) {
1029 alpha = gIOHibernateProgressAlpha[y][x];
1030 result = color;
1031 if (alpha) {
1032 if (0xff != alpha) {
1033 in = display->progressSaveUnder[blob][saveindex[blob]++];
1034 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
1035 }
1036 if (1 == pixelShift) {
1037 result >>= 3;
f427ee49 1038 *((uint16_t *)out) = ((uint16_t)((result << 10) | (result << 5) | result)); // 16
0a7de745
A		 1039 } else {
1040 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1041 }
1042 }
1043 out += (1 << pixelShift);
1044 }
1045 out += (kIOHibernateProgressSpacing << pixelShift);
1046 }
1047 }
2d21ac55
A		 1048}
1049
1050/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1051
316670eb
A		 1052IOReturn
1053IOHibernateIOKitSleep(void)
1054{
0a7de745
A		 1055 IOReturn ret = kIOReturnSuccess;
1056 IOLockLock(gFSLock);
1057 if (kFSOpening == gFSState) {
1058 gFSState = kFSTimedOut;
1059 HIBLOG("hibernate file open timed out\n");
1060 ret = kIOReturnTimeout;
1061 }
1062 IOLockUnlock(gFSLock);
1063 return ret;
316670eb
A		 1064}
1065
1066/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1067
3a60a9f5
A		 1068IOReturn
1069IOHibernateSystemHasSlept(void)
1070{
0a7de745
A		 1071 IOReturn ret = kIOReturnSuccess;
1072 IOHibernateVars * vars = &gIOHibernateVars;
cb323159 1073 OSObject * obj = NULL;
0a7de745
A		 1074 OSData * data;
1075
1076 IOLockLock(gFSLock);
1077 if ((kFSOpened != gFSState) && gIOHibernateMode) {
1078 ret = kIOReturnTimeout;
1079 }
1080 IOLockUnlock(gFSLock);
1081 if (kIOReturnSuccess != ret) {
1082 return ret;
1083 }
1084
1085 if (gIOHibernateMode) {
1086 obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
1087 }
1088 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1089 if (obj && !vars->previewBuffer) {
1090 obj->release();
1091 }
f427ee49
A		 1092 if (vars->previewBuffer && (vars->previewBuffer->getLength() > UINT_MAX)) {
1093 OSSafeReleaseNULL(vars->previewBuffer);
1094 }
0a7de745
A		 1095
1096 vars->consoleMapping = NULL;
1097 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare())) {
1098 vars->previewBuffer->release();
cb323159 1099 vars->previewBuffer = NULL;
0a7de745
A		 1100 }
1101
1102 if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1103 && vars->previewBuffer
1104 && (data = OSDynamicCast(OSData,
1105 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey)))) {
1106 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
1107 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
1108
1109 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1110
1111 if (kIOHibernatePreviewUpdates & flags) {
1112 PE_Video consoleInfo;
1113 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1114
1115 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1116
f427ee49
A		 1117 graphicsInfo->width = (uint32_t) consoleInfo.v_width;
1118 graphicsInfo->height = (uint32_t) consoleInfo.v_height;
1119 graphicsInfo->rowBytes = (uint32_t) consoleInfo.v_rowBytes;
1120 graphicsInfo->depth = (uint32_t) consoleInfo.v_depth;
0a7de745
A		 1121 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1122
1123 HIBPRINT("video %p %d %d %d\n",
1124 vars->consoleMapping, graphicsInfo->depth,
1125 graphicsInfo->width, graphicsInfo->height);
1126 if (vars->consoleMapping) {
1127 ProgressInit(graphicsInfo, vars->consoleMapping,
1128 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1129 }
1130 }
1131 }
1132
1133 if (gIOOptionsEntry) {
f427ee49 1134#if __x86_64__
0a7de745 1135 gIOOptionsEntry->sync();
f427ee49
A		 1136#else
1137 if (gIOHibernateMode) {
1138 gIOOptionsEntry->sync();
1139 }
1140#endif
0a7de745
A		 1141 }
1142
1143 return ret;
3a60a9f5
A		 1144}
1145
1146/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1147
2a1bd2d3
A		 1148static const DeviceTreeNode *
1149MergeDeviceTree(const DeviceTreeNode * entry, IORegistryEntry * regEntry, OSSet * entriesToUpdate, vm_offset_t region_start, vm_size_t region_size)
6d2010ae 1150{
0a7de745 1151 DeviceTreeNodeProperty * prop;
2a1bd2d3 1152 const DeviceTreeNode * child;
0a7de745
A		 1153 IORegistryEntry * childRegEntry;
1154 const char * nameProp;
1155 unsigned int propLen, idx;
1156
2a1bd2d3
A		 1157 bool updateEntry = true;
1158 if (!regEntry) {
1159 updateEntry = false;
1160 } else if (entriesToUpdate && !entriesToUpdate->containsObject(regEntry)) {
1161 updateEntry = false;
1162 }
1163
0a7de745
A		 1164 prop = (DeviceTreeNodeProperty *) (entry + 1);
1165 for (idx = 0; idx < entry->nProperties; idx++) {
2a1bd2d3 1166 if (updateEntry && (0 != strcmp("name", prop->name))) {
0a7de745 1167 regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
6d2010ae 1168// HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
0a7de745
A		 1169 }
1170 prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
6d2010ae 1171 }
6d2010ae 1172
2a1bd2d3
A		 1173 if (entriesToUpdate) {
1174 entriesToUpdate->removeObject(regEntry);
1175 if (entriesToUpdate->getCount() == 0) {
1176 // we've updated all the entries we care about so we can stop
1177 return NULL;
1178 }
1179 }
1180
1181 child = (const DeviceTreeNode *) prop;
0a7de745 1182 for (idx = 0; idx < entry->nChildren; idx++) {
f427ee49
A		 1183 if (kSuccess != SecureDTGetPropertyRegion(child, "name", (void const **) &nameProp, &propLen,
1184 region_start, region_size)) {
0a7de745
A		 1185 panic("no name");
1186 }
1187 childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
6d2010ae 1188// HIBPRINT("%s == %p\n", nameProp, childRegEntry);
2a1bd2d3
A		 1189 child = MergeDeviceTree(child, childRegEntry, entriesToUpdate, region_start, region_size);
1190 if (!child) {
1191 // the recursive call updated the last entry we cared about, so we can stop
1192 break;
1193 }
0a7de745
A		 1194 }
1195 return child;
6d2010ae
A		 1196}
1197
3a60a9f5
A		 1198IOReturn
1199IOHibernateSystemWake(void)
1200{
0a7de745
A		 1201 if (kFSOpened == gFSState) {
1202 IOPolledFilePollersClose(gIOHibernateVars.fileVars, kIOPolledPostflightState);
1203 IOHibernateDone(&gIOHibernateVars);
1204 } else {
1205 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1206 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1207 }
f427ee49
A		 1208
1209 if (gIOOptionsEntry && gIOHibernateBootImageKey) {
1210 // if we got this far, clear boot-image
1211 // we don't need to sync immediately; the booter should have already removed this entry
1212 // we just want to make sure that if anyone syncs nvram after this point, we don't re-write
1213 // a stale boot-image value
1214 gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
1215 }
1216
0a7de745 1217 return kIOReturnSuccess;
316670eb 1218}
3a60a9f5 1219
316670eb
A		 1220static IOReturn
1221IOHibernateDone(IOHibernateVars * vars)
1222{
0a7de745
A		 1223 IOReturn err;
1224 OSData * data;
1225
1226 hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
1227
1228 if (vars->videoMapping) {
1229 if (vars->videoMapSize) {
1230 // remove mappings
1231 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1232 }
1233 if (vars->videoAllocSize) {
1234 // dealloc range
1235 kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
1236 }
1237 }
1238
1239 if (vars->previewBuffer) {
1240 vars->previewBuffer->release();
cb323159 1241 vars->previewBuffer = NULL;
0a7de745
A		 1242 }
1243
1244 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
1245 IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
1246 gIOHibernateCurrentHeader->options, 32);
1247 } else {
1248 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1249 }
1250
1251 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1252 && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus)) {
1253 IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
1254 &gIOHibernateGraphicsInfo->gfxStatus,
1255 sizeof(gIOHibernateGraphicsInfo->gfxStatus));
1256 } else {
1257 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1258 }
1259
1260 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
3a60a9f5 1261
b0d623f7 1262#if defined(__i386__) || defined(__x86_64__)
060df5ea 1263 IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
6d2010ae 1264 IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
2d21ac55
A		 1265
1266 /*
1267 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1268 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1269 */
1270 if (gIOOptionsEntry) {
0a7de745
A		 1271 if (gIOHibernateRTCVariablesKey) {
1272 if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
1273 gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
1274 }
060df5ea 1275 }
060df5ea 1276
0a7de745
A		 1277 if (gIOHibernateBootNextKey) {
1278 if (gIOHibernateBootNextSave) {
1279 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
1280 gIOHibernateBootNextSave->release();
1281 gIOHibernateBootNextSave = NULL;
1282 } else {
1283 gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
1284 }
1285 }
1286 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) {
1287 gIOOptionsEntry->sync();
2d21ac55
A		 1288 }
1289 }
0c530ab8 1290#endif
3a60a9f5 1291
0a7de745
A		 1292 if (vars->srcBuffer) {
1293 vars->srcBuffer->release();
1294 }
f427ee49 1295
f427ee49 1296
0a7de745
A		 1297 bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
1298 if (vars->handoffBuffer) {
1299 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
1300 IOHibernateHandoff * handoff;
1301 bool done = false;
1302 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
1303 !done;
1304 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount]) {
1305 HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
f427ee49 1306 uint8_t * __unused data = &handoff->data[0];
0a7de745
A		 1307 switch (handoff->type) {
1308 case kIOHibernateHandoffTypeEnd:
1309 done = true;
1310 break;
1311
1312 case kIOHibernateHandoffTypeDeviceTree:
2a1bd2d3 1313 {
f427ee49 1314#if defined(__i386__) || defined(__x86_64__)
2a1bd2d3
A		 1315 // On Intel, process the entirety of the passed in device tree
1316 OSSet * entriesToUpdate = NULL;
1317#elif defined(__arm64__)
1318 // On ARM, only allow hibernation to update specific entries
1319 const char *mergePaths[] = {
1320 kIODeviceTreePlane ":/chosen/boot-object-manifests",
1321 kIODeviceTreePlane ":/chosen/secure-boot-hashes",
1322 };
1323 const size_t mergePathCount = sizeof(mergePaths) / sizeof(mergePaths[0]);
1324 OSSet * entriesToUpdate = OSSet::withCapacity(mergePathCount);
1325 for (size_t i = 0; i < mergePathCount; i++) {
1326 IORegistryEntry *entry = IORegistryEntry::fromPath(mergePaths[i]);
1327 if (!entry) {
1328 panic("failed to find %s in IORegistry", mergePaths[i]);
1329 }
1330 entriesToUpdate->setObject(entry);
1331 OSSafeReleaseNULL(entry);
1332 }
f427ee49 1333#endif
2a1bd2d3
A		 1334 MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot(), entriesToUpdate,
1335 (vm_offset_t)data, (vm_size_t)handoff->bytecount);
1336 OSSafeReleaseNULL(entriesToUpdate);
0a7de745 1337 break;
2a1bd2d3 1338 }
0a7de745
A		 1339
1340 case kIOHibernateHandoffTypeKeyStore:
6d2010ae 1341#if defined(__i386__) || defined(__x86_64__)
0a7de745
A		 1342 {
1343 IOBufferMemoryDescriptor *
1344 md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
1345 if (md) {
1346 IOSetKeyStoreData(md);
1347 }
1348 }
6d2010ae 1349#endif
0a7de745 1350 break;
a39ff7e2 1351
0a7de745
A		 1352 default:
1353 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
1354 break;
1355 }
1356 }
a39ff7e2 1357#if defined(__i386__) || defined(__x86_64__)
0a7de745
A		 1358 if (vars->volumeCryptKeySize) {
1359 IOBufferMemoryDescriptor *
1360 bmd = IOBufferMemoryDescriptor::withBytes(&vars->volumeCryptKey[0],
1361 vars->volumeCryptKeySize, kIODirectionOutIn);
1362 if (!bmd) {
1363 panic("IOBufferMemoryDescriptor");
1364 }
1365 IOSetAPFSKeyStoreData(bmd);
1366 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
1367 }
a39ff7e2 1368#endif
0a7de745
A		 1369 }
1370 vars->handoffBuffer->release();
6d2010ae 1371 }
3a60a9f5 1372
0a7de745
A		 1373 if (gIOChosenEntry
1374 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey)))
1375 && (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength())) {
1376 bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0],
1377 sizeof(gIOHibernateBridgeBootSessionUUIDString));
1378 }
a39ff7e2 1379
0a7de745
A		 1380 if (vars->hwEncrypt) {
1381 err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, NULL, 0);
1382 HIBLOG("IOPolledFilePollersSetEncryptionKey(0,%x)\n", err);
1383 }
a39ff7e2 1384
0a7de745 1385 bzero(vars, sizeof(*vars));
3a60a9f5
A		 1386
1387// gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
f427ee49 1388 gIOHibernateCount++;
3a60a9f5 1389
0a7de745 1390 return kIOReturnSuccess;
3a60a9f5
A		 1391}
1392
5ba3f43e 1393static void
813fb2f6 1394IOHibernateSystemPostWakeTrim(void * p1, void * p2)
3a60a9f5 1395{
0a7de745
A		 1396 // invalidate & close the image file
1397 if (p1) {
1398 IOLockLock(gFSLock);
1399 }
1400 if (kFSTrimDelay == gFSState) {
1401 IOPolledFileIOVars * vars = &gFileVars;
1402 IOPolledFileClose(&vars,
39236c6e 1403#if DISABLE_TRIM
0a7de745 1404 0, NULL, 0, 0, 0);
39236c6e 1405#else
0a7de745
A		 1406 0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader),
1407 sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize);
39236c6e 1408#endif
0a7de745
A		 1409 gFSState = kFSIdle;
1410 }
1411 if (p1) {
1412 IOLockUnlock(gFSLock);
1413 }
813fb2f6 1414}
fe8ab488 1415
813fb2f6 1416IOReturn
5ba3f43e 1417IOHibernateSystemPostWake(bool now)
813fb2f6 1418{
0a7de745 1419 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
cb323159 1420 IOSetBootImageNVRAM(NULL);
0a7de745
A		 1421
1422 IOLockLock(gFSLock);
1423 if (kFSTrimDelay == gFSState) {
1424 thread_call_cancel(gIOHibernateTrimCalloutEntry);
1425 IOHibernateSystemPostWakeTrim(NULL, NULL);
1426 } else if (kFSOpened != gFSState) {
1427 gFSState = kFSIdle;
1428 } else {
1429 gFSState = kFSTrimDelay;
1430 if (now) {
1431 thread_call_cancel(gIOHibernateTrimCalloutEntry);
1432 IOHibernateSystemPostWakeTrim(NULL, NULL);
1433 } else {
1434 AbsoluteTime deadline;
1435 clock_interval_to_deadline(TRIM_DELAY, kMillisecondScale, &deadline );
1436 thread_call_enter1_delayed(gIOHibernateTrimCalloutEntry, NULL, deadline);
1437 }
1438 }
1439 IOLockUnlock(gFSLock);
1440
1441 return kIOReturnSuccess;
3a60a9f5
A		 1442}
1443
0a7de745
A		 1444uint32_t
1445IOHibernateWasScreenLocked(void)
316670eb 1446{
0a7de745
A		 1447 uint32_t ret = 0;
1448 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry) {
1449 OSData *
1450 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
1451 if (data) {
1452 ret = ((uint32_t *)data->getBytesNoCopy())[0];
1453 gIOChosenEntry->setProperty(kIOBooterScreenLockStateKey, data);
1454 }
1455 } else {
1456 gIOChosenEntry->removeProperty(kIOBooterScreenLockStateKey);
1457 }
1458
1459 return ret;
316670eb
A		 1460}
1461
3a60a9f5
A		 1462/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1463
a39ff7e2 1464SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
c3c9b80d 1465 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_LOCKED,
0a7de745 1466 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
a39ff7e2 1467SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
c3c9b80d 1468 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_LOCKED,
0a7de745 1469 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
a39ff7e2 1470SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
c3c9b80d 1471 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_LOCKED,
0a7de745 1472 &gIOHibernateMode, 0, "");
39236c6e 1473SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
c3c9b80d 1474 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED,
0a7de745 1475 &_hibernateStats, hibernate_statistics_t, "");
c3c9b80d
A		 1476SYSCTL_OID_MANUAL(_kern_bridge, OID_AUTO, bootsessionuuid,
1477 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1478 gIOHibernateBridgeBootSessionUUIDString, sizeof(gIOHibernateBridgeBootSessionUUIDString),
1479 sysctl_handle_string, "A", "");
39236c6e
A		 1480
1481SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
c3c9b80d 1482 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_ANYBODY,
0a7de745 1483 &_hibernateStats.graphicsReadyTime, 0, "");
39236c6e 1484SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
c3c9b80d 1485 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_ANYBODY,
0a7de745 1486 &_hibernateStats.wakeNotificationTime, 0, "");
39236c6e 1487SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
c3c9b80d 1488 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_ANYBODY,
0a7de745 1489 &_hibernateStats.lockScreenReadyTime, 0, "");
39236c6e 1490SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
c3c9b80d 1491 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_ANYBODY,
0a7de745 1492 &_hibernateStats.hidReadyTime, 0, "");
39236c6e 1493
f427ee49 1494SYSCTL_UINT(_kern, OID_AUTO, hibernatecount,
c3c9b80d 1495 CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_ANYBODY,
f427ee49
A		 1496 &gIOHibernateCount, 0, "");
1497
1498/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1499
1500static int
1501hibernate_set_preview SYSCTL_HANDLER_ARGS
1502{
1503#pragma unused(oidp, arg1, arg2)
1504
1505 if (!IOTaskHasEntitlement(current_task(), kIOHibernateSetPreviewEntitlementKey)) {
1506 return EPERM;
1507 }
1508
1509 if ((req->newptr == USER_ADDR_NULL) || (!req->newlen)) {
1510 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewBufferKey);
1511 return 0;
1512 }
1513
1514 size_t rounded_size = round_page(req->newlen);
1515 IOBufferMemoryDescriptor *md = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, rounded_size, page_size);
1516 if (!md) {
1517 return ENOMEM;
1518 }
1519
1520 uint8_t *bytes = (uint8_t *)md->getBytesNoCopy();
1521 int error = SYSCTL_IN(req, bytes, req->newlen);
1522 if (error) {
1523 md->release();
1524 return error;
1525 }
1526
1527 IOService::getPMRootDomain()->setProperty(kIOHibernatePreviewBufferKey, md);
1528 md->release();
1529
1530 return 0;
1531}
1532
1533SYSCTL_PROC(_kern, OID_AUTO, hibernatepreview,
1534 CTLTYPE_OPAQUE | CTLFLAG_WR | CTLFLAG_LOCKED | CTLFLAG_ANYBODY, NULL, 0,
1535 hibernate_set_preview, "S", "");
1536
1537/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1538
3a60a9f5
A		 1539void
1540IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1541{
0a7de745
A		 1542 gIOHibernateBootImageKey = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1543 gIOHibernateBootSignatureKey = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
1544 gIOBridgeBootSessionUUIDKey = OSSymbol::withCStringNoCopy(kIOBridgeBootSessionUUIDKey);
3e170ce0
A		 1545
1546#if defined(__i386__) || defined(__x86_64__)
0a7de745
A		 1547 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1548 gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
1549 gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
1550 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
3e170ce0
A		 1551#endif /* defined(__i386__) || defined(__x86_64__) */
1552
0a7de745
A		 1553 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
1554 if (data) {
1555 rootDomain->setProperty(kIOHibernateStateKey, data);
1556 data->release();
1557 }
1558
1559 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename))) {
1560 gIOHibernateMode = kIOHibernateModeOn;
1561 } else {
1562 gIOHibernateFilename[0] = 0;
1563 }
1564
0a7de745
A		 1565 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1566
1567 if (gIOChosenEntry
1568 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey)))
1569 && (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength())) {
1570 sysctl_register_oid(&sysctl__kern_bridge_bootsessionuuid);
1571 bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0], sizeof(gIOHibernateBridgeBootSessionUUIDString));
1572 }
1573
1574 gFSLock = IOLockAlloc();
f427ee49 1575 gIOHibernateCount = 0;
3a60a9f5
A		 1576}
1577
3a60a9f5
A		 1578/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1579
a39ff7e2 1580static IOReturn
f427ee49 1581IOHibernatePolledFileWrite(IOHibernateVars * vars,
0a7de745
A		 1582 const uint8_t * bytes, IOByteCount size,
1583 IOPolledFileCryptVars * cryptvars)
6d2010ae 1584{
0a7de745 1585 IOReturn err;
6d2010ae 1586
f427ee49
A		 1587
1588 err = IOPolledFileWrite(vars->fileVars, bytes, size, cryptvars);
0a7de745
A		 1589 if ((kIOReturnSuccess == err) && hibernate_should_abort()) {
1590 err = kIOReturnAborted;
1591 }
6d2010ae 1592
f427ee49 1593
0a7de745 1594 return err;
6d2010ae
A		 1595}
1596
3e170ce0 1597/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
6d2010ae 1598
2d21ac55 1599extern "C" uint32_t
3a60a9f5
A		 1600hibernate_write_image(void)
1601{
0a7de745
A		 1602 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
1603 IOHibernateVars * vars = &gIOHibernateVars;
1604 IOPolledFileExtent * fileExtents;
1605
f427ee49 1606#if !defined(__arm64__)
0a7de745 1607 _static_assert_1_arg(sizeof(IOHibernateImageHeader) == 512);
f427ee49 1608#endif /* !defined(__arm64__) */
0a7de745
A		 1609
1610 uint32_t pageCount, pagesDone;
1611 IOReturn err;
f427ee49
A		 1612 ppnum_t ppnum, page;
1613 vm_offset_t count;
0a7de745
A		 1614 uint8_t * src;
1615 uint8_t * data;
1616 uint8_t * compressed;
1617 uint8_t * scratch;
1618 IOByteCount pageCompressedSize;
1619 uint64_t compressedSize, uncompressedSize;
1620 uint64_t image1Size = 0;
1621 uint32_t bitmap_size;
1622 bool iterDone, pollerOpen, needEncrypt;
0a7de745
A		 1623 int wkresult;
1624 uint32_t tag;
1625 uint32_t pageType;
1626 uint32_t pageAndCount[2];
1627 addr64_t phys64;
1628 IOByteCount segLen;
f427ee49 1629 uint32_t restore1Sum = 0, sum = 0, sum1 = 0, sum2 = 0;
0a7de745
A		 1630 uintptr_t hibernateBase;
1631 uintptr_t hibernateEnd;
1632
1633 AbsoluteTime startTime, endTime;
1634 AbsoluteTime allTime, compTime;
1635 uint64_t compBytes;
1636 uint64_t nsec;
f427ee49
A		 1637 uint64_t lastProgressStamp = 0;
1638 uint64_t progressStamp;
0a7de745
A		 1639 uint32_t blob, lastBlob = (uint32_t) -1L;
1640
1641 uint32_t wiredPagesEncrypted;
1642 uint32_t dirtyPagesEncrypted;
1643 uint32_t wiredPagesClear;
1644 uint32_t svPageCount;
1645 uint32_t zvPageCount;
1646
1647 IOPolledFileCryptVars _cryptvars;
cb323159 1648 IOPolledFileCryptVars * cryptvars = NULL;
0a7de745
A		 1649
1650 wiredPagesEncrypted = 0;
1651 dirtyPagesEncrypted = 0;
1652 wiredPagesClear = 0;
1653 svPageCount = 0;
1654 zvPageCount = 0;
1655
1656 if (!vars->fileVars
1657 || !vars->fileVars->pollers
1658 || !(kIOHibernateModeOn & gIOHibernateMode)) {
1659 return kIOHibernatePostWriteSleep;
1660 }
1661
f427ee49
A		 1662
1663#if !defined(__arm64__)
0a7de745
A		 1664 if (kIOHibernateModeSleep & gIOHibernateMode) {
1665 kdebug_enable = save_kdebug_enable;
1666 }
f427ee49
A		 1667#endif /* !defined(__arm64__) */
1668
1669 pal_hib_write_hook();
0a7de745
A		 1670
1671 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START);
1672 IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
1673
2d21ac55 1674#if CRYPTO
0a7de745
A		 1675 // encryption data. "iv" is the "initial vector".
1676 if (kIOHibernateModeEncrypt & gIOHibernateMode) {
1677 static const unsigned char first_iv[AES_BLOCK_SIZE]
1678 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
1679 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
1680
1681 cryptvars = &gIOHibernateCryptWakeContext;
1682 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1683 aes_encrypt_key(vars->cryptKey,
1684 kIOHibernateAESKeySize,
1685 &cryptvars->ctx.encrypt);
1686 aes_decrypt_key(vars->cryptKey,
1687 kIOHibernateAESKeySize,
1688 &cryptvars->ctx.decrypt);
1689
1690 cryptvars = &_cryptvars;
1691 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1692 for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++) {
1693 vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
1694 }
1695 aes_encrypt_key(vars->wiredCryptKey,
1696 kIOHibernateAESKeySize,
1697 &cryptvars->ctx.encrypt);
1698
1699 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1700 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1701 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
1702 }
2d21ac55 1703#endif /* CRYPTO */
3a60a9f5 1704
0a7de745
A		 1705 hibernate_page_list_setall(vars->page_list,
1706 vars->page_list_wired,
1707 vars->page_list_pal,
1708 false /* !preflight */,
1709 /* discard_all */
1710 ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
1711 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
1712 &pageCount);
3a60a9f5 1713
0a7de745 1714 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
3a60a9f5 1715
0a7de745 1716 fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy();
3a60a9f5
A		 1717
1718#if 0
0a7de745
A		 1719 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
1720 for (page = 0; page < count; page++) {
1721 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
1722 fileExtents[page].start, fileExtents[page].length,
1723 fileExtents[page].start + fileExtents[page].length);
1724 }
3a60a9f5
A		 1725#endif
1726
0a7de745
A		 1727 needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
1728 AbsoluteTime_to_scalar(&compTime) = 0;
1729 compBytes = 0;
1730
1731 clock_get_uptime(&allTime);
1732 IOService::getPMRootDomain()->pmStatsRecordEvent(
1733 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
1734 do{
1735 compressedSize = 0;
1736 uncompressedSize = 0;
1737 svPageCount = 0;
1738 zvPageCount = 0;
1739
1740 IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
1741
1742 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
1743 ml_get_interrupts_enabled());
1744 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState,
1745 // abortable if not low battery
1746 !IOService::getPMRootDomain()->mustHibernate());
1747 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
1748 pollerOpen = (kIOReturnSuccess == err);
1749 if (!pollerOpen) {
1750 break;
1751 }
1752
1753 if (vars->volumeCryptKeySize) {
1754 err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
1755 HIBLOG("IOPolledFilePollersSetEncryptionKey(%x)\n", err);
1756 vars->hwEncrypt = (kIOReturnSuccess == err);
1757 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
1758 if (vars->hwEncrypt) {
1759 header->options |= kIOHibernateOptionHWEncrypt;
1760 }
1761 }
1762
1763 // copy file block extent list if larger than header
1764
1765 count = vars->fileVars->fileExtents->getLength();
1766 if (count > sizeof(header->fileExtentMap)) {
1767 count -= sizeof(header->fileExtentMap);
f427ee49 1768 err = IOHibernatePolledFileWrite(vars,
0a7de745
A		 1769 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
1770 if (kIOReturnSuccess != err) {
1771 break;
1772 }
1773 }
1774
0a7de745
A		 1775 // copy out restore1 code
1776
1777 for (count = 0;
1778 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1779 count += segLen) {
1780 for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++) {
f427ee49 1781 gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64_ppnum(phys64) + pagesDone;
0a7de745
A		 1782 }
1783 }
1784
f427ee49
A		 1785 hibernateBase = HIB_BASE; /* Defined in PAL headers */
1786 hibernateEnd = (segHIBB + segSizeHIB);
1787
0a7de745
A		 1788 page = atop_32(kvtophys(hibernateBase));
1789 count = atop_32(round_page(hibernateEnd) - hibernateBase);
f427ee49
A		 1790 uintptr_t entrypoint = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
1791 uintptr_t stack = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
1792 if ((count > UINT_MAX) || (entrypoint > UINT_MAX) || (stack > UINT_MAX)) {
1793 panic("malformed kernel layout");
1794 }
1795 header->restore1CodePhysPage = (ppnum_t) page;
0a7de745 1796 header->restore1CodeVirt = hibernateBase;
f427ee49
A		 1797 header->restore1PageCount = (uint32_t) count;
1798 header->restore1CodeOffset = (uint32_t) entrypoint;
1799 header->restore1StackOffset = (uint32_t) stack;
0a7de745
A		 1800
1801 if (uuid_parse(&gIOHibernateBridgeBootSessionUUIDString[0], &header->bridgeBootSessionUUID[0])) {
1802 bzero(&header->bridgeBootSessionUUID[0], sizeof(header->bridgeBootSessionUUID));
1803 }
1804
1805 // sum __HIB seg, with zeros for the stack
1806 src = (uint8_t *) trunc_page(hibernateBase);
1807 for (page = 0; page < count; page++) {
1808 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0])) {
f427ee49 1809 restore1Sum += hibernate_sum_page(src, (uint32_t) (header->restore1CodeVirt + page));
0a7de745
A		 1810 } else {
1811 restore1Sum += 0x00000000;
1812 }
1813 src += page_size;
1814 }
1815 sum1 = restore1Sum;
1816
1817 // write the __HIB seg, with zeros for the stack
1818
1819 src = (uint8_t *) trunc_page(hibernateBase);
1820 count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
1821 if (count) {
f427ee49 1822 err = IOHibernatePolledFileWrite(vars, src, count, cryptvars);
0a7de745
A		 1823 if (kIOReturnSuccess != err) {
1824 break;
1825 }
1826 }
f427ee49 1827 err = IOHibernatePolledFileWrite(vars,
cb323159 1828 (uint8_t *) NULL,
0a7de745
A		 1829 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
1830 cryptvars);
1831 if (kIOReturnSuccess != err) {
1832 break;
1833 }
1834 src = &gIOHibernateRestoreStackEnd[0];
1835 count = round_page(hibernateEnd) - ((uintptr_t) src);
1836 if (count) {
f427ee49 1837 err = IOHibernatePolledFileWrite(vars, src, count, cryptvars);
0a7de745
A		 1838 if (kIOReturnSuccess != err) {
1839 break;
1840 }
1841 }
1842
1843 if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) {
1844 vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
1845 vars->fileVars->encryptEnd = UINT64_MAX;
1846 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1847 }
1848
1849 // write the preview buffer
1850
1851 if (vars->previewBuffer) {
1852 ppnum = 0;
1853 count = 0;
1854 do{
1855 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
f427ee49
A		 1856 pageAndCount[0] = atop_64_ppnum(phys64);
1857 pageAndCount[1] = atop_64_ppnum(segLen);
1858 err = IOHibernatePolledFileWrite(vars,
0a7de745
A		 1859 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1860 cryptvars);
1861 if (kIOReturnSuccess != err) {
1862 break;
1863 }
1864 count += segLen;
1865 ppnum += sizeof(pageAndCount);
1866 }while (phys64);
1867 if (kIOReturnSuccess != err) {
1868 break;
1869 }
1870
1871 src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
316670eb
A		 1872
1873 ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
1874
f427ee49 1875 count = (uint32_t) vars->previewBuffer->getLength();
0a7de745 1876
f427ee49
A		 1877 header->previewPageListSize = ((uint32_t) ppnum);
1878 header->previewSize = ((uint32_t) (count + ppnum));
0a7de745
A		 1879
1880 for (page = 0; page < count; page += page_size) {
1881 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
f427ee49 1882 sum1 += hibernate_sum_page(src + page, atop_64_ppnum(phys64));
f427ee49
A		 1883 }
1884 if (kIOReturnSuccess != err) {
1885 break;
0a7de745 1886 }
f427ee49 1887 err = IOHibernatePolledFileWrite(vars, src, count, cryptvars);
0a7de745
A		 1888 if (kIOReturnSuccess != err) {
1889 break;
1890 }
1891 }
1892
1893 // mark areas for no save
f427ee49
A		 1894 hibernate_set_descriptor_page_state(vars, IOPolledFileGetIOBuffer(vars->fileVars),
1895 kIOHibernatePageStateFree, &pageCount);
1896 hibernate_set_descriptor_page_state(vars, vars->srcBuffer,
1897 kIOHibernatePageStateFree, &pageCount);
0a7de745
A		 1898
1899 // copy out bitmap of pages available for trashing during restore
1900
1901 bitmap_size = vars->page_list_wired->list_size;
1902 src = (uint8_t *) vars->page_list_wired;
f427ee49 1903 err = IOHibernatePolledFileWrite(vars, src, bitmap_size, cryptvars);
0a7de745
A		 1904 if (kIOReturnSuccess != err) {
1905 break;
1906 }
1907
1908 // mark more areas for no save, but these are not available
1909 // for trashing during restore
1910
1911 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
1912
f427ee49
A		 1913#if defined(__i386__) || defined(__x86_64__)
1914 // __HIB is explicitly saved above so we don't have to save it again
0a7de745
A		 1915 page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
1916 count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
1917 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1918 page, count,
1919 kIOHibernatePageStateFree);
1920 pageCount -= count;
f427ee49
A		 1921#elif defined(__arm64__)
1922 // the segments described in IOHibernateHibSegInfo are stored directly in the
1923 // hibernation file, so they don't need to be saved again
1924 extern unsigned long gPhysBase, gPhysSize;
1925 for (size_t i = 0; i < NUM_HIBSEGINFO_SEGMENTS; i++) {
1926 page = segInfo->segments[i].physPage;
1927 count = segInfo->segments[i].pageCount;
1928 uint64_t physAddr = ptoa_64(page);
1929 uint64_t size = ptoa_64(count);
1930 if (size &&
1931 (physAddr >= gPhysBase) &&
1932 (physAddr + size <= gPhysBase + gPhysSize)) {
0a7de745 1933 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
f427ee49 1934 page, count,
0a7de745 1935 kIOHibernatePageStateFree);
f427ee49 1936 pageCount -= count;
0a7de745
A		 1937 }
1938 }
f427ee49
A		 1939#else
1940#error unimplemented
1941#endif
0a7de745 1942
f427ee49
A		 1943 hibernate_set_descriptor_page_state(vars, vars->previewBuffer,
1944 kIOHibernatePageStateFree, &pageCount);
1945 hibernate_set_descriptor_page_state(vars, vars->handoffBuffer,
1946 kIOHibernatePageStateFree, &pageCount);
6d2010ae 1947
5ba3f43e
A		 1948#if KASAN
1949 vm_size_t shadow_pages_free = atop_64(shadow_ptop) - atop_64(shadow_pnext);
1950
1951 /* no need to save unused shadow pages */
1952 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
0a7de745
A		 1953 atop_64(shadow_pnext),
1954 shadow_pages_free,
1955 kIOHibernatePageStateFree);
5ba3f43e
A		 1956#endif
1957
0a7de745
A		 1958 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
1959 compressed = src + page_size;
1960 scratch = compressed + page_size;
1961
1962 pagesDone = 0;
1963 lastBlob = 0;
1964
1965 HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
1966 bitmap_size, header->previewSize,
1967 pageCount, vars->fileVars->position);
1968
f427ee49 1969
0a7de745
A		 1970 enum
1971 // pageType
d9a64523 1972 {
0a7de745
A		 1973 kWired = 0x02,
1974 kEncrypt = 0x01,
1975 kWiredEncrypt = kWired | kEncrypt,
1976 kWiredClear = kWired,
1977 kUnwiredEncrypt = kEncrypt
1978 };
1979
f427ee49 1980#if defined(__i386__) || defined(__x86_64__)
0a7de745 1981 bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features()));
f427ee49
A		 1982#else /* defined(__i386__) || defined(__x86_64__) */
1983 static const bool cpuAES = true;
1984#endif /* defined(__i386__) || defined(__x86_64__) */
0a7de745
A		 1985
1986 for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--) {
1987 if (kUnwiredEncrypt == pageType) {
1988 // start unwired image
1989 if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) {
1990 vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
1991 vars->fileVars->encryptEnd = UINT64_MAX;
1992 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1993 }
1994 bcopy(&cryptvars->aes_iv[0],
1995 &gIOHibernateCryptWakeContext.aes_iv[0],
1996 sizeof(cryptvars->aes_iv));
1997 cryptvars = &gIOHibernateCryptWakeContext;
1998 }
1999 for (iterDone = false, ppnum = 0; !iterDone;) {
2000 if (cpuAES && (pageType == kWiredClear)) {
2001 count = 0;
2002 } else {
2003 count = hibernate_page_list_iterate((kWired & pageType) ? vars->page_list_wired : vars->page_list,
2004 &ppnum);
f427ee49
A		 2005 if (count > UINT_MAX) {
2006 count = UINT_MAX;
2007 }
0a7de745 2008 }
0b4c1975 2009// kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
0a7de745
A		 2010 iterDone = !count;
2011
2012 if (!cpuAES) {
2013 if (count && (kWired & pageType) && needEncrypt) {
2014 uint32_t checkIndex;
2015 for (checkIndex = 0;
2016 (checkIndex < count)
f427ee49 2017 && (((kEncrypt & pageType) == 0) == pmap_is_noencrypt(((ppnum_t)(ppnum + checkIndex))));
0a7de745
A		 2018 checkIndex++) {
2019 }
2020 if (!checkIndex) {
2021 ppnum++;
2022 continue;
2023 }
2024 count = checkIndex;
2025 }
2026 }
2027
2028 switch (pageType) {
2029 case kWiredEncrypt: wiredPagesEncrypted += count; break;
2030 case kWiredClear: wiredPagesClear += count; break;
2031 case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
2032 }
2033
2034 if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */
2035 } else {
f427ee49
A		 2036 pageAndCount[0] = (uint32_t) ppnum;
2037 pageAndCount[1] = (uint32_t) count;
2038 err = IOHibernatePolledFileWrite(vars,
0a7de745
A		 2039 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
2040 cryptvars);
2041 if (kIOReturnSuccess != err) {
2042 break;
2043 }
2044 }
2045
2046 for (page = ppnum; page < (ppnum + count); page++) {
2047 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
2048 if (err) {
2049 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
2050 break;
2051 }
2052
f427ee49 2053 sum = hibernate_sum_page(src, (uint32_t) page);
0a7de745
A		 2054 if (kWired & pageType) {
2055 sum1 += sum;
2056 } else {
2057 sum2 += sum;
2058 }
2059
2060 clock_get_uptime(&startTime);
2061 wkresult = WKdm_compress_new((const WK_word*) src,
2062 (WK_word*) compressed,
2063 (WK_word*) scratch,
f427ee49 2064 (uint32_t) (page_size - 4));
0a7de745
A		 2065
2066 clock_get_uptime(&endTime);
2067 ADD_ABSOLUTETIME(&compTime, &endTime);
2068 SUB_ABSOLUTETIME(&compTime, &startTime);
2069
2070 compBytes += page_size;
2071 pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
2072
2073 if (pageCompressedSize == 0) {
2074 pageCompressedSize = 4;
2075 data = src;
2076
2077 if (*(uint32_t *)src) {
2078 svPageCount++;
2079 } else {
2080 zvPageCount++;
2081 }
2082 } else {
2083 if (pageCompressedSize != page_size) {
2084 data = compressed;
2085 } else {
2086 data = src;
2087 }
2088 }
2089
f427ee49
A		 2090 assert(pageCompressedSize <= page_size);
2091 tag = ((uint32_t) pageCompressedSize) | kIOHibernateTagSignature;
2092 err = IOHibernatePolledFileWrite(vars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
0a7de745
A		 2093 if (kIOReturnSuccess != err) {
2094 break;
2095 }
2096
f427ee49 2097 err = IOHibernatePolledFileWrite(vars, data, (pageCompressedSize + 3) & ~3, cryptvars);
0a7de745
A		 2098 if (kIOReturnSuccess != err) {
2099 break;
2100 }
2101
2102 compressedSize += pageCompressedSize;
2103 uncompressedSize += page_size;
2104 pagesDone++;
2105
2106 if (vars->consoleMapping && (0 == (1023 & pagesDone))) {
2107 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
2108 if (blob != lastBlob) {
2109 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
2110 lastBlob = blob;
2111 }
2112 }
2113 if (0 == (8191 & pagesDone)) {
2114 clock_get_uptime(&endTime);
2115 SUB_ABSOLUTETIME(&endTime, &allTime);
2116 absolutetime_to_nanoseconds(endTime, &nsec);
2117 progressStamp = nsec / 750000000ULL;
2118 if (progressStamp != lastProgressStamp) {
2119 lastProgressStamp = progressStamp;
2120 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
2121 }
2122 }
2123 }
2124 if (kIOReturnSuccess != err) {
2125 break;
2126 }
2127 ppnum = page;
2128 }
39037602 2129
0a7de745
A		 2130 if (kIOReturnSuccess != err) {
2131 break;
2132 }
2133
2134 if ((kEncrypt & pageType) && vars->fileVars->encryptStart) {
2135 vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
2136 HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
d9a64523 2137 }
0a7de745
A		 2138
2139 if (kWiredEncrypt != pageType) {
2140 // end of image1/2 - fill to next block
f427ee49 2141 err = IOHibernatePolledFileWrite(vars, NULL, 0, cryptvars);
0a7de745
A		 2142 if (kIOReturnSuccess != err) {
2143 break;
2144 }
2145 }
2146 if (kWiredClear == pageType) {
2147 // enlarge wired image for test
f427ee49 2148// err = IOHibernatePolledFileWrite(vars, 0, 0x60000000, cryptvars);
7ddcb079 2149
0a7de745
A		 2150 // end wired image
2151 header->encryptStart = vars->fileVars->encryptStart;
2152 header->encryptEnd = vars->fileVars->encryptEnd;
2153 image1Size = vars->fileVars->position;
2154 HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
2155 image1Size, header->encryptStart, header->encryptEnd);
2156 }
2157 }
2158 if (kIOReturnSuccess != err) {
2159 if (kIOReturnOverrun == err) {
2160 // update actual compression ratio on not enough space (for retry)
2161 gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
2162 }
2163
2164 // update partial amount written (for IOPolledFileClose cleanup/unmap)
2165 header->imageSize = vars->fileVars->position;
2166 break;
2167 }
2168
f427ee49 2169
0a7de745
A		 2170 // Header:
2171
2172 header->imageSize = vars->fileVars->position;
2173 header->image1Size = image1Size;
2174 header->bitmapSize = bitmap_size;
2175 header->pageCount = pageCount;
2176
2177 header->restore1Sum = restore1Sum;
2178 header->image1Sum = sum1;
2179 header->image2Sum = sum2;
2180 header->sleepTime = gIOLastSleepTime.tv_sec;
2181
f427ee49 2182 header->compression = ((uint32_t)((compressedSize << 8) / uncompressedSize));
a991bd8d
A		 2183#if defined(__arm64__)
2184 /*
2185 * We don't support retry on hibernation failure and so
2186 * we don't want to set this value to anything smaller
2187 * just because we may have been lucky this time around.
2188 * Though we'll let it go higher.
2189 */
2190 if (header->compression < HIB_COMPR_RATIO_ARM64) {
2191 header->compression = HIB_COMPR_RATIO_ARM64;
2192 }
2193#endif /* __arm64__ */
2194
0a7de745
A		 2195 gIOHibernateCompression = header->compression;
2196
2197 count = vars->fileVars->fileExtents->getLength();
2198 if (count > sizeof(header->fileExtentMap)) {
f427ee49 2199 header->fileExtentMapSize = ((uint32_t) count);
0a7de745
A		 2200 count = sizeof(header->fileExtentMap);
2201 } else {
2202 header->fileExtentMapSize = sizeof(header->fileExtentMap);
2203 }
2204 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
2205
2206 header->deviceBase = vars->fileVars->block0;
2207 header->deviceBlockSize = vars->fileVars->blockSize;
f427ee49
A		 2208 header->lastHibAbsTime = mach_absolute_time();
2209 header->lastHibContTime = mach_continuous_time();
2210
0a7de745
A		 2211
2212 IOPolledFileSeek(vars->fileVars, 0);
f427ee49 2213 err = IOHibernatePolledFileWrite(vars,
0a7de745
A		 2214 (uint8_t *) header, sizeof(IOHibernateImageHeader),
2215 cryptvars);
2216 if (kIOReturnSuccess != err) {
2217 break;
2218 }
f427ee49
A		 2219
2220 err = IOHibernatePolledFileWrite(vars, NULL, 0, cryptvars);
0a7de745
A		 2221 }while (false);
2222
2223 clock_get_uptime(&endTime);
2224
2225 IOService::getPMRootDomain()->pmStatsRecordEvent(
2226 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
2227
2228 SUB_ABSOLUTETIME(&endTime, &allTime);
2229 absolutetime_to_nanoseconds(endTime, &nsec);
2230 HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
2231
2232 absolutetime_to_nanoseconds(compTime, &nsec);
2233 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2234 compBytes,
2235 nsec / 1000000ULL,
2236 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2237
2238 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2239 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
2240 vars->fileVars->cryptBytes,
2241 nsec / 1000000ULL,
2242 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2243
f427ee49 2244 HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%)\n",
0a7de745
A		 2245 header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
2246 uncompressedSize, atop_32(uncompressedSize), compressedSize,
f427ee49
A		 2247 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0);
2248
f427ee49 2249 HIBLOG("\nsum1 %x, sum2 %x\n", sum1, sum2);
0a7de745
A		 2250
2251 HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
2252 svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
2253
2254 if (pollerOpen) {
2255 IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted );
2256 }
2257
2258 if (vars->consoleMapping) {
2259 ProgressUpdate(gIOHibernateGraphicsInfo,
2260 vars->consoleMapping, 0, kIOHibernateProgressCount);
2261 }
2262
2263 HIBLOG("hibernate_write_image done(%x)\n", err);
2264
2265 // should we come back via regular wake, set the state in memory.
2266 gIOHibernateState = kIOHibernateStateInactive;
3a60a9f5 2267
5ba3f43e 2268 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, wiredPagesEncrypted,
0a7de745
A		 2269 wiredPagesClear, dirtyPagesEncrypted);
2270
f427ee49
A		 2271#if defined(__arm64__)
2272 if (kIOReturnSuccess == err) {
2273 return kIOHibernatePostWriteHalt;
2274 } else {
2275 // on ARM, once ApplePMGR decides we're hibernating, we can't turn back
2276 // see: <rdar://problem/63848862> Tonga ApplePMGR diff quiesce path support
2277 panic("hibernate_write_image encountered error 0x%x", err);
2278 }
2279#else
0a7de745
A		 2280 if (kIOReturnSuccess == err) {
2281 if (kIOHibernateModeSleep & gIOHibernateMode) {
2282 return kIOHibernatePostWriteSleep;
2283 } else if (kIOHibernateModeRestart & gIOHibernateMode) {
2284 return kIOHibernatePostWriteRestart;
2285 } else {
2286 /* by default, power down */
2287 return kIOHibernatePostWriteHalt;
2288 }
2289 } else if (kIOReturnAborted == err) {
2290 return kIOHibernatePostWriteWake;
2291 } else {
2292 /* on error, sleep */
2293 return kIOHibernatePostWriteSleep;
2294 }
f427ee49 2295#endif
3a60a9f5
A		 2296}
2297
2298/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2299
a39ff7e2 2300extern "C" void
3a60a9f5
A		 2301hibernate_machine_init(void)
2302{
0a7de745
A		 2303 IOReturn err;
2304 uint32_t sum;
2305 uint32_t pagesDone;
2306 uint32_t pagesRead = 0;
2307 AbsoluteTime startTime, compTime;
2308 AbsoluteTime allTime, endTime;
2309 AbsoluteTime startIOTime, endIOTime;
2310 uint64_t nsec, nsecIO;
2311 uint64_t compBytes;
f427ee49
A		 2312 uint64_t lastProgressStamp = 0;
2313 uint64_t progressStamp;
cb323159 2314 IOPolledFileCryptVars * cryptvars = NULL;
0a7de745
A		 2315
2316 IOHibernateVars * vars = &gIOHibernateVars;
2317 bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
2318
2319 if (!vars->fileVars || !vars->fileVars->pollers) {
2320 return;
2321 }
2322
2323 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2324 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2325
2326 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) {
2327 HIBLOG("regular wake\n");
2328 return;
2329 }
2330
2331 HIBPRINT("diag %x %x %x %x\n",
a39ff7e2 2332 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
6d2010ae 2333 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
3a60a9f5 2334
f427ee49
A		 2335#if defined(__i386__) || defined(__x86_64__)
2336#define t40ms(x) ((uint32_t)((tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)))
2337#else /* defined(__i386__) || defined(__x86_64__) */
2338#define t40ms(x) x
2339#endif /* defined(__i386__) || defined(__x86_64__) */
0a7de745
A		 2340#define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
2341 tStat(booterStart, booterStart);
2342 gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart;
2343 tStat(booterDuration0, booterTime0);
2344 tStat(booterDuration1, booterTime1);
2345 tStat(booterDuration2, booterTime2);
2346 tStat(booterDuration, booterTime);
2347 tStat(booterConnectDisplayDuration, connectDisplayTime);
2348 tStat(booterSplashDuration, splashTime);
2349 tStat(trampolineDuration, trampolineTime);
2350
2351 gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size;
2352 gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize;
2353 gIOHibernateStats->image1Pages = pagesDone;
39236c6e 2354
5ba3f43e
A		 2355 /* HIBERNATE_stats */
2356 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 14), gIOHibernateStats->smcStart,
0a7de745
A		 2357 gIOHibernateStats->booterStart, gIOHibernateStats->booterDuration,
2358 gIOHibernateStats->trampolineDuration);
2359
2360 HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
2361 gIOHibernateStats->booterStart,
2362 gIOHibernateStats->smcStart,
2363 gIOHibernateStats->booterDuration0,
2364 gIOHibernateStats->booterDuration1,
2365 gIOHibernateStats->booterDuration2,
2366 gIOHibernateStats->booterDuration,
2367 gIOHibernateStats->booterConnectDisplayDuration,
2368 gIOHibernateStats->booterSplashDuration,
2369 gIOHibernateStats->trampolineDuration);
2370
2371 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
39236c6e
A		 2372 gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
2373 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
bd504ef0 2374
0a7de745
A		 2375 if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
2376 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))) {
2377 hibernate_page_list_discard(vars->page_list);
2378 }
0c530ab8 2379
f427ee49
A		 2380 if (vars->hwEncrypt) {
2381 // if vars->hwEncrypt is true, we don't need cryptvars since we supply the
2382 // decryption key via IOPolledFilePollersSetEncryptionKey
2383 cryptvars = NULL;
2384 } else {
2385 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : NULL;
2386 }
6d2010ae 2387
0a7de745
A		 2388 if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount) {
2389 panic("handoff overflow");
2390 }
6d2010ae 2391
0a7de745
A		 2392 IOHibernateHandoff * handoff;
2393 bool done = false;
2394 bool foundCryptData = false;
2395 bool foundVolumeEncryptData = false;
f427ee49
A		 2396 const uint8_t * handoffStart = (const uint8_t*)vars->handoffBuffer->getBytesNoCopy();
2397 const uint8_t * handoffEnd = handoffStart + vars->handoffBuffer->getLength();
6d2010ae 2398
0a7de745
A		 2399 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
2400 !done;
2401 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount]) {
f427ee49
A		 2402 if (((uint8_t*)handoff < handoffStart) ||
2403 (&handoff->data[handoff->bytecount] > handoffEnd)) {
2404 panic("handoff out of range");
2405 }
6d2010ae 2406// HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
0a7de745
A		 2407 uint8_t * data = &handoff->data[0];
2408 switch (handoff->type) {
2409 case kIOHibernateHandoffTypeEnd:
2410 done = true;
2411 break;
6d2010ae 2412
0a7de745
A		 2413 case kIOHibernateHandoffTypeGraphicsInfo:
2414 if (handoff->bytecount == sizeof(*gIOHibernateGraphicsInfo)) {
2415 bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
2416 }
2417 break;
6d2010ae 2418
0a7de745
A		 2419 case kIOHibernateHandoffTypeCryptVars:
2420 if (cryptvars) {
2421 hibernate_cryptwakevars_t *
2422 wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
f427ee49
A		 2423 if (handoff->bytecount == sizeof(*wakevars)) {
2424 bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
2425 } else {
2426 panic("kIOHibernateHandoffTypeCryptVars(%d)", handoff->bytecount);
2427 }
0a7de745
A		 2428 }
2429 foundCryptData = true;
2430 bzero(data, handoff->bytecount);
2431 break;
6d2010ae 2432
0a7de745
A		 2433 case kIOHibernateHandoffTypeVolumeCryptKey:
2434 if (handoff->bytecount == vars->volumeCryptKeySize) {
2435 bcopy(data, &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
2436 foundVolumeEncryptData = true;
2437 } else {
2438 panic("kIOHibernateHandoffTypeVolumeCryptKey(%d)", handoff->bytecount);
2439 }
2440 break;
a39ff7e2 2441
f427ee49 2442#if defined(__i386__) || defined(__x86_64__)
0a7de745 2443 case kIOHibernateHandoffTypeMemoryMap:
bd504ef0 2444
0a7de745 2445 clock_get_uptime(&allTime);
bd504ef0 2446
0a7de745
A		 2447 hibernate_newruntime_map(data, handoff->bytecount,
2448 gIOHibernateCurrentHeader->systemTableOffset);
bd504ef0 2449
0a7de745 2450 clock_get_uptime(&endTime);
a39ff7e2 2451
0a7de745
A		 2452 SUB_ABSOLUTETIME(&endTime, &allTime);
2453 absolutetime_to_nanoseconds(endTime, &nsec);
a39ff7e2 2454
0a7de745 2455 HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
bd504ef0 2456
0a7de745 2457 break;
6d2010ae 2458
0a7de745 2459 case kIOHibernateHandoffTypeDeviceTree:
6d2010ae
A		 2460 {
2461// DTEntry chosen = NULL;
f427ee49 2462// HIBPRINT("SecureDTLookupEntry %d\n", SecureDTLookupEntry((const DTEntry) data, "/chosen", &chosen));
6d2010ae 2463 }
0a7de745 2464 break;
f427ee49 2465#endif /* defined(__i386__) || defined(__x86_64__) */
6d2010ae 2466
0a7de745
A		 2467 default:
2468 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
2469 break;
2470 }
a39ff7e2 2471 }
a39ff7e2 2472
0a7de745
A		 2473 if (vars->hwEncrypt && !foundVolumeEncryptData) {
2474 panic("no volumeCryptKey");
2475 } else if (cryptvars && !foundCryptData) {
2476 panic("hibernate handoff");
2477 }
6d2010ae 2478
0a7de745 2479 HIBPRINT("video 0x%llx %d %d %d status %x\n",
a39ff7e2
A		 2480 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2481 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
316670eb 2482
0a7de745
A		 2483 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress) {
2484 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2485 * gIOHibernateGraphicsInfo->rowBytes);
2486 if (vars->videoMapSize > vars->videoAllocSize) {
2487 vars->videoMapSize = 0;
2488 } else {
2489 IOMapPages(kernel_map,
2490 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2491 vars->videoMapSize, kIOMapInhibitCache );
2492 }
2493 }
3a60a9f5 2494
0a7de745
A		 2495 if (vars->videoMapSize) {
2496 ProgressUpdate(gIOHibernateGraphicsInfo,
2497 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2498 }
3a60a9f5 2499
0a7de745
A		 2500 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2501 uint8_t * compressed = src + page_size;
2502 uint8_t * scratch = compressed + page_size;
2503 uint32_t decoOffset;
2504
2505 clock_get_uptime(&allTime);
2506 AbsoluteTime_to_scalar(&compTime) = 0;
2507 compBytes = 0;
2508
2509 HIBLOG("IOPolledFilePollersOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2510 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledAfterSleepState, false);
2511 clock_get_uptime(&startIOTime);
2512 endTime = startIOTime;
2513 SUB_ABSOLUTETIME(&endTime, &allTime);
2514 absolutetime_to_nanoseconds(endTime, &nsec);
2515 HIBLOG("IOPolledFilePollersOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
2516
2517 if (vars->hwEncrypt) {
2518 err = IOPolledFilePollersSetEncryptionKey(vars->fileVars,
2519 &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
2520 HIBLOG("IOPolledFilePollersSetEncryptionKey(%x) %ld\n", err, vars->volumeCryptKeySize);
2521 if (kIOReturnSuccess != err) {
2522 panic("IOPolledFilePollersSetEncryptionKey(0x%x)", err);
2523 }
cb323159 2524 cryptvars = NULL;
0a7de745 2525 }
3a60a9f5 2526
0a7de745 2527 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
3a60a9f5 2528
0a7de745
A		 2529 // kick off the read ahead
2530 vars->fileVars->bufferHalf = 0;
2531 vars->fileVars->bufferLimit = 0;
2532 vars->fileVars->lastRead = 0;
2533 vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
2534 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2535 vars->fileVars->cryptBytes = 0;
2536 AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
36401178 2537
cb323159 2538 err = IOPolledFileRead(vars->fileVars, NULL, 0, cryptvars);
f427ee49
A		 2539 if (kIOReturnSuccess != err) {
2540 panic("Hibernate restore error %x", err);
2541 }
0a7de745
A		 2542 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2543 // --
3e170ce0 2544
0a7de745 2545 HIBLOG("hibernate_machine_init reading\n");
3e170ce0 2546
f427ee49 2547
0a7de745
A		 2548 uint32_t * header = (uint32_t *) src;
2549 sum = 0;
a39ff7e2 2550
0a7de745
A		 2551 while (kIOReturnSuccess == err) {
2552 unsigned int count;
2553 unsigned int page;
2554 uint32_t tag;
f427ee49
A		 2555 vm_offset_t compressedSize;
2556 ppnum_t ppnum;
3e170ce0 2557
0a7de745
A		 2558 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2559 if (kIOReturnSuccess != err) {
f427ee49 2560 panic("Hibernate restore error %x", err);
39236c6e 2561 }
0a7de745
A		 2562
2563 ppnum = header[0];
2564 count = header[1];
2565
2566// HIBPRINT("(%x, %x)\n", ppnum, count);
2567
2568 if (!count) {
2569 break;
2570 }
2571
2572 for (page = 0; page < count; page++) {
2573 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2574 if (kIOReturnSuccess != err) {
f427ee49 2575 panic("Hibernate restore error %x", err);
0a7de745
A		 2576 }
2577
2578 compressedSize = kIOHibernateTagLength & tag;
2579 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength)) {
2580 err = kIOReturnIPCError;
f427ee49 2581 panic("Hibernate restore error %x", err);
0a7de745
A		 2582 }
2583
2584 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2585 if (kIOReturnSuccess != err) {
f427ee49 2586 panic("Hibernate restore error %x", err);
0a7de745
A		 2587 }
2588
2589 if (compressedSize < page_size) {
f427ee49 2590 decoOffset = ((uint32_t) page_size);
0a7de745
A		 2591 clock_get_uptime(&startTime);
2592
2593 if (compressedSize == 4) {
2594 int i;
2595 uint32_t *s, *d;
2596
2597 s = (uint32_t *)src;
2598 d = (uint32_t *)(uintptr_t)compressed;
2599
2600 for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++) {
2601 *d++ = *s;
2602 }
2603 } else {
f427ee49 2604 pal_hib_decompress_page(src, compressed, scratch, ((unsigned int) compressedSize));
0a7de745
A		 2605 }
2606 clock_get_uptime(&endTime);
2607 ADD_ABSOLUTETIME(&compTime, &endTime);
2608 SUB_ABSOLUTETIME(&compTime, &startTime);
2609 compBytes += page_size;
2610 } else {
2611 decoOffset = 0;
2612 }
2613
f427ee49 2614 sum += hibernate_sum_page((src + decoOffset), ((uint32_t) ppnum));
0a7de745
A		 2615 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2616 if (err) {
2617 HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
f427ee49 2618 panic("Hibernate restore error %x", err);
0a7de745
A		 2619 }
2620
f427ee49 2621
0a7de745
A		 2622 ppnum++;
2623 pagesDone++;
2624 pagesRead++;
2625
2626 if (0 == (8191 & pagesDone)) {
2627 clock_get_uptime(&endTime);
2628 SUB_ABSOLUTETIME(&endTime, &allTime);
2629 absolutetime_to_nanoseconds(endTime, &nsec);
2630 progressStamp = nsec / 750000000ULL;
2631 if (progressStamp != lastProgressStamp) {
2632 lastProgressStamp = progressStamp;
2633 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2634 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2635 }
2636 }
3a60a9f5 2637 }
3a60a9f5 2638 }
0a7de745
A		 2639 if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages)) {
2640 err = kIOReturnLockedRead;
2641 }
3a60a9f5 2642
0a7de745
A		 2643 if (kIOReturnSuccess != err) {
2644 panic("Hibernate restore error %x", err);
2645 }
36401178 2646
f427ee49 2647
0a7de745
A		 2648 gIOHibernateCurrentHeader->actualImage2Sum = sum;
2649 gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
3a60a9f5 2650
0a7de745 2651 clock_get_uptime(&endIOTime);
39236c6e 2652
0a7de745 2653 err = IOPolledFilePollersClose(vars->fileVars, kIOPolledAfterSleepState);
3a60a9f5 2654
0a7de745 2655 clock_get_uptime(&endTime);
b0d623f7 2656
0a7de745
A		 2657 IOService::getPMRootDomain()->pmStatsRecordEvent(
2658 kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
2659 IOService::getPMRootDomain()->pmStatsRecordEvent(
2660 kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
b0d623f7 2661
0a7de745
A		 2662 SUB_ABSOLUTETIME(&endTime, &allTime);
2663 absolutetime_to_nanoseconds(endTime, &nsec);
3a60a9f5 2664
0a7de745
A		 2665 SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
2666 absolutetime_to_nanoseconds(endIOTime, &nsecIO);
39236c6e 2667
f427ee49 2668 gIOHibernateStats->kernelImageReadDuration = ((uint32_t) (nsec / 1000000ULL));
0a7de745 2669 gIOHibernateStats->imagePages = pagesDone;
39236c6e 2670
0a7de745
A		 2671 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
2672 pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
2673 nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
6d2010ae 2674
0a7de745
A		 2675 absolutetime_to_nanoseconds(compTime, &nsec);
2676 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2677 compBytes,
2678 nsec / 1000000ULL,
2679 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
6d2010ae 2680
0a7de745
A		 2681 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2682 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
2683 vars->fileVars->cryptBytes,
2684 nsec / 1000000ULL,
2685 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
6d2010ae 2686
0a7de745 2687 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 2), pagesRead, pagesDone);
3a60a9f5
A		 2688}
2689
2690/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
99c3a104 2691
0a7de745
A		 2692void
2693IOHibernateSetWakeCapabilities(uint32_t capability)
39236c6e 2694{
0a7de745
A		 2695 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
2696 gIOHibernateStats->wakeCapability = capability;
8a3053a0 2697
0a7de745
A		 2698 if (kIOPMSystemCapabilityGraphics & capability) {
2699 vm_compressor_do_warmup();
2700 }
8a3053a0 2701 }
39236c6e
A		 2702}
2703
2704/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2705
0a7de745
A		 2706void
2707IOHibernateSystemRestart(void)
99c3a104 2708{
0a7de745
A		 2709 static uint8_t noteStore[32] __attribute__((aligned(32)));
2710 IORegistryEntry * regEntry;
2711 const OSSymbol * sym;
2712 OSData * noteProp;
2713 OSData * data;
2714 uintptr_t * smcVars;
2715 uint8_t * smcBytes;
2716 size_t len;
2717 addr64_t element;
2718
2719 data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
2720 if (!data) {
2721 return;
2722 }
2723
2724 smcVars = (typeof(smcVars))data->getBytesNoCopy();
2725 smcBytes = (typeof(smcBytes))smcVars[1];
2726 len = smcVars[0];
2727 if (len > sizeof(noteStore)) {
2728 len = sizeof(noteStore);
2729 }
2730 noteProp = OSData::withCapacity(3 * sizeof(element));
2731 if (!noteProp) {
2732 return;
2733 }
2734 element = len;
2735 noteProp->appendBytes(&element, sizeof(element));
2736 element = crc32(0, smcBytes, len);
2737 noteProp->appendBytes(&element, sizeof(element));
2738
2739 bcopy(smcBytes, noteStore, len);
2740 element = (addr64_t) &noteStore[0];
2741 element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
2742 noteProp->appendBytes(&element, sizeof(element));
2743
2744 if (!gIOOptionsEntry) {
2745 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
2746 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
2747 if (regEntry && !gIOOptionsEntry) {
2748 regEntry->release();
2749 }
2750 }
2751
2752 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
2753 if (gIOOptionsEntry && sym) {
2754 gIOOptionsEntry->setProperty(sym, noteProp);
2755 }
2756 if (noteProp) {
2757 noteProp->release();
2758 }
2759 if (sym) {
2760 sym->release();
2761 }
99c3a104 2762}
mirror of XNU