]> git.saurik.com Git - apple/xnu.git/blob - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-7195.50.7.100.1.tar.gz
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
1 /*
2 * Copyright (c) 2004-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30 *
31 * Sleep:
32 *
33 * - PMRootDomain calls IOHibernateSystemSleep() before system sleep
34 * (devices awake, normal execution context)
35 * - IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
36 * grabs its extents and searches for a polling driver willing to work with that IOMedia.
37 * The BSD code makes an ioctl to the storage driver to get the partition base offset to
38 * the disk, and other ioctls to get the transfer constraints
39 * If successful, the file is written to make sure its initially not bootable (in case of
40 * later failure) and nvram set to point to the first block of the file. (Has to be done
41 * here so blocking is possible in nvram support).
42 * hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
43 * page out any pages it wants to (currently zero, but probably some percentage of memory).
44 * Its assumed just allocating pages will cause the VM system to naturally select the best
45 * pages for eviction. It also copies processor flags needed for the restore path and sets
46 * a flag in the boot processor proc info.
47 * gIOHibernateState = kIOHibernateStateHibernating.
48 * - Regular sleep progresses - some drivers may inspect the root domain property
49 * kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
50 * as usual but leaves motherboard I/O on.
51 * - Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
52 * at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
53 * all ppc RC bits out of the hash table and caches into the mapping structures.
54 * - hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
55 * hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
56 * All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
57 * by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
58 * The image header and block list are written. The header includes the second file extent so
59 * only the header block is needed to read the file, regardless of filesystem.
60 * The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
61 * (only) is used to decompress the image during wake/boot.
62 * Some additional pages are removed from the bitmaps - the buffers used for hibernation.
63 * The bitmaps are written to the image.
64 * More areas are removed from the bitmaps (after they have been written to the image) - the
65 * segment "__HIB" pages and interrupt stack.
66 * Each wired page is compressed and written and then each non-wired page. Compression and
67 * disk writes are in parallel.
68 * The image header is written to the start of the file and the polling driver closed.
69 * The machine powers down (or sleeps).
70 *
71 * Boot/Wake:
72 *
73 * - BootX sees the boot-image nvram variable containing the device and block number of the image,
74 * reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
75 * - BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
76 * in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
77 * that is in the kernel's __HIB section.
78 * - BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
79 * hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
80 * only code & data in that section is safe to call since all the other wired pages are still
81 * compressed in the image.
82 * - hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
83 * It uses the bitmaps to work out which pages can be uncompressed from the image to their final
84 * location directly, and copies those that can't to interim free pages. When the image has been
85 * completed, the copies are uncompressed, overwriting the wired image pages.
86 * hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
87 * is used to get pages into place for 64bit.
88 * - the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
89 * changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
90 * are removed from the software strutures, and the hash table is reinitialized.
91 * - After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
92 * of memory, using the polled mode driver, before other threads can run or any devices are turned on.
93 * This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
94 * for the remaining non wired pages.
95 * - The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
96 * (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
97 * is closed via bsd.
98 *
99 * Polled Mode I/O:
100 *
101 * IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
102 * registry, specifying an object of calls IOPolledInterface.
103 *
104 * Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
105 * partition) that the image is going to live, looking for polled interface properties. If it finds
106 * one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
107 * interfaces found are kept in an ordered list.
108 *
109 * There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
110 * few different contexts things happen in:
111 *
112 * - there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
113 * up and running) and after wake - this is safe to allocate memory and do anything. The device
114 * ignores sleep requests from that point since its a waste of time if it goes to sleep and
115 * immediately wakes back up for the image write.
116 *
117 * - there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
118 * immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
119 * from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
120 * used to flush and set the disk to sleep.
121 *
122 * - there is an Open/Close (AfterSleep) pair made around the image read operations that happen
123 * immediately after sleep. These can't block or allocate memory. This is happening after the platform
124 * expert has woken the low level bits of the system, but most of the I/O system has not. There is only
125 * one thread running.
126 *
127 * For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
128 * (prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
129 * the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
130 * Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
131 * that is called for the hardware to check for events, and complete the I/O via the callback.
132 * The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
133 * to restrict I/O ops.
134 */
135
136 #include <sys/systm.h>
137
138 #include <IOKit/IOWorkLoop.h>
139 #include <IOKit/IOCommandGate.h>
140 #include <IOKit/IOTimerEventSource.h>
141 #include <IOKit/IOPlatformExpert.h>
142 #include <IOKit/IOKitDebug.h>
143 #include <IOKit/IOTimeStamp.h>
144 #include <IOKit/pwr_mgt/RootDomain.h>
145 #include <IOKit/pwr_mgt/IOPMPrivate.h>
146 #include <IOKit/IOMessage.h>
147 #include <IOKit/IODeviceTreeSupport.h>
148 #include <IOKit/IOBSD.h>
149 #include <IOKit/IOKitKeysPrivate.h>
150 #include "RootDomainUserClient.h"
151 #include <IOKit/pwr_mgt/IOPowerConnection.h>
152 #include "IOPMPowerStateQueue.h"
153 #include <IOKit/IOBufferMemoryDescriptor.h>
154 #include <IOKit/AppleKeyStoreInterface.h>
155 #include <libkern/crypto/aes.h>
156
157 #include <sys/uio.h>
158 #include <sys/conf.h>
159 #include <sys/stat.h>
160 #include <sys/fcntl.h> // (FWRITE, ...)
161 #include <sys/sysctl.h>
162 #include <sys/kdebug.h>
163 #include <stdint.h>
164
165 #include <IOKit/IOHibernatePrivate.h>
166 #include <IOKit/IOPolledInterface.h>
167 #include <IOKit/IONVRAM.h>
168 #include "IOHibernateInternal.h"
169 #include <vm/vm_protos.h>
170 #include "IOKitKernelInternal.h"
171 #include <pexpert/device_tree.h>
172
173 #include <machine/pal_routines.h>
174 #include <machine/pal_hibernate.h>
175 #if defined(__i386__) || defined(__x86_64__)
176 #include <i386/tsc.h>
177 #include <i386/cpuid.h>
178 #include <vm/WKdm_new.h>
179 #elif defined(__arm64__)
180 #include <arm64/amcc_rorgn.h>
181 #endif /* defined(__i386__) || defined(__x86_64__) */
182 #include <san/kasan.h>
183
184 #if HIBERNATE_HMAC_IMAGE
185 #include <arm64/hibernate_ppl_hmac.h>
186 #endif /* HIBERNATE_HMAC_IMAGE */
187
188 extern "C" addr64_t kvtophys(vm_offset_t va);
189 extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
190
191 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
192
193 #define DISABLE_TRIM 0
194 #define TRIM_DELAY 25000
195
196 extern unsigned int save_kdebug_enable;
197 extern uint32_t gIOHibernateState;
198 uint32_t gIOHibernateMode;
199 static char gIOHibernateBootSignature[256 + 1];
200 static char gIOHibernateFilename[MAXPATHLEN + 1];
201 uint32_t gIOHibernateCount;
202
203 static uuid_string_t gIOHibernateBridgeBootSessionUUIDString;
204
205 static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
206 uint32_t gIOHibernateFreeTime = 0 * 1000; // max time to spend freeing pages (ms)
207 static uint64_t gIOHibernateCompression = 0x80; // default compression 50%
208 boolean_t gIOHibernateStandbyDisabled;
209
210 static IODTNVRAM * gIOOptionsEntry;
211 static IORegistryEntry * gIOChosenEntry;
212
213 static const OSSymbol * gIOHibernateBootImageKey;
214 static const OSSymbol * gIOHibernateBootSignatureKey;
215 static const OSSymbol * gIOBridgeBootSessionUUIDKey;
216
217 #if defined(__i386__) || defined(__x86_64__)
218
219 static const OSSymbol * gIOHibernateRTCVariablesKey;
220 static const OSSymbol * gIOHibernateBoot0082Key;
221 static const OSSymbol * gIOHibernateBootNextKey;
222 static OSData * gIOHibernateBoot0082Data;
223 static OSData * gIOHibernateBootNextData;
224 static OSObject * gIOHibernateBootNextSave;
225
226 #endif /* defined(__i386__) || defined(__x86_64__) */
227
228 static IOLock * gFSLock;
229 uint32_t gFSState;
230 static thread_call_t gIOHibernateTrimCalloutEntry;
231 static IOPolledFileIOVars gFileVars;
232 static IOHibernateVars gIOHibernateVars;
233 static IOPolledFileCryptVars gIOHibernateCryptWakeContext;
234 static hibernate_graphics_t _hibernateGraphics;
235 static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
236 static hibernate_statistics_t _hibernateStats;
237 static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats;
238
239 enum{
240 kFSIdle = 0,
241 kFSOpening = 2,
242 kFSOpened = 3,
243 kFSTimedOut = 4,
244 kFSTrimDelay = 5
245 };
246
247 static IOReturn IOHibernateDone(IOHibernateVars * vars);
248 static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature);
249 static void IOSetBootImageNVRAM(OSData * data);
250 static void IOHibernateSystemPostWakeTrim(void * p1, void * p2);
251
252 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
253
254 enum { kDefaultIOSize = 128 * 1024 };
255 enum { kVideoMapSize = 80 * 1024 * 1024 };
256
257 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
258
259 // copy from phys addr to MD
260
261 #if !HIBERNATE_HMAC_IMAGE
262 static IOReturn
263 IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
264 IOByteCount offset, addr64_t bytes, IOByteCount length)
265 {
266 addr64_t srcAddr = bytes;
267 IOByteCount remaining;
268
269 remaining = length = min(length, md->getLength() - offset);
270 while (remaining) { // (process another target segment?)
271 addr64_t dstAddr64;
272 IOByteCount dstLen;
273
274 dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
275 if (!dstAddr64) {
276 break;
277 }
278
279 // Clip segment length to remaining
280 if (dstLen > remaining) {
281 dstLen = remaining;
282 }
283
284 #if 1
285 bcopy_phys(srcAddr, dstAddr64, dstLen);
286 #else
287 copypv(srcAddr, dstAddr64, dstLen,
288 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
289 #endif
290 srcAddr += dstLen;
291 offset += dstLen;
292 remaining -= dstLen;
293 }
294
295 assert(!remaining);
296
297 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
298 }
299 #endif /* !HIBERNATE_HMAC_IMAGE */
300
301 // copy from MD to phys addr
302
303 static IOReturn
304 IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
305 IOByteCount offset, addr64_t bytes, IOByteCount length)
306 {
307 addr64_t dstAddr = bytes;
308 IOByteCount remaining;
309
310 remaining = length = min(length, md->getLength() - offset);
311 while (remaining) { // (process another target segment?)
312 addr64_t srcAddr64;
313 IOByteCount dstLen;
314
315 srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
316 if (!srcAddr64) {
317 break;
318 }
319
320 // Clip segment length to remaining
321 if (dstLen > remaining) {
322 dstLen = remaining;
323 }
324
325 #if 1
326 bcopy_phys(srcAddr64, dstAddr, dstLen);
327 #else
328 copypv(srcAddr, dstAddr64, dstLen,
329 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
330 #endif
331 dstAddr += dstLen;
332 offset += dstLen;
333 remaining -= dstLen;
334 }
335
336 assert(!remaining);
337
338 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
339 }
340
341 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
342
343 void
344 hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
345 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
346 {
347 count += ppnum;
348
349 if (count > UINT_MAX) {
350 panic("hibernate_set_page_state ppnum");
351 }
352
353 switch (kind) {
354 case kIOHibernatePageStateUnwiredSave:
355 // unwired save
356 for (; ppnum < count; ppnum++) {
357 hibernate_page_bitset(page_list, FALSE, (uint32_t) ppnum);
358 hibernate_page_bitset(page_list_wired, TRUE, (uint32_t) ppnum);
359 }
360 break;
361 case kIOHibernatePageStateWiredSave:
362 // wired save
363 for (; ppnum < count; ppnum++) {
364 hibernate_page_bitset(page_list, FALSE, (uint32_t) ppnum);
365 hibernate_page_bitset(page_list_wired, FALSE, (uint32_t) ppnum);
366 }
367 break;
368 case kIOHibernatePageStateFree:
369 // free page
370 for (; ppnum < count; ppnum++) {
371 hibernate_page_bitset(page_list, TRUE, (uint32_t) ppnum);
372 hibernate_page_bitset(page_list_wired, TRUE, (uint32_t) ppnum);
373 }
374 break;
375 default:
376 panic("hibernate_set_page_state");
377 }
378 }
379
380 static void
381 hibernate_set_descriptor_page_state(IOHibernateVars *vars,
382 IOMemoryDescriptor *descriptor,
383 uint32_t kind,
384 uint32_t *pageCount)
385 {
386 IOItemCount count;
387 addr64_t phys64;
388 IOByteCount segLen;
389 if (descriptor) {
390 for (count = 0;
391 (phys64 = descriptor->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
392 count += segLen) {
393 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
394 atop_64(phys64), atop_32(segLen),
395 kind);
396 *pageCount -= atop_32(segLen);
397 }
398 }
399 }
400
401 static vm_offset_t
402 hibernate_page_list_iterate(hibernate_page_list_t * list, ppnum_t * pPage)
403 {
404 uint32_t page = ((typeof(page)) * pPage);
405 uint32_t count;
406 hibernate_bitmap_t * bitmap;
407
408 while ((bitmap = hibernate_page_bitmap_pin(list, &page))) {
409 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
410 if (!count) {
411 break;
412 }
413 page += count;
414 if (page <= bitmap->last_page) {
415 break;
416 }
417 }
418
419 *pPage = page;
420 if (bitmap) {
421 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
422 } else {
423 count = 0;
424 }
425
426 return count;
427 }
428
429 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
430
431 IOReturn
432 IOHibernateSystemSleep(void)
433 {
434 IOReturn err;
435 OSData * nvramData;
436 OSObject * obj;
437 OSString * str;
438 OSNumber * num;
439 bool dsSSD, vmflush, swapPinned;
440 IOHibernateVars * vars;
441 uint64_t setFileSize = 0;
442
443 gIOHibernateState = kIOHibernateStateInactive;
444
445 gIOHibernateDebugFlags = 0;
446 if (kIOLogHibernate & gIOKitDebug) {
447 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
448 }
449
450 if (IOService::getPMRootDomain()->getHibernateSettings(
451 &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime)) {
452 if (kIOHibernateModeSleep & gIOHibernateMode) {
453 // default to discard clean for safe sleep
454 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
455 | kIOHibernateModeDiscardCleanActive);
456 }
457 }
458
459 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey))) {
460 if ((str = OSDynamicCast(OSString, obj))) {
461 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
462 sizeof(gIOHibernateFilename));
463 }
464 obj->release();
465 }
466
467 if (!gIOHibernateMode || !gIOHibernateFilename[0]) {
468 return kIOReturnUnsupported;
469 }
470
471 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
472
473 vars = IONew(IOHibernateVars, 1);
474 if (!vars) {
475 return kIOReturnNoMemory;
476 }
477 bzero(vars, sizeof(*vars));
478
479 IOLockLock(gFSLock);
480 if (!gIOHibernateTrimCalloutEntry) {
481 gIOHibernateTrimCalloutEntry = thread_call_allocate(&IOHibernateSystemPostWakeTrim, &gFSLock);
482 }
483 IOHibernateSystemPostWakeTrim(NULL, NULL);
484 thread_call_cancel(gIOHibernateTrimCalloutEntry);
485 if (kFSIdle != gFSState) {
486 HIBLOG("hibernate file busy\n");
487 IOLockUnlock(gFSLock);
488 IODelete(vars, IOHibernateVars, 1);
489 return kIOReturnBusy;
490 }
491 gFSState = kFSOpening;
492 IOLockUnlock(gFSLock);
493
494 swapPinned = false;
495 do{
496 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
497 HIBERNATION_SRC_BUFFER_SIZE, page_size);
498
499 vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
500 ptoa_64(gIOHibernateHandoffPageCount), page_size);
501
502 if (!vars->srcBuffer || !vars->handoffBuffer) {
503 err = kIOReturnNoMemory;
504 break;
505 }
506
507 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey))) {
508 if ((num = OSDynamicCast(OSNumber, obj))) {
509 vars->fileMinSize = num->unsigned64BitValue();
510 }
511 obj->release();
512 }
513 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey))) {
514 if ((num = OSDynamicCast(OSNumber, obj))) {
515 vars->fileMaxSize = num->unsigned64BitValue();
516 }
517 obj->release();
518 }
519
520 boolean_t encryptedswap = true;
521 uint32_t pageCount;
522 AbsoluteTime startTime, endTime;
523 uint64_t nsec;
524
525 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
526 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
527 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
528
529 vmflush = ((kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
530 err = hibernate_alloc_page_lists(&vars->page_list,
531 &vars->page_list_wired,
532 &vars->page_list_pal);
533 if (KERN_SUCCESS != err) {
534 HIBLOG("%s err, hibernate_alloc_page_lists return 0x%x\n", __FUNCTION__, err);
535 break;
536 }
537
538 err = hibernate_pin_swap(TRUE);
539 if (KERN_SUCCESS != err) {
540 HIBLOG("%s error, hibernate_pin_swap return 0x%x\n", __FUNCTION__, err);
541 break;
542 }
543 swapPinned = true;
544
545 if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode)) {
546 hibernate_page_list_setall(vars->page_list,
547 vars->page_list_wired,
548 vars->page_list_pal,
549 true /* preflight */,
550 vmflush /* discard */,
551 &pageCount);
552 PE_Video consoleInfo;
553 bzero(&consoleInfo, sizeof(consoleInfo));
554 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
555
556 // estimate: 6% increase in pages compressed
557 // screen preview 2 images compressed 0%
558 setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
559 + vars->page_list->list_size
560 + (consoleInfo.v_width * consoleInfo.v_height * 8);
561 enum { setFileRound = 1024 * 1024ULL };
562 setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
563
564 HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
565 pageCount, (100ULL * gIOHibernateCompression) >> 8,
566 setFileSize, vars->fileMinSize);
567
568 if (!(kIOHibernateModeFileResize & gIOHibernateMode)
569 && (setFileSize < vars->fileMinSize)) {
570 setFileSize = vars->fileMinSize;
571 }
572 }
573
574 vars->volumeCryptKeySize = sizeof(vars->volumeCryptKey);
575 err = IOPolledFileOpen(gIOHibernateFilename,
576 (kIOPolledFileCreate | kIOPolledFileHibernate),
577 setFileSize, 0,
578 gIOHibernateCurrentHeader, sizeof(gIOHibernateCurrentHeader),
579 &vars->fileVars, &nvramData,
580 &vars->volumeCryptKey[0], &vars->volumeCryptKeySize);
581
582 if (KERN_SUCCESS != err) {
583 IOLockLock(gFSLock);
584 if (kFSOpening != gFSState) {
585 err = kIOReturnTimeout;
586 }
587 IOLockUnlock(gFSLock);
588 }
589
590 if (KERN_SUCCESS != err) {
591 HIBLOG("IOPolledFileOpen(%x)\n", err);
592 break;
593 }
594
595 // write extents for debug data usage in EFI
596 IOWriteExtentsToFile(vars->fileVars, kIOHibernateHeaderOpenSignature);
597
598 err = IOPolledFilePollersSetup(vars->fileVars, kIOPolledPreflightState);
599 if (KERN_SUCCESS != err) {
600 break;
601 }
602
603 clock_get_uptime(&startTime);
604 err = hibernate_setup(gIOHibernateCurrentHeader,
605 vmflush,
606 vars->page_list, vars->page_list_wired, vars->page_list_pal);
607 clock_get_uptime(&endTime);
608 SUB_ABSOLUTETIME(&endTime, &startTime);
609 absolutetime_to_nanoseconds(endTime, &nsec);
610
611 boolean_t haveSwapPin, hibFileSSD;
612 haveSwapPin = vm_swap_files_pinned();
613
614 hibFileSSD = (kIOPolledFileSSD & vars->fileVars->flags);
615
616 HIBLOG("hibernate_setup(%d) took %qd ms, swapPin(%d) ssd(%d)\n",
617 err, nsec / 1000000ULL,
618 haveSwapPin, hibFileSSD);
619 if (KERN_SUCCESS != err) {
620 break;
621 }
622
623 gIOHibernateStandbyDisabled = ((!haveSwapPin || !hibFileSSD));
624
625 dsSSD = ((0 != (kIOPolledFileSSD & vars->fileVars->flags))
626 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
627
628 if (dsSSD) {
629 gIOHibernateCurrentHeader->options |= kIOHibernateOptionSSD | kIOHibernateOptionColor;
630 } else {
631 gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
632 }
633
634 #if HIBERNATE_HMAC_IMAGE
635 // inform HMAC driver that we're going to hibernate
636 ppl_hmac_hibernate_begin();
637 #endif /* HIBERNATE_HMAC_IMAGE */
638
639 #if defined(__i386__) || defined(__x86_64__)
640 if (vars->volumeCryptKeySize &&
641 (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey))) {
642 uintptr_t smcVars[2];
643 smcVars[0] = vars->volumeCryptKeySize;
644 smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
645
646 IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
647 bzero(smcVars, sizeof(smcVars));
648 }
649 #endif
650
651
652 if (encryptedswap || vars->volumeCryptKeySize) {
653 gIOHibernateMode ^= kIOHibernateModeEncrypt;
654 }
655
656 if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options) {
657 vars->videoAllocSize = kVideoMapSize;
658 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize, VM_KERN_MEMORY_IOKIT)) {
659 vars->videoMapping = 0;
660 }
661 }
662
663 // generate crypt keys
664 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++) {
665 vars->wiredCryptKey[i] = ((uint8_t) random());
666 }
667 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++) {
668 vars->cryptKey[i] = ((uint8_t) random());
669 }
670
671 // set nvram
672
673 IOSetBootImageNVRAM(nvramData);
674 nvramData->release();
675
676 #if defined(__i386__) || defined(__x86_64__)
677 {
678 struct AppleRTCHibernateVars {
679 uint8_t signature[4];
680 uint32_t revision;
681 uint8_t booterSignature[20];
682 uint8_t wiredCryptKey[16];
683 };
684 AppleRTCHibernateVars rtcVars;
685 OSData * data;
686
687 rtcVars.signature[0] = 'A';
688 rtcVars.signature[1] = 'A';
689 rtcVars.signature[2] = 'P';
690 rtcVars.signature[3] = 'L';
691 rtcVars.revision = 1;
692 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
693
694 if (gIOChosenEntry
695 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOHibernateBootSignatureKey)))
696 && (sizeof(rtcVars.booterSignature) <= data->getLength())) {
697 bcopy(data->getBytesNoCopy(), &rtcVars.booterSignature[0], sizeof(rtcVars.booterSignature));
698 } else if (gIOHibernateBootSignature[0]) {
699 char c;
700 uint8_t value = 0;
701 uint32_t in, out, digits;
702 for (in = out = digits = 0;
703 (c = gIOHibernateBootSignature[in]) && (in < sizeof(gIOHibernateBootSignature));
704 in++) {
705 if ((c >= 'a') && (c <= 'f')) {
706 c -= 'a' - 10;
707 } else if ((c >= 'A') && (c <= 'F')) {
708 c -= 'A' - 10;
709 } else if ((c >= '0') && (c <= '9')) {
710 c -= '0';
711 } else {
712 if (c == '=') {
713 out = digits = value = 0;
714 }
715 continue;
716 }
717 value = ((uint8_t) ((value << 4) | c));
718 if (digits & 1) {
719 rtcVars.booterSignature[out++] = value;
720 if (out >= sizeof(rtcVars.booterSignature)) {
721 break;
722 }
723 }
724 digits++;
725 }
726 }
727 #if DEBUG || DEVELOPMENT
728 if (kIOLogHibernate & gIOKitDebug) {
729 IOKitKernelLogBuffer("H> rtc:",
730 &rtcVars, sizeof(rtcVars), &kprintf);
731 }
732 #endif /* DEBUG || DEVELOPMENT */
733
734 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
735 if (data) {
736 if (gIOHibernateRTCVariablesKey) {
737 IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
738 }
739 data->release();
740 }
741 if (gIOChosenEntry && gIOOptionsEntry) {
742 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
743 if (data) {
744 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
745 }
746 // set BootNext
747 if (!gIOHibernateBoot0082Data) {
748 OSData * fileData = NULL;
749 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
750 if (data && data->getLength() >= 4) {
751 fileData = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-file-path"));
752 }
753 if (data && (data->getLength() <= UINT16_MAX)) {
754 // AppleNVRAM_EFI_LOAD_OPTION
755 struct {
756 uint32_t Attributes;
757 uint16_t FilePathLength;
758 uint16_t Desc;
759 } loadOptionHeader;
760 loadOptionHeader.Attributes = 1;
761 loadOptionHeader.FilePathLength = ((uint16_t) data->getLength());
762 loadOptionHeader.Desc = 0;
763 if (fileData) {
764 loadOptionHeader.FilePathLength -= 4;
765 loadOptionHeader.FilePathLength += fileData->getLength();
766 }
767 gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
768 if (gIOHibernateBoot0082Data) {
769 gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
770 if (fileData) {
771 gIOHibernateBoot0082Data->appendBytes(data->getBytesNoCopy(), data->getLength() - 4);
772 gIOHibernateBoot0082Data->appendBytes(fileData);
773 } else {
774 gIOHibernateBoot0082Data->appendBytes(data);
775 }
776 }
777 }
778 }
779 if (!gIOHibernateBootNextData) {
780 uint16_t bits = 0x0082;
781 gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
782 }
783
784 #if DEBUG || DEVELOPMENT
785 if (kIOLogHibernate & gIOKitDebug) {
786 IOKitKernelLogBuffer("H> bootnext:",
787 gIOHibernateBoot0082Data->getBytesNoCopy(), gIOHibernateBoot0082Data->getLength(), &kprintf);
788 }
789 #endif /* DEBUG || DEVELOPMENT */
790 if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData) {
791 gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
792 gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
793 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
794 }
795 // BootNext
796 }
797 }
798 #endif /* !i386 && !x86_64 */
799 }while (false);
800
801 if (swapPinned) {
802 hibernate_pin_swap(FALSE);
803 }
804
805 IOLockLock(gFSLock);
806 if ((kIOReturnSuccess == err) && (kFSOpening != gFSState)) {
807 HIBLOG("hibernate file close due timeout\n");
808 err = kIOReturnTimeout;
809 }
810 if (kIOReturnSuccess == err) {
811 gFSState = kFSOpened;
812 gIOHibernateVars = *vars;
813 gFileVars = *vars->fileVars;
814 gFileVars.allocated = false;
815 gIOHibernateVars.fileVars = &gFileVars;
816 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
817 gIOHibernateCurrentHeader->kernVirtSlide = vm_kernel_slide;
818 gIOHibernateState = kIOHibernateStateHibernating;
819
820 #if DEBUG || DEVELOPMENT
821 if (kIOLogHibernate & gIOKitDebug) {
822 OSData * data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
823 if (data) {
824 uintptr_t * smcVars = (typeof(smcVars))data->getBytesNoCopy();
825 IOKitKernelLogBuffer("H> smc:",
826 (const void *)smcVars[1], smcVars[0], &kprintf);
827 }
828 }
829 #endif /* DEBUG || DEVELOPMENT */
830 } else {
831 IOPolledFileIOVars * fileVars = vars->fileVars;
832 IOHibernateDone(vars);
833 IOPolledFileClose(&fileVars,
834 #if DISABLE_TRIM
835 0, NULL, 0, 0, 0);
836 #else
837 0, NULL, 0, sizeof(IOHibernateImageHeader), setFileSize);
838 #endif
839 gFSState = kFSIdle;
840 }
841 IOLockUnlock(gFSLock);
842
843 if (vars->fileVars) {
844 IODelete(vars->fileVars, IOPolledFileIOVars, 1);
845 }
846 IODelete(vars, IOHibernateVars, 1);
847
848 return err;
849 }
850
851 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
852
853 static void
854 IOSetBootImageNVRAM(OSData * data)
855 {
856 IORegistryEntry * regEntry;
857
858 if (!gIOOptionsEntry) {
859 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
860 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
861 if (regEntry && !gIOOptionsEntry) {
862 regEntry->release();
863 }
864 }
865 if (gIOOptionsEntry && gIOHibernateBootImageKey) {
866 if (data) {
867 gIOOptionsEntry->setProperty(gIOHibernateBootImageKey, data);
868 #if DEBUG || DEVELOPMENT
869 if (kIOLogHibernate & gIOKitDebug) {
870 IOKitKernelLogBuffer("H> boot-image:",
871 data->getBytesNoCopy(), data->getLength(), &kprintf);
872 }
873 #endif /* DEBUG || DEVELOPMENT */
874 } else {
875 gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
876 #if __x86_64__
877 gIOOptionsEntry->sync();
878 #else
879 if (gIOHibernateState == kIOHibernateStateWakingFromHibernate) {
880 // if we woke from hibernation, the booter may have changed the state of NVRAM, so force a sync
881 gIOOptionsEntry->sync();
882 }
883 #endif
884 }
885 }
886 }
887
888 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
889 /*
890 * Writes header to disk with signature, block size and file extents data.
891 * If there are more than 2 extents, then they are written on second block.
892 */
893 static IOReturn
894 IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature)
895 {
896 IOHibernateImageHeader hdr;
897 IOItemCount count;
898 IOReturn err = kIOReturnSuccess;
899 int rc;
900 IOPolledFileExtent * fileExtents;
901
902 fileExtents = (typeof(fileExtents))vars->fileExtents->getBytesNoCopy();
903
904 memset(&hdr, 0, sizeof(IOHibernateImageHeader));
905 count = vars->fileExtents->getLength();
906 if (count > sizeof(hdr.fileExtentMap)) {
907 hdr.fileExtentMapSize = count;
908 count = sizeof(hdr.fileExtentMap);
909 } else {
910 hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap);
911 }
912
913 bcopy(fileExtents, &hdr.fileExtentMap[0], count);
914
915 // copy file block extent list if larger than header
916 if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap)) {
917 count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap);
918 rc = kern_write_file(vars->fileRef, vars->blockSize,
919 (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)),
920 count, IO_SKIP_ENCRYPTION);
921 if (rc != 0) {
922 HIBLOG("kern_write_file returned %d\n", rc);
923 err = kIOReturnIOError;
924 goto exit;
925 }
926 }
927 hdr.signature = signature;
928 hdr.deviceBlockSize = vars->blockSize;
929
930 rc = kern_write_file(vars->fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION);
931 if (rc != 0) {
932 HIBLOG("kern_write_file returned %d\n", rc);
933 err = kIOReturnIOError;
934 goto exit;
935 }
936
937 exit:
938 return err;
939 }
940
941 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
942
943 DECLARE_IOHIBERNATEPROGRESSALPHA
944
945 static void
946 ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
947 {
948 uint32_t rowBytes, pixelShift;
949 uint32_t x, y;
950 int32_t blob;
951 uint32_t alpha, color, result;
952 uint8_t * out, in;
953 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
954
955 rowBytes = display->rowBytes;
956 pixelShift = display->depth >> 4;
957 if (pixelShift < 1) {
958 return;
959 }
960
961 screen += ((display->width
962 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
963 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
964
965 for (y = 0; y < kIOHibernateProgressHeight; y++) {
966 out = screen + y * rowBytes;
967 for (blob = 0; blob < kIOHibernateProgressCount; blob++) {
968 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
969 for (x = 0; x < kIOHibernateProgressWidth; x++) {
970 alpha = gIOHibernateProgressAlpha[y][x];
971 result = color;
972 if (alpha) {
973 if (0xff != alpha) {
974 if (1 == pixelShift) {
975 in = *((uint16_t *)out) & 0x1f; // 16
976 in = ((uint8_t)(in << 3)) | ((uint8_t)(in >> 2));
977 } else {
978 in = *((uint32_t *)out) & 0xff; // 32
979 }
980 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
981 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
982 }
983 if (1 == pixelShift) {
984 result >>= 3;
985 *((uint16_t *)out) = ((uint16_t)((result << 10) | (result << 5) | result)); // 16
986 } else {
987 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
988 }
989 }
990 out += (1 << pixelShift);
991 }
992 out += (kIOHibernateProgressSpacing << pixelShift);
993 }
994 }
995 }
996
997
998 static void
999 ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
1000 {
1001 uint32_t rowBytes, pixelShift;
1002 uint32_t x, y;
1003 int32_t blob, lastBlob;
1004 uint32_t alpha, in, color, result;
1005 uint8_t * out;
1006 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1007
1008 pixelShift = display->depth >> 4;
1009 if (pixelShift < 1) {
1010 return;
1011 }
1012
1013 rowBytes = display->rowBytes;
1014
1015 screen += ((display->width
1016 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1017 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1018
1019 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
1020
1021 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
1022
1023 for (y = 0; y < kIOHibernateProgressHeight; y++) {
1024 out = screen + y * rowBytes;
1025 for (blob = firstBlob; blob <= lastBlob; blob++) {
1026 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
1027 for (x = 0; x < kIOHibernateProgressWidth; x++) {
1028 alpha = gIOHibernateProgressAlpha[y][x];
1029 result = color;
1030 if (alpha) {
1031 if (0xff != alpha) {
1032 in = display->progressSaveUnder[blob][saveindex[blob]++];
1033 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
1034 }
1035 if (1 == pixelShift) {
1036 result >>= 3;
1037 *((uint16_t *)out) = ((uint16_t)((result << 10) | (result << 5) | result)); // 16
1038 } else {
1039 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1040 }
1041 }
1042 out += (1 << pixelShift);
1043 }
1044 out += (kIOHibernateProgressSpacing << pixelShift);
1045 }
1046 }
1047 }
1048
1049 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1050
1051 IOReturn
1052 IOHibernateIOKitSleep(void)
1053 {
1054 IOReturn ret = kIOReturnSuccess;
1055 IOLockLock(gFSLock);
1056 if (kFSOpening == gFSState) {
1057 gFSState = kFSTimedOut;
1058 HIBLOG("hibernate file open timed out\n");
1059 ret = kIOReturnTimeout;
1060 }
1061 IOLockUnlock(gFSLock);
1062 return ret;
1063 }
1064
1065 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1066
1067 IOReturn
1068 IOHibernateSystemHasSlept(void)
1069 {
1070 IOReturn ret = kIOReturnSuccess;
1071 IOHibernateVars * vars = &gIOHibernateVars;
1072 OSObject * obj = NULL;
1073 OSData * data;
1074
1075 IOLockLock(gFSLock);
1076 if ((kFSOpened != gFSState) && gIOHibernateMode) {
1077 ret = kIOReturnTimeout;
1078 }
1079 IOLockUnlock(gFSLock);
1080 if (kIOReturnSuccess != ret) {
1081 return ret;
1082 }
1083
1084 if (gIOHibernateMode) {
1085 obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
1086 }
1087 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1088 if (obj && !vars->previewBuffer) {
1089 obj->release();
1090 }
1091 if (vars->previewBuffer && (vars->previewBuffer->getLength() > UINT_MAX)) {
1092 OSSafeReleaseNULL(vars->previewBuffer);
1093 }
1094
1095 vars->consoleMapping = NULL;
1096 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare())) {
1097 vars->previewBuffer->release();
1098 vars->previewBuffer = NULL;
1099 }
1100
1101 if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1102 && vars->previewBuffer
1103 && (data = OSDynamicCast(OSData,
1104 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey)))) {
1105 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
1106 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
1107
1108 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1109
1110 if (kIOHibernatePreviewUpdates & flags) {
1111 PE_Video consoleInfo;
1112 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1113
1114 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1115
1116 graphicsInfo->width = (uint32_t) consoleInfo.v_width;
1117 graphicsInfo->height = (uint32_t) consoleInfo.v_height;
1118 graphicsInfo->rowBytes = (uint32_t) consoleInfo.v_rowBytes;
1119 graphicsInfo->depth = (uint32_t) consoleInfo.v_depth;
1120 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1121
1122 HIBPRINT("video %p %d %d %d\n",
1123 vars->consoleMapping, graphicsInfo->depth,
1124 graphicsInfo->width, graphicsInfo->height);
1125 if (vars->consoleMapping) {
1126 ProgressInit(graphicsInfo, vars->consoleMapping,
1127 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1128 }
1129 }
1130 }
1131
1132 if (gIOOptionsEntry) {
1133 #if __x86_64__
1134 gIOOptionsEntry->sync();
1135 #else
1136 if (gIOHibernateMode) {
1137 gIOOptionsEntry->sync();
1138 }
1139 #endif
1140 }
1141
1142 return ret;
1143 }
1144
1145 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1146
1147 #if defined(__i386__) || defined(__x86_64__)
1148 static DeviceTreeNode *
1149 MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry, vm_offset_t region_start, vm_size_t region_size)
1150 {
1151 DeviceTreeNodeProperty * prop;
1152 DeviceTreeNode * child;
1153 IORegistryEntry * childRegEntry;
1154 const char * nameProp;
1155 unsigned int propLen, idx;
1156
1157 prop = (DeviceTreeNodeProperty *) (entry + 1);
1158 for (idx = 0; idx < entry->nProperties; idx++) {
1159 if (regEntry && (0 != strcmp("name", prop->name))) {
1160 regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
1161 // HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
1162 }
1163 prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
1164 }
1165
1166 child = (DeviceTreeNode *) prop;
1167 for (idx = 0; idx < entry->nChildren; idx++) {
1168 if (kSuccess != SecureDTGetPropertyRegion(child, "name", (void const **) &nameProp, &propLen,
1169 region_start, region_size)) {
1170 panic("no name");
1171 }
1172 childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
1173 // HIBPRINT("%s == %p\n", nameProp, childRegEntry);
1174 child = MergeDeviceTree(child, childRegEntry, region_start, region_size);
1175 }
1176 return child;
1177 }
1178 #endif
1179
1180
1181 IOReturn
1182 IOHibernateSystemWake(void)
1183 {
1184 if (kFSOpened == gFSState) {
1185 IOPolledFilePollersClose(gIOHibernateVars.fileVars, kIOPolledPostflightState);
1186 IOHibernateDone(&gIOHibernateVars);
1187 } else {
1188 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1189 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1190 }
1191
1192 if (gIOOptionsEntry && gIOHibernateBootImageKey) {
1193 // if we got this far, clear boot-image
1194 // we don't need to sync immediately; the booter should have already removed this entry
1195 // we just want to make sure that if anyone syncs nvram after this point, we don't re-write
1196 // a stale boot-image value
1197 gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
1198 }
1199
1200 return kIOReturnSuccess;
1201 }
1202
1203 static IOReturn
1204 IOHibernateDone(IOHibernateVars * vars)
1205 {
1206 IOReturn err;
1207 OSData * data;
1208
1209 hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
1210
1211 if (vars->videoMapping) {
1212 if (vars->videoMapSize) {
1213 // remove mappings
1214 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1215 }
1216 if (vars->videoAllocSize) {
1217 // dealloc range
1218 kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
1219 }
1220 }
1221
1222 if (vars->previewBuffer) {
1223 vars->previewBuffer->release();
1224 vars->previewBuffer = NULL;
1225 }
1226
1227 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
1228 IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
1229 gIOHibernateCurrentHeader->options, 32);
1230 } else {
1231 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1232 }
1233
1234 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1235 && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus)) {
1236 IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
1237 &gIOHibernateGraphicsInfo->gfxStatus,
1238 sizeof(gIOHibernateGraphicsInfo->gfxStatus));
1239 } else {
1240 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1241 }
1242
1243 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1244
1245 #if defined(__i386__) || defined(__x86_64__)
1246 IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
1247 IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
1248
1249 /*
1250 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1251 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1252 */
1253 if (gIOOptionsEntry) {
1254 if (gIOHibernateRTCVariablesKey) {
1255 if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
1256 gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
1257 }
1258 }
1259
1260 if (gIOHibernateBootNextKey) {
1261 if (gIOHibernateBootNextSave) {
1262 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
1263 gIOHibernateBootNextSave->release();
1264 gIOHibernateBootNextSave = NULL;
1265 } else {
1266 gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
1267 }
1268 }
1269 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) {
1270 gIOOptionsEntry->sync();
1271 }
1272 }
1273 #endif
1274
1275 if (vars->srcBuffer) {
1276 vars->srcBuffer->release();
1277 }
1278
1279 #if HIBERNATE_HMAC_IMAGE
1280 // inform HMAC driver that we're done hibernating
1281 ppl_hmac_hibernate_end();
1282 #endif /* HIBERNATE_HMAC_IMAGE */
1283
1284 bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
1285 if (vars->handoffBuffer) {
1286 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
1287 IOHibernateHandoff * handoff;
1288 bool done = false;
1289 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
1290 !done;
1291 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount]) {
1292 HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
1293 uint8_t * __unused data = &handoff->data[0];
1294 switch (handoff->type) {
1295 case kIOHibernateHandoffTypeEnd:
1296 done = true;
1297 break;
1298
1299 case kIOHibernateHandoffTypeDeviceTree:
1300 #if defined(__i386__) || defined(__x86_64__)
1301 MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot(),
1302 (vm_offset_t)data, (vm_size_t)handoff->bytecount);
1303 #else
1304 // On ARM, the device tree is confined to its region covered by CTRR, so effectively immutable.
1305 panic("kIOHibernateHandoffTypeDeviceTree not supported on this platform.");
1306 #endif
1307 break;
1308
1309 case kIOHibernateHandoffTypeKeyStore:
1310 #if defined(__i386__) || defined(__x86_64__)
1311 {
1312 IOBufferMemoryDescriptor *
1313 md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
1314 if (md) {
1315 IOSetKeyStoreData(md);
1316 }
1317 }
1318 #endif
1319 break;
1320
1321 default:
1322 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
1323 break;
1324 }
1325 }
1326 #if defined(__i386__) || defined(__x86_64__)
1327 if (vars->volumeCryptKeySize) {
1328 IOBufferMemoryDescriptor *
1329 bmd = IOBufferMemoryDescriptor::withBytes(&vars->volumeCryptKey[0],
1330 vars->volumeCryptKeySize, kIODirectionOutIn);
1331 if (!bmd) {
1332 panic("IOBufferMemoryDescriptor");
1333 }
1334 IOSetAPFSKeyStoreData(bmd);
1335 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
1336 }
1337 #endif
1338 }
1339 vars->handoffBuffer->release();
1340 }
1341
1342 if (gIOChosenEntry
1343 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey)))
1344 && (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength())) {
1345 bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0],
1346 sizeof(gIOHibernateBridgeBootSessionUUIDString));
1347 }
1348
1349 if (vars->hwEncrypt) {
1350 err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, NULL, 0);
1351 HIBLOG("IOPolledFilePollersSetEncryptionKey(0,%x)\n", err);
1352 }
1353
1354 bzero(vars, sizeof(*vars));
1355
1356 // gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1357 gIOHibernateCount++;
1358
1359 return kIOReturnSuccess;
1360 }
1361
1362 static void
1363 IOHibernateSystemPostWakeTrim(void * p1, void * p2)
1364 {
1365 // invalidate & close the image file
1366 if (p1) {
1367 IOLockLock(gFSLock);
1368 }
1369 if (kFSTrimDelay == gFSState) {
1370 IOPolledFileIOVars * vars = &gFileVars;
1371 IOPolledFileClose(&vars,
1372 #if DISABLE_TRIM
1373 0, NULL, 0, 0, 0);
1374 #else
1375 0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader),
1376 sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize);
1377 #endif
1378 gFSState = kFSIdle;
1379 }
1380 if (p1) {
1381 IOLockUnlock(gFSLock);
1382 }
1383 }
1384
1385 IOReturn
1386 IOHibernateSystemPostWake(bool now)
1387 {
1388 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1389 IOSetBootImageNVRAM(NULL);
1390
1391 IOLockLock(gFSLock);
1392 if (kFSTrimDelay == gFSState) {
1393 thread_call_cancel(gIOHibernateTrimCalloutEntry);
1394 IOHibernateSystemPostWakeTrim(NULL, NULL);
1395 } else if (kFSOpened != gFSState) {
1396 gFSState = kFSIdle;
1397 } else {
1398 gFSState = kFSTrimDelay;
1399 if (now) {
1400 thread_call_cancel(gIOHibernateTrimCalloutEntry);
1401 IOHibernateSystemPostWakeTrim(NULL, NULL);
1402 } else {
1403 AbsoluteTime deadline;
1404 clock_interval_to_deadline(TRIM_DELAY, kMillisecondScale, &deadline );
1405 thread_call_enter1_delayed(gIOHibernateTrimCalloutEntry, NULL, deadline);
1406 }
1407 }
1408 IOLockUnlock(gFSLock);
1409
1410 return kIOReturnSuccess;
1411 }
1412
1413 uint32_t
1414 IOHibernateWasScreenLocked(void)
1415 {
1416 uint32_t ret = 0;
1417 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry) {
1418 OSData *
1419 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
1420 if (data) {
1421 ret = ((uint32_t *)data->getBytesNoCopy())[0];
1422 gIOChosenEntry->setProperty(kIOBooterScreenLockStateKey, data);
1423 }
1424 } else {
1425 gIOChosenEntry->removeProperty(kIOBooterScreenLockStateKey);
1426 }
1427
1428 return ret;
1429 }
1430
1431 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1432
1433 SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
1434 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1435 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
1436 SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
1437 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1438 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
1439 SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
1440 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1441 &gIOHibernateMode, 0, "");
1442 SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
1443 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1444 &_hibernateStats, hibernate_statistics_t, "");
1445 SYSCTL_STRING(_kern_bridge, OID_AUTO, bootsessionuuid,
1446 CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1447 gIOHibernateBridgeBootSessionUUIDString, sizeof(gIOHibernateBridgeBootSessionUUIDString), "");
1448
1449 SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
1450 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1451 &_hibernateStats.graphicsReadyTime, 0, "");
1452 SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
1453 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1454 &_hibernateStats.wakeNotificationTime, 0, "");
1455 SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
1456 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1457 &_hibernateStats.lockScreenReadyTime, 0, "");
1458 SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
1459 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1460 &_hibernateStats.hidReadyTime, 0, "");
1461
1462 SYSCTL_UINT(_kern, OID_AUTO, hibernatecount,
1463 CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1464 &gIOHibernateCount, 0, "");
1465
1466 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1467
1468 static int
1469 hibernate_set_preview SYSCTL_HANDLER_ARGS
1470 {
1471 #pragma unused(oidp, arg1, arg2)
1472
1473 if (!IOTaskHasEntitlement(current_task(), kIOHibernateSetPreviewEntitlementKey)) {
1474 return EPERM;
1475 }
1476
1477 if ((req->newptr == USER_ADDR_NULL) || (!req->newlen)) {
1478 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewBufferKey);
1479 return 0;
1480 }
1481
1482 size_t rounded_size = round_page(req->newlen);
1483 IOBufferMemoryDescriptor *md = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, rounded_size, page_size);
1484 if (!md) {
1485 return ENOMEM;
1486 }
1487
1488 uint8_t *bytes = (uint8_t *)md->getBytesNoCopy();
1489 int error = SYSCTL_IN(req, bytes, req->newlen);
1490 if (error) {
1491 md->release();
1492 return error;
1493 }
1494
1495 IOService::getPMRootDomain()->setProperty(kIOHibernatePreviewBufferKey, md);
1496 md->release();
1497
1498 return 0;
1499 }
1500
1501 SYSCTL_PROC(_kern, OID_AUTO, hibernatepreview,
1502 CTLTYPE_OPAQUE | CTLFLAG_WR | CTLFLAG_LOCKED | CTLFLAG_ANYBODY, NULL, 0,
1503 hibernate_set_preview, "S", "");
1504
1505 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1506
1507 void
1508 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1509 {
1510 gIOHibernateBootImageKey = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1511 gIOHibernateBootSignatureKey = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
1512 gIOBridgeBootSessionUUIDKey = OSSymbol::withCStringNoCopy(kIOBridgeBootSessionUUIDKey);
1513
1514 #if defined(__i386__) || defined(__x86_64__)
1515 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1516 gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
1517 gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
1518 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1519 #endif /* defined(__i386__) || defined(__x86_64__) */
1520
1521 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
1522 if (data) {
1523 rootDomain->setProperty(kIOHibernateStateKey, data);
1524 data->release();
1525 }
1526
1527 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename))) {
1528 gIOHibernateMode = kIOHibernateModeOn;
1529 } else {
1530 gIOHibernateFilename[0] = 0;
1531 }
1532
1533 sysctl_register_oid(&sysctl__kern_hibernatefile);
1534 sysctl_register_oid(&sysctl__kern_bootsignature);
1535 sysctl_register_oid(&sysctl__kern_hibernatemode);
1536 sysctl_register_oid(&sysctl__kern_hibernatestatistics);
1537 sysctl_register_oid(&sysctl__kern_hibernategraphicsready);
1538 sysctl_register_oid(&sysctl__kern_hibernatewakenotification);
1539 sysctl_register_oid(&sysctl__kern_hibernatelockscreenready);
1540 sysctl_register_oid(&sysctl__kern_hibernatehidready);
1541 sysctl_register_oid(&sysctl__kern_hibernatecount);
1542
1543 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1544
1545 if (gIOChosenEntry
1546 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey)))
1547 && (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength())) {
1548 sysctl_register_oid(&sysctl__kern_bridge_bootsessionuuid);
1549 bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0], sizeof(gIOHibernateBridgeBootSessionUUIDString));
1550 }
1551
1552 gFSLock = IOLockAlloc();
1553 gIOHibernateCount = 0;
1554 }
1555
1556 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1557
1558 static IOReturn
1559 IOHibernatePolledFileWrite(IOHibernateVars * vars,
1560 const uint8_t * bytes, IOByteCount size,
1561 IOPolledFileCryptVars * cryptvars)
1562 {
1563 IOReturn err;
1564
1565 #if HIBERNATE_HMAC_IMAGE
1566 uint64_t originalPosition = 0;
1567 if (!bytes && !size) {
1568 originalPosition = vars->fileVars->position;
1569 }
1570 #endif /* HIBERNATE_HMAC_IMAGE */
1571
1572 err = IOPolledFileWrite(vars->fileVars, bytes, size, cryptvars);
1573 if ((kIOReturnSuccess == err) && hibernate_should_abort()) {
1574 err = kIOReturnAborted;
1575 }
1576
1577 #if HIBERNATE_HMAC_IMAGE
1578 if ((kIOReturnSuccess == err) && (vars->imageShaCtx)) {
1579 if (!bytes && !size) {
1580 // determine how many bytes were written
1581 size = vars->fileVars->position - originalPosition;
1582 }
1583 if (bytes) {
1584 SHA256_Update(vars->imageShaCtx, bytes, size);
1585 } else {
1586 // update with zeroes
1587 uint8_t zeroes[512] = {};
1588 size_t len = size;
1589 while (len) {
1590 IOByteCount toHash = min(len, sizeof(zeroes));
1591 SHA256_Update(vars->imageShaCtx, zeroes, toHash);
1592 len -= toHash;
1593 }
1594 }
1595 }
1596 #endif /* HIBERNATE_HMAC_IMAGE */
1597
1598 return err;
1599 }
1600
1601 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1602
1603 extern "C" uint32_t
1604 hibernate_write_image(void)
1605 {
1606 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
1607 IOHibernateVars * vars = &gIOHibernateVars;
1608 IOPolledFileExtent * fileExtents;
1609
1610 #if !defined(__arm64__)
1611 _static_assert_1_arg(sizeof(IOHibernateImageHeader) == 512);
1612 #endif /* !defined(__arm64__) */
1613
1614 uint32_t pageCount, pagesDone;
1615 IOReturn err;
1616 ppnum_t ppnum, page;
1617 vm_offset_t count;
1618 uint8_t * src;
1619 uint8_t * data;
1620 uint8_t * compressed;
1621 uint8_t * scratch;
1622 IOByteCount pageCompressedSize;
1623 uint64_t compressedSize, uncompressedSize;
1624 uint64_t image1Size = 0;
1625 uint32_t bitmap_size;
1626 bool iterDone, pollerOpen, needEncrypt;
1627 int wkresult;
1628 uint32_t tag;
1629 uint32_t pageType;
1630 uint32_t pageAndCount[2];
1631 addr64_t phys64;
1632 IOByteCount segLen;
1633 #if !HIBERNATE_HMAC_IMAGE
1634 uint32_t restore1Sum = 0, sum = 0, sum1 = 0, sum2 = 0;
1635 uintptr_t hibernateBase;
1636 uintptr_t hibernateEnd;
1637 #endif /* HIBERNATE_HMAC_IMAGE */
1638
1639 AbsoluteTime startTime, endTime;
1640 AbsoluteTime allTime, compTime;
1641 uint64_t compBytes;
1642 uint64_t nsec;
1643 uint64_t lastProgressStamp = 0;
1644 uint64_t progressStamp;
1645 uint32_t blob, lastBlob = (uint32_t) -1L;
1646
1647 uint32_t wiredPagesEncrypted;
1648 uint32_t dirtyPagesEncrypted;
1649 uint32_t wiredPagesClear;
1650 uint32_t svPageCount;
1651 uint32_t zvPageCount;
1652
1653 IOPolledFileCryptVars _cryptvars;
1654 IOPolledFileCryptVars * cryptvars = NULL;
1655
1656 wiredPagesEncrypted = 0;
1657 dirtyPagesEncrypted = 0;
1658 wiredPagesClear = 0;
1659 svPageCount = 0;
1660 zvPageCount = 0;
1661
1662 if (!vars->fileVars
1663 || !vars->fileVars->pollers
1664 || !(kIOHibernateModeOn & gIOHibernateMode)) {
1665 return kIOHibernatePostWriteSleep;
1666 }
1667
1668 #if HIBERNATE_HMAC_IMAGE
1669 // set up SHA and HMAC context to hash image1 (wired pages)
1670 SHA256_CTX imageShaCtx;
1671 vars->imageShaCtx = &imageShaCtx;
1672 SHA256_Init(vars->imageShaCtx);
1673 ppl_hmac_reset(true);
1674 #endif /* HIBERNATE_HMAC_IMAGE */
1675
1676 #if !defined(__arm64__)
1677 if (kIOHibernateModeSleep & gIOHibernateMode) {
1678 kdebug_enable = save_kdebug_enable;
1679 }
1680 #endif /* !defined(__arm64__) */
1681
1682 pal_hib_write_hook();
1683
1684 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START);
1685 IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
1686
1687 #if CRYPTO
1688 // encryption data. "iv" is the "initial vector".
1689 if (kIOHibernateModeEncrypt & gIOHibernateMode) {
1690 static const unsigned char first_iv[AES_BLOCK_SIZE]
1691 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
1692 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
1693
1694 cryptvars = &gIOHibernateCryptWakeContext;
1695 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1696 aes_encrypt_key(vars->cryptKey,
1697 kIOHibernateAESKeySize,
1698 &cryptvars->ctx.encrypt);
1699 aes_decrypt_key(vars->cryptKey,
1700 kIOHibernateAESKeySize,
1701 &cryptvars->ctx.decrypt);
1702
1703 cryptvars = &_cryptvars;
1704 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1705 for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++) {
1706 vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
1707 }
1708 aes_encrypt_key(vars->wiredCryptKey,
1709 kIOHibernateAESKeySize,
1710 &cryptvars->ctx.encrypt);
1711
1712 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1713 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1714 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
1715 }
1716 #endif /* CRYPTO */
1717
1718 hibernate_page_list_setall(vars->page_list,
1719 vars->page_list_wired,
1720 vars->page_list_pal,
1721 false /* !preflight */,
1722 /* discard_all */
1723 ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
1724 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
1725 &pageCount);
1726
1727 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
1728
1729 fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy();
1730
1731 #if 0
1732 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
1733 for (page = 0; page < count; page++) {
1734 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
1735 fileExtents[page].start, fileExtents[page].length,
1736 fileExtents[page].start + fileExtents[page].length);
1737 }
1738 #endif
1739
1740 needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
1741 AbsoluteTime_to_scalar(&compTime) = 0;
1742 compBytes = 0;
1743
1744 clock_get_uptime(&allTime);
1745 IOService::getPMRootDomain()->pmStatsRecordEvent(
1746 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
1747 do{
1748 compressedSize = 0;
1749 uncompressedSize = 0;
1750 svPageCount = 0;
1751 zvPageCount = 0;
1752
1753 IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
1754
1755 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
1756 ml_get_interrupts_enabled());
1757 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState,
1758 // abortable if not low battery
1759 !IOService::getPMRootDomain()->mustHibernate());
1760 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
1761 pollerOpen = (kIOReturnSuccess == err);
1762 if (!pollerOpen) {
1763 break;
1764 }
1765
1766 if (vars->volumeCryptKeySize) {
1767 err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
1768 HIBLOG("IOPolledFilePollersSetEncryptionKey(%x)\n", err);
1769 vars->hwEncrypt = (kIOReturnSuccess == err);
1770 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
1771 if (vars->hwEncrypt) {
1772 header->options |= kIOHibernateOptionHWEncrypt;
1773 }
1774 }
1775
1776 // copy file block extent list if larger than header
1777
1778 count = vars->fileVars->fileExtents->getLength();
1779 if (count > sizeof(header->fileExtentMap)) {
1780 count -= sizeof(header->fileExtentMap);
1781 err = IOHibernatePolledFileWrite(vars,
1782 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
1783 if (kIOReturnSuccess != err) {
1784 break;
1785 }
1786 }
1787
1788 // copy out restore1 code
1789
1790 for (count = 0;
1791 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1792 count += segLen) {
1793 for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++) {
1794 gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64_ppnum(phys64) + pagesDone;
1795 }
1796 }
1797
1798 #if HIBERNATE_HMAC_IMAGE
1799 if (vars->fileVars->position > UINT32_MAX) {
1800 err = kIOReturnNoSpace;
1801 break;
1802 }
1803 header->segmentsFileOffset = (uint32_t)vars->fileVars->position;
1804
1805 // fetch the IOHibernateHibSegInfo and the actual pages to write
1806 // we use srcBuffer as scratch space
1807 IOHibernateHibSegInfo *segInfo = &header->hibSegInfo;
1808 void *segInfoScratch = vars->srcBuffer->getBytesNoCopy();
1809
1810 // This call also enables PMAP hibernation asserts which will prevent modification
1811 // of PMAP data structures. This needs to occur before pages start getting written
1812 // into the image.
1813 ppl_hmac_fetch_hibseg_and_info(segInfoScratch, vars->srcBuffer->getCapacity(), segInfo);
1814
1815 // write each segment to the file
1816 size_t segInfoScratchPos = 0;
1817 int hibSectIdx = -1;
1818 uint32_t hibSegPageCount = 0;
1819 for (int i = 0; i < NUM_HIBSEGINFO_SEGMENTS; i++) {
1820 hibSegPageCount += segInfo->segments[i].pageCount;
1821 size_t size = ptoa_64(segInfo->segments[i].pageCount);
1822 if (size) {
1823 err = IOHibernatePolledFileWrite(vars,
1824 (uint8_t*)segInfoScratch + segInfoScratchPos, size, cryptvars);
1825 if (kIOReturnSuccess != err) {
1826 break;
1827 }
1828 segInfoScratchPos += size;
1829
1830 // is this sectHIBTEXTB?
1831 if (ptoa_64(segInfo->segments[i].physPage) == trunc_page(kvtophys(sectHIBTEXTB))) {
1832 // remember which segment is sectHIBTEXTB because we'll need it later
1833 hibSectIdx = i;
1834 }
1835 }
1836 }
1837
1838 if (hibSectIdx == -1) {
1839 panic("couldn't find sectHIBTEXTB in segInfo");
1840 }
1841
1842 // set the header fields corresponding to the HIB segments
1843 header->restore1CodePhysPage = segInfo->segments[hibSectIdx].physPage;
1844 header->restore1CodeVirt = trunc_page(sectHIBTEXTB);
1845 header->restore1PageCount = hibSegPageCount;
1846 header->restore1CodeOffset = (uint32_t)(((uintptr_t) &hibernate_machine_entrypoint) - header->restore1CodeVirt);
1847
1848 // set restore1StackOffset to the physical page of the top of the stack to simplify the restore code
1849 vm_offset_t stackFirstPage, stackPageSize;
1850 pal_hib_get_stack_pages(&stackFirstPage, &stackPageSize);
1851 header->restore1StackOffset = (uint32_t)(stackFirstPage + stackPageSize);
1852 #else /* !HIBERNATE_HMAC_IMAGE */
1853 hibernateBase = HIB_BASE; /* Defined in PAL headers */
1854 hibernateEnd = (segHIBB + segSizeHIB);
1855
1856 page = atop_32(kvtophys(hibernateBase));
1857 count = atop_32(round_page(hibernateEnd) - hibernateBase);
1858 uintptr_t entrypoint = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
1859 uintptr_t stack = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
1860 if ((count > UINT_MAX) || (entrypoint > UINT_MAX) || (stack > UINT_MAX)) {
1861 panic("malformed kernel layout");
1862 }
1863 header->restore1CodePhysPage = (ppnum_t) page;
1864 header->restore1CodeVirt = hibernateBase;
1865 header->restore1PageCount = (uint32_t) count;
1866 header->restore1CodeOffset = (uint32_t) entrypoint;
1867 header->restore1StackOffset = (uint32_t) stack;
1868
1869 if (uuid_parse(&gIOHibernateBridgeBootSessionUUIDString[0], &header->bridgeBootSessionUUID[0])) {
1870 bzero(&header->bridgeBootSessionUUID[0], sizeof(header->bridgeBootSessionUUID));
1871 }
1872
1873 // sum __HIB seg, with zeros for the stack
1874 src = (uint8_t *) trunc_page(hibernateBase);
1875 for (page = 0; page < count; page++) {
1876 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0])) {
1877 restore1Sum += hibernate_sum_page(src, (uint32_t) (header->restore1CodeVirt + page));
1878 } else {
1879 restore1Sum += 0x00000000;
1880 }
1881 src += page_size;
1882 }
1883 sum1 = restore1Sum;
1884
1885 // write the __HIB seg, with zeros for the stack
1886
1887 src = (uint8_t *) trunc_page(hibernateBase);
1888 count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
1889 if (count) {
1890 err = IOHibernatePolledFileWrite(vars, src, count, cryptvars);
1891 if (kIOReturnSuccess != err) {
1892 break;
1893 }
1894 }
1895 err = IOHibernatePolledFileWrite(vars,
1896 (uint8_t *) NULL,
1897 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
1898 cryptvars);
1899 if (kIOReturnSuccess != err) {
1900 break;
1901 }
1902 src = &gIOHibernateRestoreStackEnd[0];
1903 count = round_page(hibernateEnd) - ((uintptr_t) src);
1904 if (count) {
1905 err = IOHibernatePolledFileWrite(vars, src, count, cryptvars);
1906 if (kIOReturnSuccess != err) {
1907 break;
1908 }
1909 }
1910 #endif /* !HIBERNATE_HMAC_IMAGE */
1911
1912 if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) {
1913 vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
1914 vars->fileVars->encryptEnd = UINT64_MAX;
1915 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1916 }
1917
1918 // write the preview buffer
1919
1920 if (vars->previewBuffer) {
1921 ppnum = 0;
1922 count = 0;
1923 do{
1924 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
1925 pageAndCount[0] = atop_64_ppnum(phys64);
1926 pageAndCount[1] = atop_64_ppnum(segLen);
1927 err = IOHibernatePolledFileWrite(vars,
1928 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1929 cryptvars);
1930 if (kIOReturnSuccess != err) {
1931 break;
1932 }
1933 count += segLen;
1934 ppnum += sizeof(pageAndCount);
1935 }while (phys64);
1936 if (kIOReturnSuccess != err) {
1937 break;
1938 }
1939
1940 src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
1941
1942 ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
1943
1944 count = (uint32_t) vars->previewBuffer->getLength();
1945
1946 header->previewPageListSize = ((uint32_t) ppnum);
1947 header->previewSize = ((uint32_t) (count + ppnum));
1948
1949 for (page = 0; page < count; page += page_size) {
1950 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
1951 #if HIBERNATE_HMAC_IMAGE
1952 err = ppl_hmac_update_and_compress_page(atop_64_ppnum(phys64), NULL, NULL);
1953 if (kIOReturnSuccess != err) {
1954 break;
1955 }
1956 #else /* !HIBERNATE_HMAC_IMAGE */
1957 sum1 += hibernate_sum_page(src + page, atop_64_ppnum(phys64));
1958 #endif /* !HIBERNATE_HMAC_IMAGE */
1959 }
1960 if (kIOReturnSuccess != err) {
1961 break;
1962 }
1963 err = IOHibernatePolledFileWrite(vars, src, count, cryptvars);
1964 if (kIOReturnSuccess != err) {
1965 break;
1966 }
1967 }
1968
1969 // mark areas for no save
1970 hibernate_set_descriptor_page_state(vars, IOPolledFileGetIOBuffer(vars->fileVars),
1971 kIOHibernatePageStateFree, &pageCount);
1972 hibernate_set_descriptor_page_state(vars, vars->srcBuffer,
1973 kIOHibernatePageStateFree, &pageCount);
1974
1975 // copy out bitmap of pages available for trashing during restore
1976
1977 bitmap_size = vars->page_list_wired->list_size;
1978 src = (uint8_t *) vars->page_list_wired;
1979 err = IOHibernatePolledFileWrite(vars, src, bitmap_size, cryptvars);
1980 if (kIOReturnSuccess != err) {
1981 break;
1982 }
1983
1984 // mark more areas for no save, but these are not available
1985 // for trashing during restore
1986
1987 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
1988
1989 #if defined(__i386__) || defined(__x86_64__)
1990 // __HIB is explicitly saved above so we don't have to save it again
1991 page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
1992 count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
1993 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1994 page, count,
1995 kIOHibernatePageStateFree);
1996 pageCount -= count;
1997 #elif defined(__arm64__)
1998 // the segments described in IOHibernateHibSegInfo are stored directly in the
1999 // hibernation file, so they don't need to be saved again
2000 extern unsigned long gPhysBase, gPhysSize;
2001 for (size_t i = 0; i < NUM_HIBSEGINFO_SEGMENTS; i++) {
2002 page = segInfo->segments[i].physPage;
2003 count = segInfo->segments[i].pageCount;
2004 uint64_t physAddr = ptoa_64(page);
2005 uint64_t size = ptoa_64(count);
2006 if (size &&
2007 (physAddr >= gPhysBase) &&
2008 (physAddr + size <= gPhysBase + gPhysSize)) {
2009 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2010 page, count,
2011 kIOHibernatePageStateFree);
2012 pageCount -= count;
2013 }
2014 }
2015 #else
2016 #error unimplemented
2017 #endif
2018
2019 hibernate_set_descriptor_page_state(vars, vars->previewBuffer,
2020 kIOHibernatePageStateFree, &pageCount);
2021 hibernate_set_descriptor_page_state(vars, vars->handoffBuffer,
2022 kIOHibernatePageStateFree, &pageCount);
2023
2024 #if KASAN
2025 vm_size_t shadow_pages_free = atop_64(shadow_ptop) - atop_64(shadow_pnext);
2026
2027 /* no need to save unused shadow pages */
2028 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2029 atop_64(shadow_pnext),
2030 shadow_pages_free,
2031 kIOHibernatePageStateFree);
2032 #endif
2033
2034 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2035 compressed = src + page_size;
2036 scratch = compressed + page_size;
2037
2038 pagesDone = 0;
2039 lastBlob = 0;
2040
2041 HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
2042 bitmap_size, header->previewSize,
2043 pageCount, vars->fileVars->position);
2044
2045 #if HIBERNATE_HMAC_IMAGE
2046 // we don't need to sign the page data into imageHeaderHMAC because it's
2047 // already signed into image1PagesHMAC/image2PagesHMAC
2048 vars->imageShaCtx = NULL;
2049 header->imageHeaderHMACSize = (uint32_t)vars->fileVars->position;
2050 #endif /* HIBERNATE_HMAC_IMAGE */
2051
2052 enum
2053 // pageType
2054 {
2055 kWired = 0x02,
2056 kEncrypt = 0x01,
2057 kWiredEncrypt = kWired | kEncrypt,
2058 kWiredClear = kWired,
2059 kUnwiredEncrypt = kEncrypt
2060 };
2061
2062 #if defined(__i386__) || defined(__x86_64__)
2063 bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features()));
2064 #else /* defined(__i386__) || defined(__x86_64__) */
2065 static const bool cpuAES = true;
2066 #endif /* defined(__i386__) || defined(__x86_64__) */
2067
2068 for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--) {
2069 if (kUnwiredEncrypt == pageType) {
2070 // start unwired image
2071 if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) {
2072 vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
2073 vars->fileVars->encryptEnd = UINT64_MAX;
2074 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
2075 }
2076 bcopy(&cryptvars->aes_iv[0],
2077 &gIOHibernateCryptWakeContext.aes_iv[0],
2078 sizeof(cryptvars->aes_iv));
2079 cryptvars = &gIOHibernateCryptWakeContext;
2080 }
2081 for (iterDone = false, ppnum = 0; !iterDone;) {
2082 if (cpuAES && (pageType == kWiredClear)) {
2083 count = 0;
2084 } else {
2085 count = hibernate_page_list_iterate((kWired & pageType) ? vars->page_list_wired : vars->page_list,
2086 &ppnum);
2087 if (count > UINT_MAX) {
2088 count = UINT_MAX;
2089 }
2090 }
2091 // kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
2092 iterDone = !count;
2093
2094 if (!cpuAES) {
2095 if (count && (kWired & pageType) && needEncrypt) {
2096 uint32_t checkIndex;
2097 for (checkIndex = 0;
2098 (checkIndex < count)
2099 && (((kEncrypt & pageType) == 0) == pmap_is_noencrypt(((ppnum_t)(ppnum + checkIndex))));
2100 checkIndex++) {
2101 }
2102 if (!checkIndex) {
2103 ppnum++;
2104 continue;
2105 }
2106 count = checkIndex;
2107 }
2108 }
2109
2110 switch (pageType) {
2111 case kWiredEncrypt: wiredPagesEncrypted += count; break;
2112 case kWiredClear: wiredPagesClear += count; break;
2113 case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
2114 }
2115
2116 if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */
2117 } else {
2118 pageAndCount[0] = (uint32_t) ppnum;
2119 pageAndCount[1] = (uint32_t) count;
2120 err = IOHibernatePolledFileWrite(vars,
2121 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
2122 cryptvars);
2123 if (kIOReturnSuccess != err) {
2124 break;
2125 }
2126 }
2127
2128 for (page = ppnum; page < (ppnum + count); page++) {
2129 #if HIBERNATE_HMAC_IMAGE
2130 wkresult = ppl_hmac_update_and_compress_page(page, (void **)&src, compressed);
2131 #else /* !HIBERNATE_HMAC_IMAGE */
2132 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
2133 if (err) {
2134 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
2135 break;
2136 }
2137
2138 sum = hibernate_sum_page(src, (uint32_t) page);
2139 if (kWired & pageType) {
2140 sum1 += sum;
2141 } else {
2142 sum2 += sum;
2143 }
2144
2145 clock_get_uptime(&startTime);
2146 wkresult = WKdm_compress_new((const WK_word*) src,
2147 (WK_word*) compressed,
2148 (WK_word*) scratch,
2149 (uint32_t) (page_size - 4));
2150 #endif /* !HIBERNATE_HMAC_IMAGE */
2151
2152 clock_get_uptime(&endTime);
2153 ADD_ABSOLUTETIME(&compTime, &endTime);
2154 SUB_ABSOLUTETIME(&compTime, &startTime);
2155
2156 compBytes += page_size;
2157 pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
2158
2159 if (pageCompressedSize == 0) {
2160 pageCompressedSize = 4;
2161 data = src;
2162
2163 if (*(uint32_t *)src) {
2164 svPageCount++;
2165 } else {
2166 zvPageCount++;
2167 }
2168 } else {
2169 if (pageCompressedSize != page_size) {
2170 data = compressed;
2171 } else {
2172 data = src;
2173 }
2174 }
2175
2176 assert(pageCompressedSize <= page_size);
2177 tag = ((uint32_t) pageCompressedSize) | kIOHibernateTagSignature;
2178 err = IOHibernatePolledFileWrite(vars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
2179 if (kIOReturnSuccess != err) {
2180 break;
2181 }
2182
2183 err = IOHibernatePolledFileWrite(vars, data, (pageCompressedSize + 3) & ~3, cryptvars);
2184 if (kIOReturnSuccess != err) {
2185 break;
2186 }
2187
2188 compressedSize += pageCompressedSize;
2189 uncompressedSize += page_size;
2190 pagesDone++;
2191
2192 if (vars->consoleMapping && (0 == (1023 & pagesDone))) {
2193 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
2194 if (blob != lastBlob) {
2195 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
2196 lastBlob = blob;
2197 }
2198 }
2199 if (0 == (8191 & pagesDone)) {
2200 clock_get_uptime(&endTime);
2201 SUB_ABSOLUTETIME(&endTime, &allTime);
2202 absolutetime_to_nanoseconds(endTime, &nsec);
2203 progressStamp = nsec / 750000000ULL;
2204 if (progressStamp != lastProgressStamp) {
2205 lastProgressStamp = progressStamp;
2206 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
2207 }
2208 }
2209 }
2210 if (kIOReturnSuccess != err) {
2211 break;
2212 }
2213 ppnum = page;
2214 }
2215
2216 if (kIOReturnSuccess != err) {
2217 break;
2218 }
2219
2220 if ((kEncrypt & pageType) && vars->fileVars->encryptStart) {
2221 vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
2222 HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
2223 }
2224
2225 if (kWiredEncrypt != pageType) {
2226 // end of image1/2 - fill to next block
2227 err = IOHibernatePolledFileWrite(vars, NULL, 0, cryptvars);
2228 if (kIOReturnSuccess != err) {
2229 break;
2230 }
2231 }
2232 if (kWiredClear == pageType) {
2233 // enlarge wired image for test
2234 // err = IOHibernatePolledFileWrite(vars, 0, 0x60000000, cryptvars);
2235
2236 // end wired image
2237 header->encryptStart = vars->fileVars->encryptStart;
2238 header->encryptEnd = vars->fileVars->encryptEnd;
2239 image1Size = vars->fileVars->position;
2240 HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
2241 image1Size, header->encryptStart, header->encryptEnd);
2242 #if HIBERNATE_HMAC_IMAGE
2243 // compute the image1 HMAC
2244 ppl_hmac_final(header->image1PagesHMAC, sizeof(header->image1PagesHMAC));
2245 // reset the PPL context so we can compute the image2 (non-wired pages) HMAC
2246 ppl_hmac_reset(false);
2247 #endif /* HIBERNATE_HMAC_IMAGE */
2248 }
2249 }
2250 if (kIOReturnSuccess != err) {
2251 if (kIOReturnOverrun == err) {
2252 // update actual compression ratio on not enough space (for retry)
2253 gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
2254 }
2255
2256 // update partial amount written (for IOPolledFileClose cleanup/unmap)
2257 header->imageSize = vars->fileVars->position;
2258 break;
2259 }
2260
2261 #if HIBERNATE_HMAC_IMAGE
2262 // compute the image2 HMAC
2263 ppl_hmac_final(header->image2PagesHMAC, sizeof(header->image2PagesHMAC));
2264 #endif /* HIBERNATE_HMAC_IMAGE */
2265
2266 // Header:
2267
2268 header->imageSize = vars->fileVars->position;
2269 header->image1Size = image1Size;
2270 header->bitmapSize = bitmap_size;
2271 header->pageCount = pageCount;
2272
2273 #if !HIBERNATE_HMAC_IMAGE
2274 header->restore1Sum = restore1Sum;
2275 header->image1Sum = sum1;
2276 header->image2Sum = sum2;
2277 #endif /* !HIBERNATE_HMAC_IMAGE */
2278 header->sleepTime = gIOLastSleepTime.tv_sec;
2279
2280 header->compression = ((uint32_t)((compressedSize << 8) / uncompressedSize));
2281 gIOHibernateCompression = header->compression;
2282
2283 count = vars->fileVars->fileExtents->getLength();
2284 if (count > sizeof(header->fileExtentMap)) {
2285 header->fileExtentMapSize = ((uint32_t) count);
2286 count = sizeof(header->fileExtentMap);
2287 } else {
2288 header->fileExtentMapSize = sizeof(header->fileExtentMap);
2289 }
2290 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
2291
2292 header->deviceBase = vars->fileVars->block0;
2293 header->deviceBlockSize = vars->fileVars->blockSize;
2294 header->lastHibAbsTime = mach_absolute_time();
2295 header->lastHibContTime = mach_continuous_time();
2296
2297 #if HIBERNATE_HMAC_IMAGE
2298 // include the headers in the SHA calculation
2299 SHA256_Update(&imageShaCtx, header, sizeof(*header));
2300
2301 // finalize the image header SHA
2302 uint8_t imageHash[CCSHA256_OUTPUT_SIZE];
2303 SHA256_Final(imageHash, &imageShaCtx);
2304
2305 // compute the header HMAC
2306 ppl_hmac_finalize_image(imageHash, sizeof(imageHash), header->imageHeaderHMAC, sizeof(header->imageHeaderHMAC));
2307 #endif /* HIBERNATE_HMAC_IMAGE */
2308
2309 IOPolledFileSeek(vars->fileVars, 0);
2310 err = IOHibernatePolledFileWrite(vars,
2311 (uint8_t *) header, sizeof(IOHibernateImageHeader),
2312 cryptvars);
2313 if (kIOReturnSuccess != err) {
2314 break;
2315 }
2316
2317 err = IOHibernatePolledFileWrite(vars, NULL, 0, cryptvars);
2318 }while (false);
2319
2320 clock_get_uptime(&endTime);
2321
2322 IOService::getPMRootDomain()->pmStatsRecordEvent(
2323 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
2324
2325 SUB_ABSOLUTETIME(&endTime, &allTime);
2326 absolutetime_to_nanoseconds(endTime, &nsec);
2327 HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
2328
2329 absolutetime_to_nanoseconds(compTime, &nsec);
2330 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2331 compBytes,
2332 nsec / 1000000ULL,
2333 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2334
2335 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2336 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
2337 vars->fileVars->cryptBytes,
2338 nsec / 1000000ULL,
2339 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2340
2341 HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%)\n",
2342 header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
2343 uncompressedSize, atop_32(uncompressedSize), compressedSize,
2344 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0);
2345
2346 #if !HIBERNATE_HMAC_IMAGE
2347 HIBLOG("\nsum1 %x, sum2 %x\n", sum1, sum2);
2348 #endif /* !HIBERNATE_HMAC_IMAGE */
2349
2350 HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
2351 svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
2352
2353 if (pollerOpen) {
2354 IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted );
2355 }
2356
2357 if (vars->consoleMapping) {
2358 ProgressUpdate(gIOHibernateGraphicsInfo,
2359 vars->consoleMapping, 0, kIOHibernateProgressCount);
2360 }
2361
2362 HIBLOG("hibernate_write_image done(%x)\n", err);
2363
2364 // should we come back via regular wake, set the state in memory.
2365 gIOHibernateState = kIOHibernateStateInactive;
2366
2367 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, wiredPagesEncrypted,
2368 wiredPagesClear, dirtyPagesEncrypted);
2369
2370 #if defined(__arm64__)
2371 if (kIOReturnSuccess == err) {
2372 return kIOHibernatePostWriteHalt;
2373 } else {
2374 // on ARM, once ApplePMGR decides we're hibernating, we can't turn back
2375 // see: <rdar://problem/63848862> Tonga ApplePMGR diff quiesce path support
2376 panic("hibernate_write_image encountered error 0x%x", err);
2377 }
2378 #else
2379 if (kIOReturnSuccess == err) {
2380 if (kIOHibernateModeSleep & gIOHibernateMode) {
2381 return kIOHibernatePostWriteSleep;
2382 } else if (kIOHibernateModeRestart & gIOHibernateMode) {
2383 return kIOHibernatePostWriteRestart;
2384 } else {
2385 /* by default, power down */
2386 return kIOHibernatePostWriteHalt;
2387 }
2388 } else if (kIOReturnAborted == err) {
2389 return kIOHibernatePostWriteWake;
2390 } else {
2391 /* on error, sleep */
2392 return kIOHibernatePostWriteSleep;
2393 }
2394 #endif
2395 }
2396
2397 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2398
2399 extern "C" void
2400 hibernate_machine_init(void)
2401 {
2402 IOReturn err;
2403 uint32_t sum;
2404 uint32_t pagesDone;
2405 uint32_t pagesRead = 0;
2406 AbsoluteTime startTime, compTime;
2407 AbsoluteTime allTime, endTime;
2408 AbsoluteTime startIOTime, endIOTime;
2409 uint64_t nsec, nsecIO;
2410 uint64_t compBytes;
2411 uint64_t lastProgressStamp = 0;
2412 uint64_t progressStamp;
2413 IOPolledFileCryptVars * cryptvars = NULL;
2414
2415 IOHibernateVars * vars = &gIOHibernateVars;
2416 bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
2417
2418 if (!vars->fileVars || !vars->fileVars->pollers) {
2419 return;
2420 }
2421
2422 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2423 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2424
2425 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) {
2426 HIBLOG("regular wake\n");
2427 return;
2428 }
2429
2430 HIBPRINT("diag %x %x %x %x\n",
2431 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
2432 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
2433
2434 #if defined(__i386__) || defined(__x86_64__)
2435 #define t40ms(x) ((uint32_t)((tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)))
2436 #else /* defined(__i386__) || defined(__x86_64__) */
2437 #define t40ms(x) x
2438 #endif /* defined(__i386__) || defined(__x86_64__) */
2439 #define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
2440 tStat(booterStart, booterStart);
2441 gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart;
2442 tStat(booterDuration0, booterTime0);
2443 tStat(booterDuration1, booterTime1);
2444 tStat(booterDuration2, booterTime2);
2445 tStat(booterDuration, booterTime);
2446 tStat(booterConnectDisplayDuration, connectDisplayTime);
2447 tStat(booterSplashDuration, splashTime);
2448 tStat(trampolineDuration, trampolineTime);
2449
2450 gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size;
2451 gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize;
2452 gIOHibernateStats->image1Pages = pagesDone;
2453
2454 /* HIBERNATE_stats */
2455 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 14), gIOHibernateStats->smcStart,
2456 gIOHibernateStats->booterStart, gIOHibernateStats->booterDuration,
2457 gIOHibernateStats->trampolineDuration);
2458
2459 HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
2460 gIOHibernateStats->booterStart,
2461 gIOHibernateStats->smcStart,
2462 gIOHibernateStats->booterDuration0,
2463 gIOHibernateStats->booterDuration1,
2464 gIOHibernateStats->booterDuration2,
2465 gIOHibernateStats->booterDuration,
2466 gIOHibernateStats->booterConnectDisplayDuration,
2467 gIOHibernateStats->booterSplashDuration,
2468 gIOHibernateStats->trampolineDuration);
2469
2470 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
2471 gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
2472 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
2473
2474 if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
2475 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))) {
2476 hibernate_page_list_discard(vars->page_list);
2477 }
2478
2479 if (vars->hwEncrypt) {
2480 // if vars->hwEncrypt is true, we don't need cryptvars since we supply the
2481 // decryption key via IOPolledFilePollersSetEncryptionKey
2482 cryptvars = NULL;
2483 } else {
2484 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : NULL;
2485 }
2486
2487 if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount) {
2488 panic("handoff overflow");
2489 }
2490
2491 IOHibernateHandoff * handoff;
2492 bool done = false;
2493 bool foundCryptData = false;
2494 bool foundVolumeEncryptData = false;
2495 const uint8_t * handoffStart = (const uint8_t*)vars->handoffBuffer->getBytesNoCopy();
2496 const uint8_t * handoffEnd = handoffStart + vars->handoffBuffer->getLength();
2497
2498 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
2499 !done;
2500 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount]) {
2501 if (((uint8_t*)handoff < handoffStart) ||
2502 (&handoff->data[handoff->bytecount] > handoffEnd)) {
2503 panic("handoff out of range");
2504 }
2505 // HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
2506 uint8_t * data = &handoff->data[0];
2507 switch (handoff->type) {
2508 case kIOHibernateHandoffTypeEnd:
2509 done = true;
2510 break;
2511
2512 case kIOHibernateHandoffTypeGraphicsInfo:
2513 if (handoff->bytecount == sizeof(*gIOHibernateGraphicsInfo)) {
2514 bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
2515 }
2516 break;
2517
2518 case kIOHibernateHandoffTypeCryptVars:
2519 if (cryptvars) {
2520 hibernate_cryptwakevars_t *
2521 wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
2522 if (handoff->bytecount == sizeof(*wakevars)) {
2523 bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
2524 } else {
2525 panic("kIOHibernateHandoffTypeCryptVars(%d)", handoff->bytecount);
2526 }
2527 }
2528 foundCryptData = true;
2529 bzero(data, handoff->bytecount);
2530 break;
2531
2532 case kIOHibernateHandoffTypeVolumeCryptKey:
2533 if (handoff->bytecount == vars->volumeCryptKeySize) {
2534 bcopy(data, &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
2535 foundVolumeEncryptData = true;
2536 } else {
2537 panic("kIOHibernateHandoffTypeVolumeCryptKey(%d)", handoff->bytecount);
2538 }
2539 break;
2540
2541 #if defined(__i386__) || defined(__x86_64__)
2542 case kIOHibernateHandoffTypeMemoryMap:
2543
2544 clock_get_uptime(&allTime);
2545
2546 hibernate_newruntime_map(data, handoff->bytecount,
2547 gIOHibernateCurrentHeader->systemTableOffset);
2548
2549 clock_get_uptime(&endTime);
2550
2551 SUB_ABSOLUTETIME(&endTime, &allTime);
2552 absolutetime_to_nanoseconds(endTime, &nsec);
2553
2554 HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
2555
2556 break;
2557
2558 case kIOHibernateHandoffTypeDeviceTree:
2559 {
2560 // DTEntry chosen = NULL;
2561 // HIBPRINT("SecureDTLookupEntry %d\n", SecureDTLookupEntry((const DTEntry) data, "/chosen", &chosen));
2562 }
2563 break;
2564 #endif /* defined(__i386__) || defined(__x86_64__) */
2565
2566 default:
2567 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
2568 break;
2569 }
2570 }
2571
2572 if (vars->hwEncrypt && !foundVolumeEncryptData) {
2573 panic("no volumeCryptKey");
2574 } else if (cryptvars && !foundCryptData) {
2575 panic("hibernate handoff");
2576 }
2577
2578 HIBPRINT("video 0x%llx %d %d %d status %x\n",
2579 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2580 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
2581
2582 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress) {
2583 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2584 * gIOHibernateGraphicsInfo->rowBytes);
2585 if (vars->videoMapSize > vars->videoAllocSize) {
2586 vars->videoMapSize = 0;
2587 } else {
2588 IOMapPages(kernel_map,
2589 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2590 vars->videoMapSize, kIOMapInhibitCache );
2591 }
2592 }
2593
2594 if (vars->videoMapSize) {
2595 ProgressUpdate(gIOHibernateGraphicsInfo,
2596 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2597 }
2598
2599 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2600 uint8_t * compressed = src + page_size;
2601 uint8_t * scratch = compressed + page_size;
2602 uint32_t decoOffset;
2603
2604 clock_get_uptime(&allTime);
2605 AbsoluteTime_to_scalar(&compTime) = 0;
2606 compBytes = 0;
2607
2608 HIBLOG("IOPolledFilePollersOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2609 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledAfterSleepState, false);
2610 clock_get_uptime(&startIOTime);
2611 endTime = startIOTime;
2612 SUB_ABSOLUTETIME(&endTime, &allTime);
2613 absolutetime_to_nanoseconds(endTime, &nsec);
2614 HIBLOG("IOPolledFilePollersOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
2615
2616 if (vars->hwEncrypt) {
2617 err = IOPolledFilePollersSetEncryptionKey(vars->fileVars,
2618 &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
2619 HIBLOG("IOPolledFilePollersSetEncryptionKey(%x) %ld\n", err, vars->volumeCryptKeySize);
2620 if (kIOReturnSuccess != err) {
2621 panic("IOPolledFilePollersSetEncryptionKey(0x%x)", err);
2622 }
2623 cryptvars = NULL;
2624 }
2625
2626 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2627
2628 // kick off the read ahead
2629 vars->fileVars->bufferHalf = 0;
2630 vars->fileVars->bufferLimit = 0;
2631 vars->fileVars->lastRead = 0;
2632 vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
2633 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2634 vars->fileVars->cryptBytes = 0;
2635 AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
2636
2637 err = IOPolledFileRead(vars->fileVars, NULL, 0, cryptvars);
2638 if (kIOReturnSuccess != err) {
2639 panic("Hibernate restore error %x", err);
2640 }
2641 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2642 // --
2643
2644 HIBLOG("hibernate_machine_init reading\n");
2645
2646 #if HIBERNATE_HMAC_IMAGE
2647 // Reset SHA context to verify image2 hash (non-wired pages).
2648 ppl_hmac_reset(false);
2649 #endif /* HIBERNATE_HMAC_IMAGE */
2650
2651 uint32_t * header = (uint32_t *) src;
2652 sum = 0;
2653
2654 while (kIOReturnSuccess == err) {
2655 unsigned int count;
2656 unsigned int page;
2657 uint32_t tag;
2658 vm_offset_t compressedSize;
2659 ppnum_t ppnum;
2660
2661 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2662 if (kIOReturnSuccess != err) {
2663 panic("Hibernate restore error %x", err);
2664 }
2665
2666 ppnum = header[0];
2667 count = header[1];
2668
2669 // HIBPRINT("(%x, %x)\n", ppnum, count);
2670
2671 if (!count) {
2672 break;
2673 }
2674
2675 for (page = 0; page < count; page++) {
2676 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2677 if (kIOReturnSuccess != err) {
2678 panic("Hibernate restore error %x", err);
2679 }
2680
2681 compressedSize = kIOHibernateTagLength & tag;
2682 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength)) {
2683 err = kIOReturnIPCError;
2684 panic("Hibernate restore error %x", err);
2685 }
2686
2687 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2688 if (kIOReturnSuccess != err) {
2689 panic("Hibernate restore error %x", err);
2690 }
2691
2692 if (compressedSize < page_size) {
2693 decoOffset = ((uint32_t) page_size);
2694 clock_get_uptime(&startTime);
2695
2696 if (compressedSize == 4) {
2697 int i;
2698 uint32_t *s, *d;
2699
2700 s = (uint32_t *)src;
2701 d = (uint32_t *)(uintptr_t)compressed;
2702
2703 for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++) {
2704 *d++ = *s;
2705 }
2706 } else {
2707 pal_hib_decompress_page(src, compressed, scratch, ((unsigned int) compressedSize));
2708 }
2709 clock_get_uptime(&endTime);
2710 ADD_ABSOLUTETIME(&compTime, &endTime);
2711 SUB_ABSOLUTETIME(&compTime, &startTime);
2712 compBytes += page_size;
2713 } else {
2714 decoOffset = 0;
2715 }
2716
2717 sum += hibernate_sum_page((src + decoOffset), ((uint32_t) ppnum));
2718 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2719 if (err) {
2720 HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
2721 panic("Hibernate restore error %x", err);
2722 }
2723
2724 #if HIBERNATE_HMAC_IMAGE
2725 err = ppl_hmac_update_and_compress_page(ppnum, NULL, NULL);
2726 if (err) {
2727 panic("Hibernate restore error %x", err);
2728 }
2729 #endif /* HIBERNATE_HMAC_IMAGE */
2730
2731 ppnum++;
2732 pagesDone++;
2733 pagesRead++;
2734
2735 if (0 == (8191 & pagesDone)) {
2736 clock_get_uptime(&endTime);
2737 SUB_ABSOLUTETIME(&endTime, &allTime);
2738 absolutetime_to_nanoseconds(endTime, &nsec);
2739 progressStamp = nsec / 750000000ULL;
2740 if (progressStamp != lastProgressStamp) {
2741 lastProgressStamp = progressStamp;
2742 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2743 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2744 }
2745 }
2746 }
2747 }
2748 if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages)) {
2749 err = kIOReturnLockedRead;
2750 }
2751
2752 if (kIOReturnSuccess != err) {
2753 panic("Hibernate restore error %x", err);
2754 }
2755
2756 #if HIBERNATE_HMAC_IMAGE
2757 uint8_t image2PagesHMAC[HIBERNATE_HMAC_SIZE];
2758 ppl_hmac_final(image2PagesHMAC, sizeof(image2PagesHMAC));
2759 if (memcmp(image2PagesHMAC, gIOHibernateCurrentHeader->image2PagesHMAC, sizeof(image2PagesHMAC)) != 0) {
2760 panic("image2 pages corrupted");
2761 }
2762 #endif /* HIBERNATE_HMAC_IMAGE */
2763
2764 gIOHibernateCurrentHeader->actualImage2Sum = sum;
2765 gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
2766
2767 clock_get_uptime(&endIOTime);
2768
2769 err = IOPolledFilePollersClose(vars->fileVars, kIOPolledAfterSleepState);
2770
2771 clock_get_uptime(&endTime);
2772
2773 IOService::getPMRootDomain()->pmStatsRecordEvent(
2774 kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
2775 IOService::getPMRootDomain()->pmStatsRecordEvent(
2776 kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
2777
2778 SUB_ABSOLUTETIME(&endTime, &allTime);
2779 absolutetime_to_nanoseconds(endTime, &nsec);
2780
2781 SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
2782 absolutetime_to_nanoseconds(endIOTime, &nsecIO);
2783
2784 gIOHibernateStats->kernelImageReadDuration = ((uint32_t) (nsec / 1000000ULL));
2785 gIOHibernateStats->imagePages = pagesDone;
2786
2787 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
2788 pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
2789 nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
2790
2791 absolutetime_to_nanoseconds(compTime, &nsec);
2792 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2793 compBytes,
2794 nsec / 1000000ULL,
2795 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2796
2797 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2798 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
2799 vars->fileVars->cryptBytes,
2800 nsec / 1000000ULL,
2801 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2802
2803 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 2), pagesRead, pagesDone);
2804 }
2805
2806 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2807
2808 void
2809 IOHibernateSetWakeCapabilities(uint32_t capability)
2810 {
2811 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
2812 gIOHibernateStats->wakeCapability = capability;
2813
2814 if (kIOPMSystemCapabilityGraphics & capability) {
2815 vm_compressor_do_warmup();
2816 }
2817 }
2818 }
2819
2820 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2821
2822 void
2823 IOHibernateSystemRestart(void)
2824 {
2825 static uint8_t noteStore[32] __attribute__((aligned(32)));
2826 IORegistryEntry * regEntry;
2827 const OSSymbol * sym;
2828 OSData * noteProp;
2829 OSData * data;
2830 uintptr_t * smcVars;
2831 uint8_t * smcBytes;
2832 size_t len;
2833 addr64_t element;
2834
2835 data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
2836 if (!data) {
2837 return;
2838 }
2839
2840 smcVars = (typeof(smcVars))data->getBytesNoCopy();
2841 smcBytes = (typeof(smcBytes))smcVars[1];
2842 len = smcVars[0];
2843 if (len > sizeof(noteStore)) {
2844 len = sizeof(noteStore);
2845 }
2846 noteProp = OSData::withCapacity(3 * sizeof(element));
2847 if (!noteProp) {
2848 return;
2849 }
2850 element = len;
2851 noteProp->appendBytes(&element, sizeof(element));
2852 element = crc32(0, smcBytes, len);
2853 noteProp->appendBytes(&element, sizeof(element));
2854
2855 bcopy(smcBytes, noteStore, len);
2856 element = (addr64_t) &noteStore[0];
2857 element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
2858 noteProp->appendBytes(&element, sizeof(element));
2859
2860 if (!gIOOptionsEntry) {
2861 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
2862 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
2863 if (regEntry && !gIOOptionsEntry) {
2864 regEntry->release();
2865 }
2866 }
2867
2868 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
2869 if (gIOOptionsEntry && sym) {
2870 gIOOptionsEntry->setProperty(sym, noteProp);
2871 }
2872 if (noteProp) {
2873 noteProp->release();
2874 }
2875 if (sym) {
2876 sym->release();
2877 }
2878 }
mirror of XNU