]> git.saurik.com Git - apple/xnu.git/blob - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blob
? search:


6db950f67b8ed212d12ee12048639ca5ca1df8ea
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
1 /*
2 * Copyright (c) 2004-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30
31 Sleep:
32
33 - PMRootDomain calls IOHibernateSystemSleep() before system sleep
34 (devices awake, normal execution context)
35 - IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
36 grabs its extents and searches for a polling driver willing to work with that IOMedia.
37 The BSD code makes an ioctl to the storage driver to get the partition base offset to
38 the disk, and other ioctls to get the transfer constraints
39 If successful, the file is written to make sure its initially not bootable (in case of
40 later failure) and nvram set to point to the first block of the file. (Has to be done
41 here so blocking is possible in nvram support).
42 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
43 page out any pages it wants to (currently zero, but probably some percentage of memory).
44 Its assumed just allocating pages will cause the VM system to naturally select the best
45 pages for eviction. It also copies processor flags needed for the restore path and sets
46 a flag in the boot processor proc info.
47 gIOHibernateState = kIOHibernateStateHibernating.
48 - Regular sleep progresses - some drivers may inspect the root domain property
49 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
50 as usual but leaves motherboard I/O on.
51 - Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
52 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
53 all ppc RC bits out of the hash table and caches into the mapping structures.
54 - hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
55 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
56 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
57 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
58 The image header and block list are written. The header includes the second file extent so
59 only the header block is needed to read the file, regardless of filesystem.
60 The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
61 (only) is used to decompress the image during wake/boot.
62 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
63 The bitmaps are written to the image.
64 More areas are removed from the bitmaps (after they have been written to the image) - the
65 segment "__HIB" pages and interrupt stack.
66 Each wired page is compressed and written and then each non-wired page. Compression and
67 disk writes are in parallel.
68 The image header is written to the start of the file and the polling driver closed.
69 The machine powers down (or sleeps).
70
71 Boot/Wake:
72
73 - BootX sees the boot-image nvram variable containing the device and block number of the image,
74 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
75 - BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
76 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
77 that is in the kernel's __HIB section.
78 - BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
79 hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
80 only code & data in that section is safe to call since all the other wired pages are still
81 compressed in the image.
82 - hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
83 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
84 location directly, and copies those that can't to interim free pages. When the image has been
85 completed, the copies are uncompressed, overwriting the wired image pages.
86 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
87 is used to get pages into place for 64bit.
88 - the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
89 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
90 are removed from the software strutures, and the hash table is reinitialized.
91 - After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
92 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
93 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
94 for the remaining non wired pages.
95 - The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
96 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
97 is closed via bsd.
98
99 Polled Mode I/O:
100
101 IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
102 registry, specifying an object of calls IOPolledInterface.
103
104 Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
105 partition) that the image is going to live, looking for polled interface properties. If it finds
106 one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
107 interfaces found are kept in an ordered list.
108
109 There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
110 few different contexts things happen in:
111
112 - there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
113 up and running) and after wake - this is safe to allocate memory and do anything. The device
114 ignores sleep requests from that point since its a waste of time if it goes to sleep and
115 immediately wakes back up for the image write.
116
117 - there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
118 immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
119 from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
120 used to flush and set the disk to sleep.
121
122 - there is an Open/Close (AfterSleep) pair made around the image read operations that happen
123 immediately after sleep. These can't block or allocate memory. This is happening after the platform
124 expert has woken the low level bits of the system, but most of the I/O system has not. There is only
125 one thread running.
126
127 For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
128 (prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
129 the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
130 Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
131 that is called for the hardware to check for events, and complete the I/O via the callback.
132 The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
133 to restrict I/O ops.
134 */
135
136 #include <sys/systm.h>
137
138 #include <IOKit/IOWorkLoop.h>
139 #include <IOKit/IOCommandGate.h>
140 #include <IOKit/IOTimerEventSource.h>
141 #include <IOKit/IOPlatformExpert.h>
142 #include <IOKit/IOKitDebug.h>
143 #include <IOKit/IOTimeStamp.h>
144 #include <IOKit/pwr_mgt/RootDomain.h>
145 #include <IOKit/pwr_mgt/IOPMPrivate.h>
146 #include <IOKit/IOMessage.h>
147 #include <IOKit/IODeviceTreeSupport.h>
148 #include <IOKit/IOBSD.h>
149 #include <IOKit/IOKitKeysPrivate.h>
150 #include "RootDomainUserClient.h"
151 #include <IOKit/pwr_mgt/IOPowerConnection.h>
152 #include "IOPMPowerStateQueue.h"
153 #include <IOKit/IOBufferMemoryDescriptor.h>
154 #include <IOKit/AppleKeyStoreInterface.h>
155 #include <libkern/crypto/aes.h>
156
157 #include <sys/uio.h>
158 #include <sys/conf.h>
159 #include <sys/stat.h>
160 #include <sys/fcntl.h> // (FWRITE, ...)
161 #include <sys/sysctl.h>
162 #include <sys/kdebug.h>
163 #include <stdint.h>
164
165 #include <IOKit/IOHibernatePrivate.h>
166 #include <IOKit/IOPolledInterface.h>
167 #include <IOKit/IONVRAM.h>
168 #include "IOHibernateInternal.h"
169 #include <vm/WKdm_new.h>
170 #include <vm/vm_protos.h>
171 #include "IOKitKernelInternal.h"
172 #include <pexpert/device_tree.h>
173
174 #include <machine/pal_routines.h>
175 #include <machine/pal_hibernate.h>
176 #include <i386/tsc.h>
177 #include <i386/cpuid.h>
178 #include <san/kasan.h>
179
180 extern "C" addr64_t kvtophys(vm_offset_t va);
181 extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
182
183 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
184
185 #define DISABLE_TRIM 0
186 #define TRIM_DELAY 25000
187
188 extern unsigned int save_kdebug_enable;
189 extern uint32_t gIOHibernateState;
190 uint32_t gIOHibernateMode;
191 static char gIOHibernateBootSignature[256+1];
192 static char gIOHibernateFilename[MAXPATHLEN+1];
193
194 static uuid_string_t gIOHibernateBridgeBootSessionUUIDString;
195
196 static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
197 uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
198 static uint64_t gIOHibernateCompression = 0x80; // default compression 50%
199 boolean_t gIOHibernateStandbyDisabled;
200
201 static IODTNVRAM * gIOOptionsEntry;
202 static IORegistryEntry * gIOChosenEntry;
203
204 static const OSSymbol * gIOHibernateBootImageKey;
205 static const OSSymbol * gIOHibernateBootSignatureKey;
206 static const OSSymbol * gIOBridgeBootSessionUUIDKey;
207
208 #if defined(__i386__) || defined(__x86_64__)
209
210 static const OSSymbol * gIOHibernateRTCVariablesKey;
211 static const OSSymbol * gIOHibernateBoot0082Key;
212 static const OSSymbol * gIOHibernateBootNextKey;
213 static OSData * gIOHibernateBoot0082Data;
214 static OSData * gIOHibernateBootNextData;
215 static OSObject * gIOHibernateBootNextSave;
216
217 static IOPolledFileIOVars * gDebugImageFileVars;
218 static IOLock * gDebugImageLock;
219
220 #endif /* defined(__i386__) || defined(__x86_64__) */
221
222 static IOLock * gFSLock;
223 uint32_t gFSState;
224 static thread_call_t gIOHibernateTrimCalloutEntry;
225 static IOPolledFileIOVars gFileVars;
226 static IOHibernateVars gIOHibernateVars;
227 static IOPolledFileCryptVars gIOHibernateCryptWakeContext;
228 static hibernate_graphics_t _hibernateGraphics;
229 static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
230 static hibernate_statistics_t _hibernateStats;
231 static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats;
232
233 enum
234 {
235 kFSIdle = 0,
236 kFSOpening = 2,
237 kFSOpened = 3,
238 kFSTimedOut = 4,
239 kFSTrimDelay = 5
240 };
241
242 static IOReturn IOHibernateDone(IOHibernateVars * vars);
243 static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature);
244 static void IOSetBootImageNVRAM(OSData * data);
245 static void IOHibernateSystemPostWakeTrim(void * p1, void * p2);
246
247 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
248
249 enum { kDefaultIOSize = 128 * 1024 };
250 enum { kVideoMapSize = 80 * 1024 * 1024 };
251
252 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
253
254 // copy from phys addr to MD
255
256 static IOReturn
257 IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
258 IOByteCount offset, addr64_t bytes, IOByteCount length)
259 {
260 addr64_t srcAddr = bytes;
261 IOByteCount remaining;
262
263 remaining = length = min(length, md->getLength() - offset);
264 while (remaining) { // (process another target segment?)
265 addr64_t dstAddr64;
266 IOByteCount dstLen;
267
268 dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
269 if (!dstAddr64)
270 break;
271
272 // Clip segment length to remaining
273 if (dstLen > remaining)
274 dstLen = remaining;
275
276 #if 1
277 bcopy_phys(srcAddr, dstAddr64, dstLen);
278 #else
279 copypv(srcAddr, dstAddr64, dstLen,
280 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
281 #endif
282 srcAddr += dstLen;
283 offset += dstLen;
284 remaining -= dstLen;
285 }
286
287 assert(!remaining);
288
289 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
290 }
291
292 // copy from MD to phys addr
293
294 static IOReturn
295 IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
296 IOByteCount offset, addr64_t bytes, IOByteCount length)
297 {
298 addr64_t dstAddr = bytes;
299 IOByteCount remaining;
300
301 remaining = length = min(length, md->getLength() - offset);
302 while (remaining) { // (process another target segment?)
303 addr64_t srcAddr64;
304 IOByteCount dstLen;
305
306 srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
307 if (!srcAddr64)
308 break;
309
310 // Clip segment length to remaining
311 if (dstLen > remaining)
312 dstLen = remaining;
313
314 #if 1
315 bcopy_phys(srcAddr64, dstAddr, dstLen);
316 #else
317 copypv(srcAddr, dstAddr64, dstLen,
318 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
319 #endif
320 dstAddr += dstLen;
321 offset += dstLen;
322 remaining -= dstLen;
323 }
324
325 assert(!remaining);
326
327 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
328 }
329
330 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
331
332 void
333 hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
334 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
335 {
336 count += ppnum;
337 switch (kind)
338 {
339 case kIOHibernatePageStateUnwiredSave:
340 // unwired save
341 for (; ppnum < count; ppnum++)
342 {
343 hibernate_page_bitset(page_list, FALSE, ppnum);
344 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
345 }
346 break;
347 case kIOHibernatePageStateWiredSave:
348 // wired save
349 for (; ppnum < count; ppnum++)
350 {
351 hibernate_page_bitset(page_list, FALSE, ppnum);
352 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
353 }
354 break;
355 case kIOHibernatePageStateFree:
356 // free page
357 for (; ppnum < count; ppnum++)
358 {
359 hibernate_page_bitset(page_list, TRUE, ppnum);
360 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
361 }
362 break;
363 default:
364 panic("hibernate_set_page_state");
365 }
366 }
367
368 static vm_offset_t
369 hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
370 {
371 uint32_t page = *pPage;
372 uint32_t count;
373 hibernate_bitmap_t * bitmap;
374
375 while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
376 {
377 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
378 if (!count)
379 break;
380 page += count;
381 if (page <= bitmap->last_page)
382 break;
383 }
384
385 *pPage = page;
386 if (bitmap)
387 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
388 else
389 count = 0;
390
391 return (count);
392 }
393
394 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
395
396 IOReturn
397 IOHibernateSystemSleep(void)
398 {
399 IOReturn err;
400 OSData * nvramData;
401 OSObject * obj;
402 OSString * str;
403 OSNumber * num;
404 bool dsSSD, vmflush, swapPinned;
405 IOHibernateVars * vars;
406 uint64_t setFileSize = 0;
407
408 gIOHibernateState = kIOHibernateStateInactive;
409
410 gIOHibernateDebugFlags = 0;
411 if (kIOLogHibernate & gIOKitDebug)
412 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
413
414 if (IOService::getPMRootDomain()->getHibernateSettings(
415 &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime))
416 {
417 if (kIOHibernateModeSleep & gIOHibernateMode)
418 // default to discard clean for safe sleep
419 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
420 | kIOHibernateModeDiscardCleanActive);
421 }
422
423 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
424 {
425 if ((str = OSDynamicCast(OSString, obj)))
426 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
427 sizeof(gIOHibernateFilename));
428 obj->release();
429 }
430
431 if (!gIOHibernateMode || !gIOHibernateFilename[0])
432 return (kIOReturnUnsupported);
433
434 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
435
436 vars = IONew(IOHibernateVars, 1);
437 if (!vars) return (kIOReturnNoMemory);
438 bzero(vars, sizeof(*vars));
439
440 IOLockLock(gFSLock);
441 if (!gIOHibernateTrimCalloutEntry)
442 {
443 gIOHibernateTrimCalloutEntry = thread_call_allocate(&IOHibernateSystemPostWakeTrim, &gFSLock);
444 }
445 IOHibernateSystemPostWakeTrim(NULL, NULL);
446 thread_call_cancel(gIOHibernateTrimCalloutEntry);
447 if (kFSIdle != gFSState)
448 {
449 HIBLOG("hibernate file busy\n");
450 IOLockUnlock(gFSLock);
451 IODelete(vars, IOHibernateVars, 1);
452 return (kIOReturnBusy);
453 }
454 gFSState = kFSOpening;
455 IOLockUnlock(gFSLock);
456
457 swapPinned = false;
458 do
459 {
460 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
461 2 * page_size + WKdm_SCRATCH_BUF_SIZE_INTERNAL, page_size);
462
463 vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
464 ptoa_64(gIOHibernateHandoffPageCount), page_size);
465
466 if (!vars->srcBuffer || !vars->handoffBuffer)
467 {
468 err = kIOReturnNoMemory;
469 break;
470 }
471
472 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey)))
473 {
474 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMinSize = num->unsigned64BitValue();
475 obj->release();
476 }
477 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey)))
478 {
479 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMaxSize = num->unsigned64BitValue();
480 obj->release();
481 }
482
483 boolean_t encryptedswap = true;
484 uint32_t pageCount;
485 AbsoluteTime startTime, endTime;
486 uint64_t nsec;
487
488 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
489 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
490 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
491
492 vmflush = ((kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
493 err = hibernate_alloc_page_lists(&vars->page_list,
494 &vars->page_list_wired,
495 &vars->page_list_pal);
496 if (KERN_SUCCESS != err) break;
497
498 err = hibernate_pin_swap(TRUE);
499 if (KERN_SUCCESS != err) break;
500 swapPinned = true;
501
502 if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode))
503 {
504 hibernate_page_list_setall(vars->page_list,
505 vars->page_list_wired,
506 vars->page_list_pal,
507 true /* preflight */,
508 vmflush /* discard */,
509 &pageCount);
510 PE_Video consoleInfo;
511 bzero(&consoleInfo, sizeof(consoleInfo));
512 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
513
514 // estimate: 6% increase in pages compressed
515 // screen preview 2 images compressed 0%
516 setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
517 + vars->page_list->list_size
518 + (consoleInfo.v_width * consoleInfo.v_height * 8);
519 enum { setFileRound = 1024*1024ULL };
520 setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
521
522 HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
523 pageCount, (100ULL * gIOHibernateCompression) >> 8,
524 setFileSize, vars->fileMinSize);
525
526 if (!(kIOHibernateModeFileResize & gIOHibernateMode)
527 && (setFileSize < vars->fileMinSize))
528 {
529 setFileSize = vars->fileMinSize;
530 }
531 }
532
533 // Invalidate the image file
534 if (gDebugImageLock) {
535 IOLockLock(gDebugImageLock);
536 if (gDebugImageFileVars != 0) {
537 IOSetBootImageNVRAM(0);
538 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
539 }
540 IOLockUnlock(gDebugImageLock);
541 }
542
543 vars->volumeCryptKeySize = sizeof(vars->volumeCryptKey);
544 err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, 0,
545 gIOHibernateCurrentHeader, sizeof(gIOHibernateCurrentHeader),
546 &vars->fileVars, &nvramData,
547 &vars->volumeCryptKey[0], &vars->volumeCryptKeySize);
548
549 if (KERN_SUCCESS != err)
550 {
551 IOLockLock(gFSLock);
552 if (kFSOpening != gFSState) err = kIOReturnTimeout;
553 IOLockUnlock(gFSLock);
554 }
555
556 if (KERN_SUCCESS != err)
557 {
558 HIBLOG("IOPolledFileOpen(%x)\n", err);
559 break;
560 }
561
562 // write extents for debug data usage in EFI
563 IOWriteExtentsToFile(vars->fileVars, kIOHibernateHeaderOpenSignature);
564
565 err = IOPolledFilePollersSetup(vars->fileVars, kIOPolledPreflightState);
566 if (KERN_SUCCESS != err) break;
567
568 clock_get_uptime(&startTime);
569 err = hibernate_setup(gIOHibernateCurrentHeader,
570 vmflush,
571 vars->page_list, vars->page_list_wired, vars->page_list_pal);
572 clock_get_uptime(&endTime);
573 SUB_ABSOLUTETIME(&endTime, &startTime);
574 absolutetime_to_nanoseconds(endTime, &nsec);
575
576 boolean_t haveSwapPin, hibFileSSD;
577 haveSwapPin = vm_swap_files_pinned();
578
579 hibFileSSD = (kIOPolledFileSSD & vars->fileVars->flags);
580
581 HIBLOG("hibernate_setup(%d) took %qd ms, swapPin(%d) ssd(%d)\n",
582 err, nsec / 1000000ULL,
583 haveSwapPin, hibFileSSD);
584 if (KERN_SUCCESS != err) break;
585
586 gIOHibernateStandbyDisabled = ((!haveSwapPin || !hibFileSSD));
587
588 dsSSD = ((0 != (kIOPolledFileSSD & vars->fileVars->flags))
589 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
590
591 if (dsSSD) gIOHibernateCurrentHeader->options |= kIOHibernateOptionSSD | kIOHibernateOptionColor;
592 else gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
593
594
595 #if defined(__i386__) || defined(__x86_64__)
596 if (vars->volumeCryptKeySize &&
597 (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey)))
598 {
599 uintptr_t smcVars[2];
600 smcVars[0] = vars->volumeCryptKeySize;
601 smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
602
603 IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
604 bzero(smcVars, sizeof(smcVars));
605 }
606 #endif
607
608
609 if (encryptedswap || vars->volumeCryptKeySize)
610 gIOHibernateMode ^= kIOHibernateModeEncrypt;
611
612 if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
613 {
614 vars->videoAllocSize = kVideoMapSize;
615 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize, VM_KERN_MEMORY_IOKIT))
616 vars->videoMapping = 0;
617 }
618
619 // generate crypt keys
620 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
621 vars->wiredCryptKey[i] = random();
622 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
623 vars->cryptKey[i] = random();
624
625 // set nvram
626
627 IOSetBootImageNVRAM(nvramData);
628 nvramData->release();
629
630 #if defined(__i386__) || defined(__x86_64__)
631 {
632 struct AppleRTCHibernateVars
633 {
634 uint8_t signature[4];
635 uint32_t revision;
636 uint8_t booterSignature[20];
637 uint8_t wiredCryptKey[16];
638 };
639 AppleRTCHibernateVars rtcVars;
640 OSData * data;
641
642 rtcVars.signature[0] = 'A';
643 rtcVars.signature[1] = 'A';
644 rtcVars.signature[2] = 'P';
645 rtcVars.signature[3] = 'L';
646 rtcVars.revision = 1;
647 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
648
649 if (gIOChosenEntry
650 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOHibernateBootSignatureKey)))
651 && (sizeof(rtcVars.booterSignature) <= data->getLength()))
652 {
653 bcopy(data->getBytesNoCopy(), &rtcVars.booterSignature[0], sizeof(rtcVars.booterSignature));
654 }
655 else if (gIOHibernateBootSignature[0])
656 {
657 char c;
658 uint8_t value = 0;
659 uint32_t in, out, digits;
660 for (in = out = digits = 0;
661 (c = gIOHibernateBootSignature[in]) && (in < sizeof(gIOHibernateBootSignature));
662 in++)
663 {
664 if ((c >= 'a') && (c <= 'f')) c -= 'a' - 10;
665 else if ((c >= 'A') && (c <= 'F')) c -= 'A' - 10;
666 else if ((c >= '0') && (c <= '9')) c -= '0';
667 else
668 {
669 if (c == '=') out = digits = value = 0;
670 continue;
671 }
672 value = (value << 4) | c;
673 if (digits & 1)
674 {
675 rtcVars.booterSignature[out++] = value;
676 if (out >= sizeof(rtcVars.booterSignature)) break;
677 }
678 digits++;
679 }
680 }
681 #if DEBUG || DEVELOPMENT
682 if (kIOLogHibernate & gIOKitDebug) IOKitKernelLogBuffer("H> rtc:",
683 &rtcVars, sizeof(rtcVars), &kprintf);
684 #endif /* DEBUG || DEVELOPMENT */
685
686 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
687 if (data)
688 {
689 if (gIOHibernateRTCVariablesKey)
690 IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
691 data->release();
692 }
693 if (gIOChosenEntry && gIOOptionsEntry)
694 {
695 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
696 if (data) gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
697 // set BootNext
698 if (!gIOHibernateBoot0082Data)
699 {
700 OSData * fileData = 0;
701 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
702 if (data && data->getLength() >= 4) fileData = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-file-path"));
703 if (data)
704 {
705 // AppleNVRAM_EFI_LOAD_OPTION
706 struct {
707 uint32_t Attributes;
708 uint16_t FilePathLength;
709 uint16_t Desc;
710 } loadOptionHeader;
711 loadOptionHeader.Attributes = 1;
712 loadOptionHeader.FilePathLength = data->getLength();
713 loadOptionHeader.Desc = 0;
714 if (fileData)
715 {
716 loadOptionHeader.FilePathLength -= 4;
717 loadOptionHeader.FilePathLength += fileData->getLength();
718 }
719 gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
720 if (gIOHibernateBoot0082Data)
721 {
722 gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
723 if (fileData)
724 {
725 gIOHibernateBoot0082Data->appendBytes(data->getBytesNoCopy(), data->getLength() - 4);
726 gIOHibernateBoot0082Data->appendBytes(fileData);
727 }
728 else gIOHibernateBoot0082Data->appendBytes(data);
729 }
730 }
731 }
732 if (!gIOHibernateBootNextData)
733 {
734 uint16_t bits = 0x0082;
735 gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
736 }
737
738 #if DEBUG || DEVELOPMENT
739 if (kIOLogHibernate & gIOKitDebug) IOKitKernelLogBuffer("H> bootnext:",
740 gIOHibernateBoot0082Data->getBytesNoCopy(), gIOHibernateBoot0082Data->getLength(), &kprintf);
741 #endif /* DEBUG || DEVELOPMENT */
742 if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData)
743 {
744 gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
745 gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
746 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
747 }
748 // BootNext
749 }
750 }
751 #endif /* !i386 && !x86_64 */
752 }
753 while (false);
754
755 if (swapPinned) hibernate_pin_swap(FALSE);
756
757 IOLockLock(gFSLock);
758 if ((kIOReturnSuccess == err) && (kFSOpening != gFSState))
759 {
760 HIBLOG("hibernate file close due timeout\n");
761 err = kIOReturnTimeout;
762 }
763 if (kIOReturnSuccess == err)
764 {
765 gFSState = kFSOpened;
766 gIOHibernateVars = *vars;
767 gFileVars = *vars->fileVars;
768 gFileVars.allocated = false;
769 gIOHibernateVars.fileVars = &gFileVars;
770 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
771 gIOHibernateState = kIOHibernateStateHibernating;
772
773 #if DEBUG || DEVELOPMENT
774 if (kIOLogHibernate & gIOKitDebug)
775 {
776 OSData * data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
777 if (data)
778 {
779 uintptr_t * smcVars = (typeof(smcVars)) data->getBytesNoCopy();
780 IOKitKernelLogBuffer("H> smc:",
781 (const void *)smcVars[1], smcVars[0], &kprintf);
782 }
783 }
784 #endif /* DEBUG || DEVELOPMENT */
785 }
786 else
787 {
788 IOPolledFileIOVars * fileVars = vars->fileVars;
789 IOHibernateDone(vars);
790 IOPolledFileClose(&fileVars,
791 #if DISABLE_TRIM
792 0, NULL, 0, 0, 0);
793 #else
794 0, NULL, 0, sizeof(IOHibernateImageHeader), setFileSize);
795 #endif
796 gFSState = kFSIdle;
797 }
798 IOLockUnlock(gFSLock);
799
800 if (vars->fileVars) IODelete(vars->fileVars, IOPolledFileIOVars, 1);
801 IODelete(vars, IOHibernateVars, 1);
802
803 return (err);
804 }
805
806 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
807
808 static void
809 IOSetBootImageNVRAM(OSData * data)
810 {
811 IORegistryEntry * regEntry;
812
813 if (!gIOOptionsEntry)
814 {
815 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
816 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
817 if (regEntry && !gIOOptionsEntry)
818 regEntry->release();
819 }
820 if (gIOOptionsEntry && gIOHibernateBootImageKey)
821 {
822 if (data)
823 {
824 gIOOptionsEntry->setProperty(gIOHibernateBootImageKey, data);
825 #if DEBUG || DEVELOPMENT
826 if (kIOLogHibernate & gIOKitDebug) IOKitKernelLogBuffer("H> boot-image:",
827 data->getBytesNoCopy(), data->getLength(), &kprintf);
828 #endif /* DEBUG || DEVELOPMENT */
829 }
830 else
831 {
832 gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
833 gIOOptionsEntry->sync();
834 }
835 }
836 }
837
838 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
839 /*
840 * Writes header to disk with signature, block size and file extents data.
841 * If there are more than 2 extents, then they are written on second block.
842 */
843 static IOReturn
844 IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature)
845 {
846 IOHibernateImageHeader hdr;
847 IOItemCount count;
848 IOReturn err = kIOReturnSuccess;
849 int rc;
850 IOPolledFileExtent * fileExtents;
851
852 fileExtents = (typeof(fileExtents)) vars->fileExtents->getBytesNoCopy();
853
854 memset(&hdr, 0, sizeof(IOHibernateImageHeader));
855 count = vars->fileExtents->getLength();
856 if (count > sizeof(hdr.fileExtentMap))
857 {
858 hdr.fileExtentMapSize = count;
859 count = sizeof(hdr.fileExtentMap);
860 }
861 else
862 hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap);
863
864 bcopy(fileExtents, &hdr.fileExtentMap[0], count);
865
866 // copy file block extent list if larger than header
867 if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap))
868 {
869 count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap);
870 rc = kern_write_file(vars->fileRef, vars->blockSize,
871 (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)),
872 count, IO_SKIP_ENCRYPTION);
873 if (rc != 0) {
874 HIBLOG("kern_write_file returned %d\n", rc);
875 err = kIOReturnIOError;
876 goto exit;
877 }
878 }
879 hdr.signature = signature;
880 hdr.deviceBlockSize = vars->blockSize;
881
882 rc = kern_write_file(vars->fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION);
883 if (rc != 0) {
884 HIBLOG("kern_write_file returned %d\n", rc);
885 err = kIOReturnIOError;
886 goto exit;
887 }
888
889 exit:
890 return err;
891 }
892
893 extern "C" boolean_t root_is_CF_drive;
894
895 void
896 IOOpenDebugDataFile(const char *fname, uint64_t size)
897 {
898 IOReturn err;
899 OSData * imagePath = NULL;
900 uint64_t padding;
901
902 if (!gDebugImageLock) {
903 gDebugImageLock = IOLockAlloc();
904 }
905
906 if (root_is_CF_drive) return;
907
908 // Try to get a lock, but don't block for getting lock
909 if (!IOLockTryLock(gDebugImageLock)) {
910 HIBLOG("IOOpenDebugDataFile: Failed to get lock\n");
911 return;
912 }
913
914 if (gDebugImageFileVars || !fname || !size) {
915 HIBLOG("IOOpenDebugDataFile: conditions failed\n");
916 goto exit;
917 }
918
919 padding = (PAGE_SIZE*2); // allocate couple more pages for header and fileextents
920 err = IOPolledFileOpen(fname, size+padding, 32ULL*1024*1024*1024,
921 NULL, 0,
922 &gDebugImageFileVars, &imagePath, NULL, 0);
923
924 if ((kIOReturnSuccess == err) && imagePath)
925 {
926 if ((gDebugImageFileVars->fileSize < (size+padding)) ||
927 (gDebugImageFileVars->fileExtents->getLength() > PAGE_SIZE)) {
928 // Can't use the file
929 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
930 HIBLOG("IOOpenDebugDataFile: too many file extents\n");
931 goto exit;
932 }
933
934 // write extents for debug data usage in EFI
935 IOWriteExtentsToFile(gDebugImageFileVars, kIOHibernateHeaderOpenSignature);
936 IOSetBootImageNVRAM(imagePath);
937 }
938
939 exit:
940 IOLockUnlock(gDebugImageLock);
941
942 if (imagePath) imagePath->release();
943 return;
944 }
945
946 void
947 IOCloseDebugDataFile()
948 {
949 IOSetBootImageNVRAM(0);
950
951 if (gDebugImageLock) {
952 IOLockLock(gDebugImageLock);
953 if (gDebugImageFileVars != 0) {
954 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
955 }
956 IOLockUnlock(gDebugImageLock);
957 }
958
959
960 }
961
962 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
963
964 DECLARE_IOHIBERNATEPROGRESSALPHA
965
966 static void
967 ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
968 {
969 uint32_t rowBytes, pixelShift;
970 uint32_t x, y;
971 int32_t blob;
972 uint32_t alpha, in, color, result;
973 uint8_t * out;
974 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
975
976 rowBytes = display->rowBytes;
977 pixelShift = display->depth >> 4;
978 if (pixelShift < 1) return;
979
980 screen += ((display->width
981 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
982 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
983
984 for (y = 0; y < kIOHibernateProgressHeight; y++)
985 {
986 out = screen + y * rowBytes;
987 for (blob = 0; blob < kIOHibernateProgressCount; blob++)
988 {
989 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
990 for (x = 0; x < kIOHibernateProgressWidth; x++)
991 {
992 alpha = gIOHibernateProgressAlpha[y][x];
993 result = color;
994 if (alpha)
995 {
996 if (0xff != alpha)
997 {
998 if (1 == pixelShift)
999 {
1000 in = *((uint16_t *)out) & 0x1f; // 16
1001 in = (in << 3) | (in >> 2);
1002 }
1003 else
1004 in = *((uint32_t *)out) & 0xff; // 32
1005 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
1006 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
1007 }
1008 if (1 == pixelShift)
1009 {
1010 result >>= 3;
1011 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1012 }
1013 else
1014 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1015 }
1016 out += (1 << pixelShift);
1017 }
1018 out += (kIOHibernateProgressSpacing << pixelShift);
1019 }
1020 }
1021 }
1022
1023
1024 static void
1025 ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
1026 {
1027 uint32_t rowBytes, pixelShift;
1028 uint32_t x, y;
1029 int32_t blob, lastBlob;
1030 uint32_t alpha, in, color, result;
1031 uint8_t * out;
1032 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1033
1034 pixelShift = display->depth >> 4;
1035 if (pixelShift < 1)
1036 return;
1037
1038 rowBytes = display->rowBytes;
1039
1040 screen += ((display->width
1041 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1042 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1043
1044 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
1045
1046 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
1047
1048 for (y = 0; y < kIOHibernateProgressHeight; y++)
1049 {
1050 out = screen + y * rowBytes;
1051 for (blob = firstBlob; blob <= lastBlob; blob++)
1052 {
1053 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
1054 for (x = 0; x < kIOHibernateProgressWidth; x++)
1055 {
1056 alpha = gIOHibernateProgressAlpha[y][x];
1057 result = color;
1058 if (alpha)
1059 {
1060 if (0xff != alpha)
1061 {
1062 in = display->progressSaveUnder[blob][saveindex[blob]++];
1063 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
1064 }
1065 if (1 == pixelShift)
1066 {
1067 result >>= 3;
1068 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1069 }
1070 else
1071 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1072 }
1073 out += (1 << pixelShift);
1074 }
1075 out += (kIOHibernateProgressSpacing << pixelShift);
1076 }
1077 }
1078 }
1079
1080 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1081
1082 IOReturn
1083 IOHibernateIOKitSleep(void)
1084 {
1085 IOReturn ret = kIOReturnSuccess;
1086 IOLockLock(gFSLock);
1087 if (kFSOpening == gFSState)
1088 {
1089 gFSState = kFSTimedOut;
1090 HIBLOG("hibernate file open timed out\n");
1091 ret = kIOReturnTimeout;
1092 }
1093 IOLockUnlock(gFSLock);
1094 return (ret);
1095 }
1096
1097 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1098
1099 IOReturn
1100 IOHibernateSystemHasSlept(void)
1101 {
1102 IOReturn ret = kIOReturnSuccess;
1103 IOHibernateVars * vars = &gIOHibernateVars;
1104 OSObject * obj = 0;
1105 OSData * data;
1106
1107 IOLockLock(gFSLock);
1108 if ((kFSOpened != gFSState) && gIOHibernateMode)
1109 {
1110 ret = kIOReturnTimeout;
1111 }
1112 IOLockUnlock(gFSLock);
1113 if (kIOReturnSuccess != ret) return (ret);
1114
1115 if (gIOHibernateMode) obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
1116 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1117 if (obj && !vars->previewBuffer)
1118 obj->release();
1119
1120 vars->consoleMapping = NULL;
1121 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
1122 {
1123 vars->previewBuffer->release();
1124 vars->previewBuffer = 0;
1125 }
1126
1127 if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1128 && vars->previewBuffer
1129 && (data = OSDynamicCast(OSData,
1130 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
1131 {
1132 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
1133 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
1134
1135 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1136
1137 if (kIOHibernatePreviewUpdates & flags)
1138 {
1139 PE_Video consoleInfo;
1140 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1141
1142 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1143
1144 graphicsInfo->width = consoleInfo.v_width;
1145 graphicsInfo->height = consoleInfo.v_height;
1146 graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
1147 graphicsInfo->depth = consoleInfo.v_depth;
1148 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1149
1150 HIBPRINT("video %p %d %d %d\n",
1151 vars->consoleMapping, graphicsInfo->depth,
1152 graphicsInfo->width, graphicsInfo->height);
1153 if (vars->consoleMapping)
1154 ProgressInit(graphicsInfo, vars->consoleMapping,
1155 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1156 }
1157 }
1158
1159 if (gIOOptionsEntry)
1160 gIOOptionsEntry->sync();
1161
1162 return (ret);
1163 }
1164
1165 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1166
1167 static DeviceTreeNode *
1168 MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry)
1169 {
1170 DeviceTreeNodeProperty * prop;
1171 DeviceTreeNode * child;
1172 IORegistryEntry * childRegEntry;
1173 const char * nameProp;
1174 unsigned int propLen, idx;
1175
1176 prop = (DeviceTreeNodeProperty *) (entry + 1);
1177 for (idx = 0; idx < entry->nProperties; idx++)
1178 {
1179 if (regEntry && (0 != strcmp("name", prop->name)))
1180 {
1181 regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
1182 // HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
1183 }
1184 prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
1185 }
1186
1187 child = (DeviceTreeNode *) prop;
1188 for (idx = 0; idx < entry->nChildren; idx++)
1189 {
1190 if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen))
1191 panic("no name");
1192 childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
1193 // HIBPRINT("%s == %p\n", nameProp, childRegEntry);
1194 child = MergeDeviceTree(child, childRegEntry);
1195 }
1196 return (child);
1197 }
1198
1199 IOReturn
1200 IOHibernateSystemWake(void)
1201 {
1202 if (kFSOpened == gFSState)
1203 {
1204 IOPolledFilePollersClose(gIOHibernateVars.fileVars, kIOPolledPostflightState);
1205 IOHibernateDone(&gIOHibernateVars);
1206 }
1207 else
1208 {
1209 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1210 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1211 }
1212 return (kIOReturnSuccess);
1213 }
1214
1215 static IOReturn
1216 IOHibernateDone(IOHibernateVars * vars)
1217 {
1218 IOReturn err;
1219 OSData * data;
1220
1221 hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
1222
1223 if (vars->videoMapping)
1224 {
1225 if (vars->videoMapSize)
1226 // remove mappings
1227 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1228 if (vars->videoAllocSize)
1229 // dealloc range
1230 kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
1231 }
1232
1233 if (vars->previewBuffer)
1234 {
1235 vars->previewBuffer->release();
1236 vars->previewBuffer = 0;
1237 }
1238
1239 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1240 {
1241 IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
1242 gIOHibernateCurrentHeader->options, 32);
1243 }
1244 else
1245 {
1246 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1247 }
1248
1249 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1250 && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus))
1251 {
1252 IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
1253 &gIOHibernateGraphicsInfo->gfxStatus,
1254 sizeof(gIOHibernateGraphicsInfo->gfxStatus));
1255 }
1256 else
1257 {
1258 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1259 }
1260
1261 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1262
1263 #if defined(__i386__) || defined(__x86_64__)
1264 IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
1265 IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
1266
1267 /*
1268 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1269 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1270 */
1271 if (gIOOptionsEntry) {
1272
1273 if (gIOHibernateRTCVariablesKey) {
1274 if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
1275 gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
1276 }
1277 }
1278
1279 if (gIOHibernateBootNextKey)
1280 {
1281 if (gIOHibernateBootNextSave)
1282 {
1283 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
1284 gIOHibernateBootNextSave->release();
1285 gIOHibernateBootNextSave = NULL;
1286 }
1287 else
1288 gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
1289 }
1290 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) gIOOptionsEntry->sync();
1291 }
1292 #endif
1293
1294 if (vars->srcBuffer) vars->srcBuffer->release();
1295 bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
1296 if (vars->handoffBuffer)
1297 {
1298 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1299 {
1300 IOHibernateHandoff * handoff;
1301 bool done = false;
1302 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
1303 !done;
1304 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
1305 {
1306 HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
1307 uint8_t * data = &handoff->data[0];
1308 switch (handoff->type)
1309 {
1310 case kIOHibernateHandoffTypeEnd:
1311 done = true;
1312 break;
1313
1314 case kIOHibernateHandoffTypeDeviceTree:
1315 MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
1316 break;
1317
1318 case kIOHibernateHandoffTypeKeyStore:
1319 #if defined(__i386__) || defined(__x86_64__)
1320 {
1321 IOBufferMemoryDescriptor *
1322 md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
1323 if (md)
1324 {
1325 IOSetKeyStoreData(md);
1326 }
1327 }
1328 #endif
1329 break;
1330
1331 default:
1332 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
1333 break;
1334 }
1335 }
1336 #if defined(__i386__) || defined(__x86_64__)
1337 if (vars->volumeCryptKeySize)
1338 {
1339 IOBufferMemoryDescriptor *
1340 bmd = IOBufferMemoryDescriptor::withBytes(&vars->volumeCryptKey[0],
1341 vars->volumeCryptKeySize, kIODirectionOutIn);
1342 if (!bmd) panic("IOBufferMemoryDescriptor");
1343 IOSetAPFSKeyStoreData(bmd);
1344 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
1345 }
1346 #endif
1347
1348 }
1349 vars->handoffBuffer->release();
1350 }
1351
1352 if (gIOChosenEntry
1353 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey)))
1354 && (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength()))
1355 {
1356 bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0],
1357 sizeof(gIOHibernateBridgeBootSessionUUIDString));
1358 }
1359
1360 if (vars->hwEncrypt)
1361 {
1362 err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, NULL, 0);
1363 HIBLOG("IOPolledFilePollersSetEncryptionKey(0,%x)\n", err);
1364 }
1365
1366 bzero(vars, sizeof(*vars));
1367
1368 // gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1369
1370 return (kIOReturnSuccess);
1371 }
1372
1373 static void
1374 IOHibernateSystemPostWakeTrim(void * p1, void * p2)
1375 {
1376 // invalidate & close the image file
1377 if (p1) IOLockLock(gFSLock);
1378 if (kFSTrimDelay == gFSState)
1379 {
1380 IOPolledFileIOVars * vars = &gFileVars;
1381 IOPolledFileClose(&vars,
1382 #if DISABLE_TRIM
1383 0, NULL, 0, 0, 0);
1384 #else
1385 0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader),
1386 sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize);
1387 #endif
1388 gFSState = kFSIdle;
1389 }
1390 if (p1) IOLockUnlock(gFSLock);
1391 }
1392
1393 IOReturn
1394 IOHibernateSystemPostWake(bool now)
1395 {
1396 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1397 IOLockLock(gFSLock);
1398 if (kFSTrimDelay == gFSState)
1399 {
1400 thread_call_cancel(gIOHibernateTrimCalloutEntry);
1401 IOHibernateSystemPostWakeTrim(NULL, NULL);
1402 }
1403 else if (kFSOpened != gFSState) gFSState = kFSIdle;
1404 else
1405 {
1406 gFSState = kFSTrimDelay;
1407 if (now)
1408 {
1409 thread_call_cancel(gIOHibernateTrimCalloutEntry);
1410 IOHibernateSystemPostWakeTrim(NULL, NULL);
1411 }
1412 else
1413 {
1414 AbsoluteTime deadline;
1415 clock_interval_to_deadline(TRIM_DELAY, kMillisecondScale, &deadline );
1416 thread_call_enter1_delayed(gIOHibernateTrimCalloutEntry, NULL, deadline);
1417 }
1418 }
1419 IOLockUnlock(gFSLock);
1420
1421 return (kIOReturnSuccess);
1422 }
1423
1424 uint32_t IOHibernateWasScreenLocked(void)
1425 {
1426 uint32_t ret = 0;
1427 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry)
1428 {
1429 OSData *
1430 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
1431 if (data)
1432 {
1433 ret = ((uint32_t *)data->getBytesNoCopy())[0];
1434 gIOChosenEntry->setProperty(kIOBooterScreenLockStateKey, data);
1435 }
1436 }
1437 else gIOChosenEntry->removeProperty(kIOBooterScreenLockStateKey);
1438
1439 return (ret);
1440 }
1441
1442 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1443
1444 SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
1445 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1446 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
1447 SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
1448 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1449 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
1450 SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
1451 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1452 &gIOHibernateMode, 0, "");
1453 SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
1454 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1455 &_hibernateStats, hibernate_statistics_t, "");
1456 SYSCTL_STRING(_kern_bridge, OID_AUTO, bootsessionuuid,
1457 CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1458 gIOHibernateBridgeBootSessionUUIDString, sizeof(gIOHibernateBridgeBootSessionUUIDString), "");
1459
1460 SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
1461 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1462 &_hibernateStats.graphicsReadyTime, 0, "");
1463 SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
1464 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1465 &_hibernateStats.wakeNotificationTime, 0, "");
1466 SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
1467 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1468 &_hibernateStats.lockScreenReadyTime, 0, "");
1469 SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
1470 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1471 &_hibernateStats.hidReadyTime, 0, "");
1472
1473 void
1474 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1475 {
1476 gIOHibernateBootImageKey = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1477 gIOHibernateBootSignatureKey = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
1478 gIOBridgeBootSessionUUIDKey = OSSymbol::withCStringNoCopy(kIOBridgeBootSessionUUIDKey);
1479
1480 #if defined(__i386__) || defined(__x86_64__)
1481 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1482 gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
1483 gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
1484 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1485 #endif /* defined(__i386__) || defined(__x86_64__) */
1486
1487 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
1488 if (data)
1489 {
1490 rootDomain->setProperty(kIOHibernateStateKey, data);
1491 data->release();
1492 }
1493
1494 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))
1495 gIOHibernateMode = kIOHibernateModeOn;
1496 else
1497 gIOHibernateFilename[0] = 0;
1498
1499 sysctl_register_oid(&sysctl__kern_hibernatefile);
1500 sysctl_register_oid(&sysctl__kern_bootsignature);
1501 sysctl_register_oid(&sysctl__kern_hibernatemode);
1502 sysctl_register_oid(&sysctl__kern_hibernatestatistics);
1503 sysctl_register_oid(&sysctl__kern_hibernategraphicsready);
1504 sysctl_register_oid(&sysctl__kern_hibernatewakenotification);
1505 sysctl_register_oid(&sysctl__kern_hibernatelockscreenready);
1506 sysctl_register_oid(&sysctl__kern_hibernatehidready);
1507
1508 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1509
1510 if (gIOChosenEntry
1511 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey)))
1512 && (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength()))
1513 {
1514 sysctl_register_oid(&sysctl__kern_bridge_bootsessionuuid);
1515 bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0], sizeof(gIOHibernateBridgeBootSessionUUIDString));
1516 }
1517
1518 gFSLock = IOLockAlloc();
1519 }
1520
1521 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1522
1523 static IOReturn
1524 IOHibernatePolledFileWrite(IOPolledFileIOVars * vars,
1525 const uint8_t * bytes, IOByteCount size,
1526 IOPolledFileCryptVars * cryptvars)
1527 {
1528 IOReturn err;
1529
1530 err = IOPolledFileWrite(vars, bytes, size, cryptvars);
1531 if ((kIOReturnSuccess == err) && hibernate_should_abort()) err = kIOReturnAborted;
1532
1533 return (err);
1534 }
1535
1536 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1537
1538 extern "C" uint32_t
1539 hibernate_write_image(void)
1540 {
1541 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
1542 IOHibernateVars * vars = &gIOHibernateVars;
1543 IOPolledFileExtent * fileExtents;
1544
1545 _static_assert_1_arg(sizeof(IOHibernateImageHeader) == 512);
1546
1547 uint32_t pageCount, pagesDone;
1548 IOReturn err;
1549 vm_offset_t ppnum, page;
1550 IOItemCount count;
1551 uint8_t * src;
1552 uint8_t * data;
1553 uint8_t * compressed;
1554 uint8_t * scratch;
1555 IOByteCount pageCompressedSize;
1556 uint64_t compressedSize, uncompressedSize;
1557 uint64_t image1Size = 0;
1558 uint32_t bitmap_size;
1559 bool iterDone, pollerOpen, needEncrypt;
1560 uint32_t restore1Sum, sum, sum1, sum2;
1561 int wkresult;
1562 uint32_t tag;
1563 uint32_t pageType;
1564 uint32_t pageAndCount[2];
1565 addr64_t phys64;
1566 IOByteCount segLen;
1567 uintptr_t hibernateBase;
1568 uintptr_t hibernateEnd;
1569
1570 AbsoluteTime startTime, endTime;
1571 AbsoluteTime allTime, compTime;
1572 uint64_t compBytes;
1573 uint64_t nsec;
1574 uint32_t lastProgressStamp = 0;
1575 uint32_t progressStamp;
1576 uint32_t blob, lastBlob = (uint32_t) -1L;
1577
1578 uint32_t wiredPagesEncrypted;
1579 uint32_t dirtyPagesEncrypted;
1580 uint32_t wiredPagesClear;
1581 uint32_t svPageCount;
1582 uint32_t zvPageCount;
1583
1584 IOPolledFileCryptVars _cryptvars;
1585 IOPolledFileCryptVars * cryptvars = 0;
1586
1587 wiredPagesEncrypted = 0;
1588 dirtyPagesEncrypted = 0;
1589 wiredPagesClear = 0;
1590 svPageCount = 0;
1591 zvPageCount = 0;
1592
1593 if (!vars->fileVars
1594 || !vars->fileVars->pollers
1595 || !(kIOHibernateModeOn & gIOHibernateMode)) return (kIOHibernatePostWriteSleep);
1596
1597 if (kIOHibernateModeSleep & gIOHibernateMode)
1598 kdebug_enable = save_kdebug_enable;
1599
1600 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START);
1601 IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
1602
1603 restore1Sum = sum1 = sum2 = 0;
1604
1605 #if CRYPTO
1606 // encryption data. "iv" is the "initial vector".
1607 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1608 {
1609 static const unsigned char first_iv[AES_BLOCK_SIZE]
1610 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
1611 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
1612
1613 cryptvars = &gIOHibernateCryptWakeContext;
1614 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1615 aes_encrypt_key(vars->cryptKey,
1616 kIOHibernateAESKeySize,
1617 &cryptvars->ctx.encrypt);
1618 aes_decrypt_key(vars->cryptKey,
1619 kIOHibernateAESKeySize,
1620 &cryptvars->ctx.decrypt);
1621
1622 cryptvars = &_cryptvars;
1623 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1624 for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++)
1625 vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
1626 aes_encrypt_key(vars->wiredCryptKey,
1627 kIOHibernateAESKeySize,
1628 &cryptvars->ctx.encrypt);
1629
1630 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1631 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1632 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
1633 }
1634 #endif /* CRYPTO */
1635
1636 hibernate_page_list_setall(vars->page_list,
1637 vars->page_list_wired,
1638 vars->page_list_pal,
1639 false /* !preflight */,
1640 /* discard_all */
1641 ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
1642 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
1643 &pageCount);
1644
1645 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
1646
1647 fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy();
1648
1649 #if 0
1650 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
1651 for (page = 0; page < count; page++)
1652 {
1653 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
1654 fileExtents[page].start, fileExtents[page].length,
1655 fileExtents[page].start + fileExtents[page].length);
1656 }
1657 #endif
1658
1659 needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
1660 AbsoluteTime_to_scalar(&compTime) = 0;
1661 compBytes = 0;
1662
1663 clock_get_uptime(&allTime);
1664 IOService::getPMRootDomain()->pmStatsRecordEvent(
1665 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
1666 do
1667 {
1668 compressedSize = 0;
1669 uncompressedSize = 0;
1670 svPageCount = 0;
1671 zvPageCount = 0;
1672
1673 IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
1674
1675 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
1676 ml_get_interrupts_enabled());
1677 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState,
1678 // abortable if not low battery
1679 !IOService::getPMRootDomain()->mustHibernate());
1680 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
1681 pollerOpen = (kIOReturnSuccess == err);
1682 if (!pollerOpen)
1683 break;
1684
1685 if (vars->volumeCryptKeySize)
1686 {
1687 err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
1688 HIBLOG("IOPolledFilePollersSetEncryptionKey(%x)\n", err);
1689 vars->hwEncrypt = (kIOReturnSuccess == err);
1690 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
1691 if (vars->hwEncrypt) header->options |= kIOHibernateOptionHWEncrypt;
1692 }
1693
1694 // copy file block extent list if larger than header
1695
1696 count = vars->fileVars->fileExtents->getLength();
1697 if (count > sizeof(header->fileExtentMap))
1698 {
1699 count -= sizeof(header->fileExtentMap);
1700 err = IOHibernatePolledFileWrite(vars->fileVars,
1701 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
1702 if (kIOReturnSuccess != err)
1703 break;
1704 }
1705
1706 hibernateBase = HIB_BASE; /* Defined in PAL headers */
1707 hibernateEnd = (segHIBB + segSizeHIB);
1708
1709 // copy out restore1 code
1710
1711 for (count = 0;
1712 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1713 count += segLen)
1714 {
1715 for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++)
1716 {
1717 gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
1718 }
1719 }
1720
1721 page = atop_32(kvtophys(hibernateBase));
1722 count = atop_32(round_page(hibernateEnd) - hibernateBase);
1723 header->restore1CodePhysPage = page;
1724 header->restore1CodeVirt = hibernateBase;
1725 header->restore1PageCount = count;
1726 header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
1727 header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
1728
1729 if (uuid_parse(&gIOHibernateBridgeBootSessionUUIDString[0], &header->bridgeBootSessionUUID[0]))
1730 {
1731 bzero(&header->bridgeBootSessionUUID[0], sizeof(header->bridgeBootSessionUUID));
1732 }
1733
1734 // sum __HIB seg, with zeros for the stack
1735 src = (uint8_t *) trunc_page(hibernateBase);
1736 for (page = 0; page < count; page++)
1737 {
1738 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
1739 restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
1740 else
1741 restore1Sum += 0x00000000;
1742 src += page_size;
1743 }
1744 sum1 = restore1Sum;
1745
1746 // write the __HIB seg, with zeros for the stack
1747
1748 src = (uint8_t *) trunc_page(hibernateBase);
1749 count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
1750 if (count)
1751 {
1752 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
1753 if (kIOReturnSuccess != err)
1754 break;
1755 }
1756 err = IOHibernatePolledFileWrite(vars->fileVars,
1757 (uint8_t *) 0,
1758 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
1759 cryptvars);
1760 if (kIOReturnSuccess != err)
1761 break;
1762 src = &gIOHibernateRestoreStackEnd[0];
1763 count = round_page(hibernateEnd) - ((uintptr_t) src);
1764 if (count)
1765 {
1766 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
1767 if (kIOReturnSuccess != err)
1768 break;
1769 }
1770
1771 if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode))
1772 {
1773 vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
1774 vars->fileVars->encryptEnd = UINT64_MAX;
1775 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1776 }
1777
1778 // write the preview buffer
1779
1780 if (vars->previewBuffer)
1781 {
1782 ppnum = 0;
1783 count = 0;
1784 do
1785 {
1786 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
1787 pageAndCount[0] = atop_64(phys64);
1788 pageAndCount[1] = atop_32(segLen);
1789 err = IOHibernatePolledFileWrite(vars->fileVars,
1790 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1791 cryptvars);
1792 if (kIOReturnSuccess != err)
1793 break;
1794 count += segLen;
1795 ppnum += sizeof(pageAndCount);
1796 }
1797 while (phys64);
1798 if (kIOReturnSuccess != err)
1799 break;
1800
1801 src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
1802
1803 ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
1804
1805 count = vars->previewBuffer->getLength();
1806
1807 header->previewPageListSize = ppnum;
1808 header->previewSize = count + ppnum;
1809
1810 for (page = 0; page < count; page += page_size)
1811 {
1812 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
1813 sum1 += hibernate_sum_page(src + page, atop_64(phys64));
1814 }
1815 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
1816 if (kIOReturnSuccess != err)
1817 break;
1818 }
1819
1820 // mark areas for no save
1821 IOMemoryDescriptor * ioBuffer;
1822 ioBuffer = IOPolledFileGetIOBuffer(vars->fileVars);
1823 for (count = 0;
1824 (phys64 = ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1825 count += segLen)
1826 {
1827 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1828 atop_64(phys64), atop_32(segLen),
1829 kIOHibernatePageStateFree);
1830 pageCount -= atop_32(segLen);
1831 }
1832
1833 for (count = 0;
1834 (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1835 count += segLen)
1836 {
1837 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1838 atop_64(phys64), atop_32(segLen),
1839 kIOHibernatePageStateFree);
1840 pageCount -= atop_32(segLen);
1841 }
1842
1843 // copy out bitmap of pages available for trashing during restore
1844
1845 bitmap_size = vars->page_list_wired->list_size;
1846 src = (uint8_t *) vars->page_list_wired;
1847 err = IOHibernatePolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
1848 if (kIOReturnSuccess != err)
1849 break;
1850
1851 // mark more areas for no save, but these are not available
1852 // for trashing during restore
1853
1854 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
1855
1856
1857 page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
1858 count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
1859 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1860 page, count,
1861 kIOHibernatePageStateFree);
1862 pageCount -= count;
1863
1864 if (vars->previewBuffer) for (count = 0;
1865 (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1866 count += segLen)
1867 {
1868 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1869 atop_64(phys64), atop_32(segLen),
1870 kIOHibernatePageStateFree);
1871 pageCount -= atop_32(segLen);
1872 }
1873
1874 for (count = 0;
1875 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1876 count += segLen)
1877 {
1878 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1879 atop_64(phys64), atop_32(segLen),
1880 kIOHibernatePageStateFree);
1881 pageCount -= atop_32(segLen);
1882 }
1883
1884 #if KASAN
1885 vm_size_t shadow_pages_free = atop_64(shadow_ptop) - atop_64(shadow_pnext);
1886
1887 /* no need to save unused shadow pages */
1888 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1889 atop_64(shadow_pnext),
1890 shadow_pages_free,
1891 kIOHibernatePageStateFree);
1892 #endif
1893
1894 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
1895 compressed = src + page_size;
1896 scratch = compressed + page_size;
1897
1898 pagesDone = 0;
1899 lastBlob = 0;
1900
1901 HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
1902 bitmap_size, header->previewSize,
1903 pageCount, vars->fileVars->position);
1904
1905 enum
1906 // pageType
1907 {
1908 kWired = 0x02,
1909 kEncrypt = 0x01,
1910 kWiredEncrypt = kWired | kEncrypt,
1911 kWiredClear = kWired,
1912 kUnwiredEncrypt = kEncrypt
1913 };
1914
1915 bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features()));
1916 #define _pmap_is_noencrypt(x) (cpuAES ? false : pmap_is_noencrypt((x)))
1917
1918 for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)
1919 {
1920 if (kUnwiredEncrypt == pageType)
1921 {
1922 // start unwired image
1923 if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode))
1924 {
1925 vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
1926 vars->fileVars->encryptEnd = UINT64_MAX;
1927 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1928 }
1929 bcopy(&cryptvars->aes_iv[0],
1930 &gIOHibernateCryptWakeContext.aes_iv[0],
1931 sizeof(cryptvars->aes_iv));
1932 cryptvars = &gIOHibernateCryptWakeContext;
1933 }
1934 for (iterDone = false, ppnum = 0; !iterDone; )
1935 {
1936 count = hibernate_page_list_iterate((kWired & pageType)
1937 ? vars->page_list_wired : vars->page_list,
1938 &ppnum);
1939 // kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
1940 iterDone = !count;
1941
1942 if (count && (kWired & pageType) && needEncrypt)
1943 {
1944 uint32_t checkIndex;
1945 for (checkIndex = 0;
1946 (checkIndex < count)
1947 && (((kEncrypt & pageType) == 0) == _pmap_is_noencrypt(ppnum + checkIndex));
1948 checkIndex++)
1949 {}
1950 if (!checkIndex)
1951 {
1952 ppnum++;
1953 continue;
1954 }
1955 count = checkIndex;
1956 }
1957
1958 switch (pageType)
1959 {
1960 case kWiredEncrypt: wiredPagesEncrypted += count; break;
1961 case kWiredClear: wiredPagesClear += count; break;
1962 case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
1963 }
1964
1965 if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */}
1966 else
1967 {
1968 pageAndCount[0] = ppnum;
1969 pageAndCount[1] = count;
1970 err = IOHibernatePolledFileWrite(vars->fileVars,
1971 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1972 cryptvars);
1973 if (kIOReturnSuccess != err)
1974 break;
1975 }
1976
1977 for (page = ppnum; page < (ppnum + count); page++)
1978 {
1979 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
1980 if (err)
1981 {
1982 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
1983 break;
1984 }
1985
1986 sum = hibernate_sum_page(src, page);
1987 if (kWired & pageType)
1988 sum1 += sum;
1989 else
1990 sum2 += sum;
1991
1992 clock_get_uptime(&startTime);
1993 wkresult = WKdm_compress_new((const WK_word*) src,
1994 (WK_word*) compressed,
1995 (WK_word*) scratch,
1996 page_size - 4);
1997
1998 clock_get_uptime(&endTime);
1999 ADD_ABSOLUTETIME(&compTime, &endTime);
2000 SUB_ABSOLUTETIME(&compTime, &startTime);
2001
2002 compBytes += page_size;
2003 pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
2004
2005 if (pageCompressedSize == 0)
2006 {
2007 pageCompressedSize = 4;
2008 data = src;
2009
2010 if (*(uint32_t *)src)
2011 svPageCount++;
2012 else
2013 zvPageCount++;
2014 }
2015 else
2016 {
2017 if (pageCompressedSize != page_size)
2018 data = compressed;
2019 else
2020 data = src;
2021 }
2022
2023 tag = pageCompressedSize | kIOHibernateTagSignature;
2024 err = IOHibernatePolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
2025 if (kIOReturnSuccess != err)
2026 break;
2027
2028 err = IOHibernatePolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
2029 if (kIOReturnSuccess != err)
2030 break;
2031
2032 compressedSize += pageCompressedSize;
2033 uncompressedSize += page_size;
2034 pagesDone++;
2035
2036 if (vars->consoleMapping && (0 == (1023 & pagesDone)))
2037 {
2038 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
2039 if (blob != lastBlob)
2040 {
2041 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
2042 lastBlob = blob;
2043 }
2044 }
2045 if (0 == (8191 & pagesDone))
2046 {
2047 clock_get_uptime(&endTime);
2048 SUB_ABSOLUTETIME(&endTime, &allTime);
2049 absolutetime_to_nanoseconds(endTime, &nsec);
2050 progressStamp = nsec / 750000000ULL;
2051 if (progressStamp != lastProgressStamp)
2052 {
2053 lastProgressStamp = progressStamp;
2054 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
2055 }
2056 }
2057 }
2058 if (kIOReturnSuccess != err)
2059 break;
2060 ppnum = page;
2061 }
2062
2063 if (kIOReturnSuccess != err)
2064 break;
2065
2066 if ((kEncrypt & pageType) && vars->fileVars->encryptStart)
2067 {
2068 vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
2069 HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
2070 }
2071
2072 if (kWiredEncrypt != pageType)
2073 {
2074 // end of image1/2 - fill to next block
2075 err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2076 if (kIOReturnSuccess != err)
2077 break;
2078 }
2079 if (kWiredClear == pageType)
2080 {
2081 // enlarge wired image for test
2082 // err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
2083
2084 // end wired image
2085 header->encryptStart = vars->fileVars->encryptStart;
2086 header->encryptEnd = vars->fileVars->encryptEnd;
2087 image1Size = vars->fileVars->position;
2088 HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
2089 image1Size, header->encryptStart, header->encryptEnd);
2090 }
2091 }
2092 if (kIOReturnSuccess != err)
2093 {
2094 if (kIOReturnOverrun == err)
2095 {
2096 // update actual compression ratio on not enough space (for retry)
2097 gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
2098 }
2099
2100 // update partial amount written (for IOPolledFileClose cleanup/unmap)
2101 header->imageSize = vars->fileVars->position;
2102 break;
2103 }
2104
2105 // Header:
2106
2107 header->imageSize = vars->fileVars->position;
2108 header->image1Size = image1Size;
2109 header->bitmapSize = bitmap_size;
2110 header->pageCount = pageCount;
2111
2112 header->restore1Sum = restore1Sum;
2113 header->image1Sum = sum1;
2114 header->image2Sum = sum2;
2115 header->sleepTime = gIOLastSleepTime.tv_sec;
2116
2117 header->compression = (compressedSize << 8) / uncompressedSize;
2118 gIOHibernateCompression = header->compression;
2119
2120 count = vars->fileVars->fileExtents->getLength();
2121 if (count > sizeof(header->fileExtentMap))
2122 {
2123 header->fileExtentMapSize = count;
2124 count = sizeof(header->fileExtentMap);
2125 }
2126 else
2127 header->fileExtentMapSize = sizeof(header->fileExtentMap);
2128 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
2129
2130 header->deviceBase = vars->fileVars->block0;
2131 header->deviceBlockSize = vars->fileVars->blockSize;
2132
2133 IOPolledFileSeek(vars->fileVars, 0);
2134 err = IOHibernatePolledFileWrite(vars->fileVars,
2135 (uint8_t *) header, sizeof(IOHibernateImageHeader),
2136 cryptvars);
2137 if (kIOReturnSuccess != err)
2138 break;
2139 err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2140 }
2141 while (false);
2142
2143 clock_get_uptime(&endTime);
2144
2145 IOService::getPMRootDomain()->pmStatsRecordEvent(
2146 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
2147
2148 SUB_ABSOLUTETIME(&endTime, &allTime);
2149 absolutetime_to_nanoseconds(endTime, &nsec);
2150 HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
2151
2152 absolutetime_to_nanoseconds(compTime, &nsec);
2153 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2154 compBytes,
2155 nsec / 1000000ULL,
2156 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2157
2158 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2159 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
2160 vars->fileVars->cryptBytes,
2161 nsec / 1000000ULL,
2162 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2163
2164 HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
2165 header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
2166 uncompressedSize, atop_32(uncompressedSize), compressedSize,
2167 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
2168 sum1, sum2);
2169
2170 HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
2171 svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
2172
2173 if (pollerOpen)
2174 IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted );
2175
2176 if (vars->consoleMapping)
2177 ProgressUpdate(gIOHibernateGraphicsInfo,
2178 vars->consoleMapping, 0, kIOHibernateProgressCount);
2179
2180 HIBLOG("hibernate_write_image done(%x)\n", err);
2181
2182 // should we come back via regular wake, set the state in memory.
2183 gIOHibernateState = kIOHibernateStateInactive;
2184
2185 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, wiredPagesEncrypted,
2186 wiredPagesClear, dirtyPagesEncrypted);
2187
2188 if (kIOReturnSuccess == err)
2189 {
2190 if (kIOHibernateModeSleep & gIOHibernateMode)
2191 {
2192 return (kIOHibernatePostWriteSleep);
2193 }
2194 else if(kIOHibernateModeRestart & gIOHibernateMode)
2195 {
2196 return (kIOHibernatePostWriteRestart);
2197 }
2198 else
2199 {
2200 /* by default, power down */
2201 return (kIOHibernatePostWriteHalt);
2202 }
2203 }
2204 else if (kIOReturnAborted == err)
2205 {
2206 return (kIOHibernatePostWriteWake);
2207 }
2208 else
2209 {
2210 /* on error, sleep */
2211 return (kIOHibernatePostWriteSleep);
2212 }
2213 }
2214
2215 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2216
2217 extern "C" void
2218 hibernate_machine_init(void)
2219 {
2220 IOReturn err;
2221 uint32_t sum;
2222 uint32_t pagesDone;
2223 uint32_t pagesRead = 0;
2224 AbsoluteTime startTime, compTime;
2225 AbsoluteTime allTime, endTime;
2226 AbsoluteTime startIOTime, endIOTime;
2227 uint64_t nsec, nsecIO;
2228 uint64_t compBytes;
2229 uint32_t lastProgressStamp = 0;
2230 uint32_t progressStamp;
2231 IOPolledFileCryptVars * cryptvars = 0;
2232
2233 IOHibernateVars * vars = &gIOHibernateVars;
2234 bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
2235
2236 if (!vars->fileVars || !vars->fileVars->pollers)
2237 return;
2238
2239 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2240 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2241
2242 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
2243 {
2244 HIBLOG("regular wake\n");
2245 return;
2246 }
2247
2248 HIBPRINT("diag %x %x %x %x\n",
2249 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
2250 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
2251
2252 #define t40ms(x) (tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)
2253 #define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
2254 tStat(booterStart, booterStart);
2255 gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart;
2256 tStat(booterDuration0, booterTime0);
2257 tStat(booterDuration1, booterTime1);
2258 tStat(booterDuration2, booterTime2);
2259 tStat(booterDuration, booterTime);
2260 tStat(booterConnectDisplayDuration, connectDisplayTime);
2261 tStat(booterSplashDuration, splashTime);
2262 tStat(trampolineDuration, trampolineTime);
2263
2264 gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size;
2265 gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize;
2266 gIOHibernateStats->image1Pages = pagesDone;
2267
2268 /* HIBERNATE_stats */
2269 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 14), gIOHibernateStats->smcStart,
2270 gIOHibernateStats->booterStart, gIOHibernateStats->booterDuration,
2271 gIOHibernateStats->trampolineDuration);
2272
2273 HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
2274 gIOHibernateStats->booterStart,
2275 gIOHibernateStats->smcStart,
2276 gIOHibernateStats->booterDuration0,
2277 gIOHibernateStats->booterDuration1,
2278 gIOHibernateStats->booterDuration2,
2279 gIOHibernateStats->booterDuration,
2280 gIOHibernateStats->booterConnectDisplayDuration,
2281 gIOHibernateStats->booterSplashDuration,
2282 gIOHibernateStats->trampolineDuration);
2283
2284 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
2285 gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
2286 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
2287
2288 if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
2289 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)))
2290 {
2291 hibernate_page_list_discard(vars->page_list);
2292 }
2293
2294 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
2295
2296 if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount)
2297 panic("handoff overflow");
2298
2299 IOHibernateHandoff * handoff;
2300 bool done = false;
2301 bool foundCryptData = false;
2302 bool foundVolumeEncryptData = false;
2303
2304 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
2305 !done;
2306 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
2307 {
2308 // HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
2309 uint8_t * data = &handoff->data[0];
2310 switch (handoff->type)
2311 {
2312 case kIOHibernateHandoffTypeEnd:
2313 done = true;
2314 break;
2315
2316 case kIOHibernateHandoffTypeGraphicsInfo:
2317 if (handoff->bytecount == sizeof(*gIOHibernateGraphicsInfo))
2318 {
2319 bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
2320 }
2321 break;
2322
2323 case kIOHibernateHandoffTypeCryptVars:
2324 if (cryptvars)
2325 {
2326 hibernate_cryptwakevars_t *
2327 wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
2328 bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
2329 }
2330 foundCryptData = true;
2331 bzero(data, handoff->bytecount);
2332 break;
2333
2334 case kIOHibernateHandoffTypeVolumeCryptKey:
2335 if (handoff->bytecount == vars->volumeCryptKeySize)
2336 {
2337 bcopy(data, &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
2338 foundVolumeEncryptData = true;
2339 }
2340 else panic("kIOHibernateHandoffTypeVolumeCryptKey(%d)", handoff->bytecount);
2341 break;
2342
2343 case kIOHibernateHandoffTypeMemoryMap:
2344
2345 clock_get_uptime(&allTime);
2346
2347 hibernate_newruntime_map(data, handoff->bytecount,
2348 gIOHibernateCurrentHeader->systemTableOffset);
2349
2350 clock_get_uptime(&endTime);
2351
2352 SUB_ABSOLUTETIME(&endTime, &allTime);
2353 absolutetime_to_nanoseconds(endTime, &nsec);
2354
2355 HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
2356
2357 break;
2358
2359 case kIOHibernateHandoffTypeDeviceTree:
2360 {
2361 // DTEntry chosen = NULL;
2362 // HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen));
2363 }
2364 break;
2365
2366 default:
2367 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
2368 break;
2369 }
2370 }
2371
2372 if (vars->hwEncrypt && !foundVolumeEncryptData)
2373 panic("no volumeCryptKey");
2374 else if (cryptvars && !foundCryptData)
2375 panic("hibernate handoff");
2376
2377 HIBPRINT("video 0x%llx %d %d %d status %x\n",
2378 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2379 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
2380
2381 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)
2382 {
2383 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2384 * gIOHibernateGraphicsInfo->rowBytes);
2385 if (vars->videoMapSize > vars->videoAllocSize) vars->videoMapSize = 0;
2386 else
2387 {
2388 IOMapPages(kernel_map,
2389 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2390 vars->videoMapSize, kIOMapInhibitCache );
2391 }
2392 }
2393
2394 if (vars->videoMapSize)
2395 ProgressUpdate(gIOHibernateGraphicsInfo,
2396 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2397
2398 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2399 uint8_t * compressed = src + page_size;
2400 uint8_t * scratch = compressed + page_size;
2401 uint32_t decoOffset;
2402
2403 clock_get_uptime(&allTime);
2404 AbsoluteTime_to_scalar(&compTime) = 0;
2405 compBytes = 0;
2406
2407 HIBLOG("IOPolledFilePollersOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2408 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledAfterSleepState, false);
2409 clock_get_uptime(&startIOTime);
2410 endTime = startIOTime;
2411 SUB_ABSOLUTETIME(&endTime, &allTime);
2412 absolutetime_to_nanoseconds(endTime, &nsec);
2413 HIBLOG("IOPolledFilePollersOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
2414
2415 if (vars->hwEncrypt)
2416 {
2417 err = IOPolledFilePollersSetEncryptionKey(vars->fileVars,
2418 &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
2419 HIBLOG("IOPolledFilePollersSetEncryptionKey(%x) %ld\n", err, vars->volumeCryptKeySize);
2420 if (kIOReturnSuccess != err) panic("IOPolledFilePollersSetEncryptionKey(0x%x)", err);
2421 cryptvars = 0;
2422 }
2423
2424 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2425
2426 // kick off the read ahead
2427 vars->fileVars->bufferHalf = 0;
2428 vars->fileVars->bufferLimit = 0;
2429 vars->fileVars->lastRead = 0;
2430 vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
2431 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2432 vars->fileVars->cryptBytes = 0;
2433 AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
2434
2435 err = IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
2436 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2437 // --
2438
2439 HIBLOG("hibernate_machine_init reading\n");
2440
2441 uint32_t * header = (uint32_t *) src;
2442 sum = 0;
2443
2444 while (kIOReturnSuccess == err)
2445 {
2446 unsigned int count;
2447 unsigned int page;
2448 uint32_t tag;
2449 vm_offset_t ppnum, compressedSize;
2450
2451 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2452 if (kIOReturnSuccess != err)
2453 break;
2454
2455 ppnum = header[0];
2456 count = header[1];
2457
2458 // HIBPRINT("(%x, %x)\n", ppnum, count);
2459
2460 if (!count)
2461 break;
2462
2463 for (page = 0; page < count; page++)
2464 {
2465 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2466 if (kIOReturnSuccess != err)
2467 break;
2468
2469 compressedSize = kIOHibernateTagLength & tag;
2470 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength))
2471 {
2472 err = kIOReturnIPCError;
2473 break;
2474 }
2475
2476 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2477 if (kIOReturnSuccess != err) break;
2478
2479 if (compressedSize < page_size)
2480 {
2481 decoOffset = page_size;
2482 clock_get_uptime(&startTime);
2483
2484 if (compressedSize == 4) {
2485 int i;
2486 uint32_t *s, *d;
2487
2488 s = (uint32_t *)src;
2489 d = (uint32_t *)(uintptr_t)compressed;
2490
2491 for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++)
2492 *d++ = *s;
2493 }
2494 else
2495 WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, compressedSize);
2496 clock_get_uptime(&endTime);
2497 ADD_ABSOLUTETIME(&compTime, &endTime);
2498 SUB_ABSOLUTETIME(&compTime, &startTime);
2499 compBytes += page_size;
2500 }
2501 else decoOffset = 0;
2502
2503 sum += hibernate_sum_page((src + decoOffset), ppnum);
2504 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2505 if (err)
2506 {
2507 HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
2508 break;
2509 }
2510
2511 ppnum++;
2512 pagesDone++;
2513 pagesRead++;
2514
2515 if (0 == (8191 & pagesDone))
2516 {
2517 clock_get_uptime(&endTime);
2518 SUB_ABSOLUTETIME(&endTime, &allTime);
2519 absolutetime_to_nanoseconds(endTime, &nsec);
2520 progressStamp = nsec / 750000000ULL;
2521 if (progressStamp != lastProgressStamp)
2522 {
2523 lastProgressStamp = progressStamp;
2524 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2525 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2526 }
2527 }
2528 }
2529 }
2530 if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages))
2531 err = kIOReturnLockedRead;
2532
2533 if (kIOReturnSuccess != err)
2534 panic("Hibernate restore error %x", err);
2535
2536 gIOHibernateCurrentHeader->actualImage2Sum = sum;
2537 gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
2538
2539 clock_get_uptime(&endIOTime);
2540
2541 err = IOPolledFilePollersClose(vars->fileVars, kIOPolledAfterSleepState);
2542
2543 clock_get_uptime(&endTime);
2544
2545 IOService::getPMRootDomain()->pmStatsRecordEvent(
2546 kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
2547 IOService::getPMRootDomain()->pmStatsRecordEvent(
2548 kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
2549
2550 SUB_ABSOLUTETIME(&endTime, &allTime);
2551 absolutetime_to_nanoseconds(endTime, &nsec);
2552
2553 SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
2554 absolutetime_to_nanoseconds(endIOTime, &nsecIO);
2555
2556 gIOHibernateStats->kernelImageReadDuration = nsec / 1000000ULL;
2557 gIOHibernateStats->imagePages = pagesDone;
2558
2559 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
2560 pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
2561 nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
2562
2563 absolutetime_to_nanoseconds(compTime, &nsec);
2564 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2565 compBytes,
2566 nsec / 1000000ULL,
2567 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2568
2569 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2570 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
2571 vars->fileVars->cryptBytes,
2572 nsec / 1000000ULL,
2573 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2574
2575 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 2), pagesRead, pagesDone);
2576 }
2577
2578 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2579
2580 void IOHibernateSetWakeCapabilities(uint32_t capability)
2581 {
2582 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
2583 {
2584 gIOHibernateStats->wakeCapability = capability;
2585
2586 if (kIOPMSystemCapabilityGraphics & capability)
2587 {
2588 vm_compressor_do_warmup();
2589 }
2590 }
2591 }
2592
2593 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2594
2595 void IOHibernateSystemRestart(void)
2596 {
2597 static uint8_t noteStore[32] __attribute__((aligned(32)));
2598 IORegistryEntry * regEntry;
2599 const OSSymbol * sym;
2600 OSData * noteProp;
2601 OSData * data;
2602 uintptr_t * smcVars;
2603 uint8_t * smcBytes;
2604 size_t len;
2605 addr64_t element;
2606
2607 data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
2608 if (!data) return;
2609
2610 smcVars = (typeof(smcVars)) data->getBytesNoCopy();
2611 smcBytes = (typeof(smcBytes)) smcVars[1];
2612 len = smcVars[0];
2613 if (len > sizeof(noteStore)) len = sizeof(noteStore);
2614 noteProp = OSData::withCapacity(3 * sizeof(element));
2615 if (!noteProp) return;
2616 element = len;
2617 noteProp->appendBytes(&element, sizeof(element));
2618 element = crc32(0, smcBytes, len);
2619 noteProp->appendBytes(&element, sizeof(element));
2620
2621 bcopy(smcBytes, noteStore, len);
2622 element = (addr64_t) &noteStore[0];
2623 element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
2624 noteProp->appendBytes(&element, sizeof(element));
2625
2626 if (!gIOOptionsEntry)
2627 {
2628 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
2629 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
2630 if (regEntry && !gIOOptionsEntry)
2631 regEntry->release();
2632 }
2633
2634 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
2635 if (gIOOptionsEntry && sym) gIOOptionsEntry->setProperty(sym, noteProp);
2636 if (noteProp) noteProp->release();
2637 if (sym) sym->release();
2638 }
mirror of XNU