]> git.saurik.com Git - apple/xnu.git/blob - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blob
? search:


4ceb8cc0f8d2d3bceaa05713a198106ad4770732
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
1 /*
2 * Copyright (c) 2004-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30
31 Sleep:
32
33 - PMRootDomain calls IOHibernateSystemSleep() before system sleep
34 (devices awake, normal execution context)
35 - IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
36 grabs its extents and searches for a polling driver willing to work with that IOMedia.
37 The BSD code makes an ioctl to the storage driver to get the partition base offset to
38 the disk, and other ioctls to get the transfer constraints
39 If successful, the file is written to make sure its initially not bootable (in case of
40 later failure) and nvram set to point to the first block of the file. (Has to be done
41 here so blocking is possible in nvram support).
42 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
43 page out any pages it wants to (currently zero, but probably some percentage of memory).
44 Its assumed just allocating pages will cause the VM system to naturally select the best
45 pages for eviction. It also copies processor flags needed for the restore path and sets
46 a flag in the boot processor proc info.
47 gIOHibernateState = kIOHibernateStateHibernating.
48 - Regular sleep progresses - some drivers may inspect the root domain property
49 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
50 as usual but leaves motherboard I/O on.
51 - Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
52 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
53 all ppc RC bits out of the hash table and caches into the mapping structures.
54 - hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
55 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
56 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
57 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
58 The image header and block list are written. The header includes the second file extent so
59 only the header block is needed to read the file, regardless of filesystem.
60 The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
61 (only) is used to decompress the image during wake/boot.
62 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
63 The bitmaps are written to the image.
64 More areas are removed from the bitmaps (after they have been written to the image) - the
65 segment "__HIB" pages and interrupt stack.
66 Each wired page is compressed and written and then each non-wired page. Compression and
67 disk writes are in parallel.
68 The image header is written to the start of the file and the polling driver closed.
69 The machine powers down (or sleeps).
70
71 Boot/Wake:
72
73 - BootX sees the boot-image nvram variable containing the device and block number of the image,
74 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
75 - BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
76 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
77 that is in the kernel's __HIB section.
78 - BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
79 hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
80 only code & data in that section is safe to call since all the other wired pages are still
81 compressed in the image.
82 - hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
83 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
84 location directly, and copies those that can't to interim free pages. When the image has been
85 completed, the copies are uncompressed, overwriting the wired image pages.
86 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
87 is used to get pages into place for 64bit.
88 - the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
89 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
90 are removed from the software strutures, and the hash table is reinitialized.
91 - After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
92 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
93 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
94 for the remaining non wired pages.
95 - The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
96 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
97 is closed via bsd.
98
99 Polled Mode I/O:
100
101 IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
102 registry, specifying an object of calls IOPolledInterface.
103
104 Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
105 partition) that the image is going to live, looking for polled interface properties. If it finds
106 one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
107 interfaces found are kept in an ordered list.
108
109 There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
110 few different contexts things happen in:
111
112 - there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
113 up and running) and after wake - this is safe to allocate memory and do anything. The device
114 ignores sleep requests from that point since its a waste of time if it goes to sleep and
115 immediately wakes back up for the image write.
116
117 - there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
118 immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
119 from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
120 used to flush and set the disk to sleep.
121
122 - there is an Open/Close (AfterSleep) pair made around the image read operations that happen
123 immediately after sleep. These can't block or allocate memory. This is happening after the platform
124 expert has woken the low level bits of the system, but most of the I/O system has not. There is only
125 one thread running.
126
127 For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
128 (prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
129 the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
130 Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
131 that is called for the hardware to check for events, and complete the I/O via the callback.
132 The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
133 to restrict I/O ops.
134 */
135
136 #include <sys/systm.h>
137
138 #include <IOKit/IOWorkLoop.h>
139 #include <IOKit/IOCommandGate.h>
140 #include <IOKit/IOTimerEventSource.h>
141 #include <IOKit/IOPlatformExpert.h>
142 #include <IOKit/IOKitDebug.h>
143 #include <IOKit/IOTimeStamp.h>
144 #include <IOKit/pwr_mgt/RootDomain.h>
145 #include <IOKit/pwr_mgt/IOPMPrivate.h>
146 #include <IOKit/IOMessage.h>
147 #include <IOKit/IODeviceTreeSupport.h>
148 #include <IOKit/IOBSD.h>
149 #include "RootDomainUserClient.h"
150 #include <IOKit/pwr_mgt/IOPowerConnection.h>
151 #include "IOPMPowerStateQueue.h"
152 #include <IOKit/IOBufferMemoryDescriptor.h>
153 #include <IOKit/AppleKeyStoreInterface.h>
154 #include <libkern/crypto/aes.h>
155
156 #include <sys/uio.h>
157 #include <sys/conf.h>
158 #include <sys/stat.h>
159 #include <sys/fcntl.h> // (FWRITE, ...)
160 #include <sys/sysctl.h>
161 #include <sys/kdebug.h>
162 #include <stdint.h>
163
164 #include <IOKit/IOHibernatePrivate.h>
165 #include <IOKit/IOPolledInterface.h>
166 #include <IOKit/IONVRAM.h>
167 #include "IOHibernateInternal.h"
168 #include <vm/WKdm_new.h>
169 #include <vm/vm_protos.h>
170 #include "IOKitKernelInternal.h"
171 #include <pexpert/device_tree.h>
172
173 #include <machine/pal_routines.h>
174 #include <machine/pal_hibernate.h>
175 #include <i386/tsc.h>
176 #include <i386/cpuid.h>
177 #include <san/kasan.h>
178
179 extern "C" addr64_t kvtophys(vm_offset_t va);
180 extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
181
182 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
183
184 #define DISABLE_TRIM 0
185 #define TRIM_DELAY 25000
186
187 extern unsigned int save_kdebug_enable;
188 extern uint32_t gIOHibernateState;
189 uint32_t gIOHibernateMode;
190 static char gIOHibernateBootSignature[256+1];
191 static char gIOHibernateFilename[MAXPATHLEN+1];
192 static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
193 uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
194 static uint64_t gIOHibernateCompression = 0x80; // default compression 50%
195 boolean_t gIOHibernateStandbyDisabled;
196
197 static IODTNVRAM * gIOOptionsEntry;
198 static IORegistryEntry * gIOChosenEntry;
199
200 static const OSSymbol * gIOHibernateBootImageKey;
201
202 #if defined(__i386__) || defined(__x86_64__)
203
204 static const OSSymbol * gIOHibernateRTCVariablesKey;
205 static const OSSymbol * gIOHibernateBoot0082Key;
206 static const OSSymbol * gIOHibernateBootNextKey;
207 static OSData * gIOHibernateBoot0082Data;
208 static OSData * gIOHibernateBootNextData;
209 static OSObject * gIOHibernateBootNextSave;
210
211 static IOPolledFileIOVars * gDebugImageFileVars;
212 static IOLock * gDebugImageLock;
213
214 #endif /* defined(__i386__) || defined(__x86_64__) */
215
216 static IOLock * gFSLock;
217 uint32_t gFSState;
218 static thread_call_t gIOHibernateTrimCalloutEntry;
219 static IOPolledFileIOVars gFileVars;
220 static IOHibernateVars gIOHibernateVars;
221 static IOPolledFileCryptVars gIOHibernateCryptWakeContext;
222 static hibernate_graphics_t _hibernateGraphics;
223 static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
224 static hibernate_statistics_t _hibernateStats;
225 static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats;
226
227 enum
228 {
229 kFSIdle = 0,
230 kFSOpening = 2,
231 kFSOpened = 3,
232 kFSTimedOut = 4,
233 kFSTrimDelay = 5
234 };
235
236 static IOReturn IOHibernateDone(IOHibernateVars * vars);
237 static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature);
238 static void IOSetBootImageNVRAM(OSData * data);
239 static void IOHibernateSystemPostWakeTrim(void * p1, void * p2);
240
241 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
242
243 enum { kDefaultIOSize = 128 * 1024 };
244 enum { kVideoMapSize = 80 * 1024 * 1024 };
245
246 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
247
248 // copy from phys addr to MD
249
250 static IOReturn
251 IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
252 IOByteCount offset, addr64_t bytes, IOByteCount length)
253 {
254 addr64_t srcAddr = bytes;
255 IOByteCount remaining;
256
257 remaining = length = min(length, md->getLength() - offset);
258 while (remaining) { // (process another target segment?)
259 addr64_t dstAddr64;
260 IOByteCount dstLen;
261
262 dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
263 if (!dstAddr64)
264 break;
265
266 // Clip segment length to remaining
267 if (dstLen > remaining)
268 dstLen = remaining;
269
270 #if 1
271 bcopy_phys(srcAddr, dstAddr64, dstLen);
272 #else
273 copypv(srcAddr, dstAddr64, dstLen,
274 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
275 #endif
276 srcAddr += dstLen;
277 offset += dstLen;
278 remaining -= dstLen;
279 }
280
281 assert(!remaining);
282
283 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
284 }
285
286 // copy from MD to phys addr
287
288 static IOReturn
289 IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
290 IOByteCount offset, addr64_t bytes, IOByteCount length)
291 {
292 addr64_t dstAddr = bytes;
293 IOByteCount remaining;
294
295 remaining = length = min(length, md->getLength() - offset);
296 while (remaining) { // (process another target segment?)
297 addr64_t srcAddr64;
298 IOByteCount dstLen;
299
300 srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
301 if (!srcAddr64)
302 break;
303
304 // Clip segment length to remaining
305 if (dstLen > remaining)
306 dstLen = remaining;
307
308 #if 1
309 bcopy_phys(srcAddr64, dstAddr, dstLen);
310 #else
311 copypv(srcAddr, dstAddr64, dstLen,
312 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
313 #endif
314 dstAddr += dstLen;
315 offset += dstLen;
316 remaining -= dstLen;
317 }
318
319 assert(!remaining);
320
321 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
322 }
323
324 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
325
326 void
327 hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
328 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
329 {
330 count += ppnum;
331 switch (kind)
332 {
333 case kIOHibernatePageStateUnwiredSave:
334 // unwired save
335 for (; ppnum < count; ppnum++)
336 {
337 hibernate_page_bitset(page_list, FALSE, ppnum);
338 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
339 }
340 break;
341 case kIOHibernatePageStateWiredSave:
342 // wired save
343 for (; ppnum < count; ppnum++)
344 {
345 hibernate_page_bitset(page_list, FALSE, ppnum);
346 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
347 }
348 break;
349 case kIOHibernatePageStateFree:
350 // free page
351 for (; ppnum < count; ppnum++)
352 {
353 hibernate_page_bitset(page_list, TRUE, ppnum);
354 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
355 }
356 break;
357 default:
358 panic("hibernate_set_page_state");
359 }
360 }
361
362 static vm_offset_t
363 hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
364 {
365 uint32_t page = *pPage;
366 uint32_t count;
367 hibernate_bitmap_t * bitmap;
368
369 while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
370 {
371 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
372 if (!count)
373 break;
374 page += count;
375 if (page <= bitmap->last_page)
376 break;
377 }
378
379 *pPage = page;
380 if (bitmap)
381 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
382 else
383 count = 0;
384
385 return (count);
386 }
387
388 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
389
390 IOReturn
391 IOHibernateSystemSleep(void)
392 {
393 IOReturn err;
394 OSData * nvramData;
395 OSObject * obj;
396 OSString * str;
397 OSNumber * num;
398 bool dsSSD, vmflush, swapPinned;
399 IOHibernateVars * vars;
400 uint64_t setFileSize = 0;
401
402 gIOHibernateState = kIOHibernateStateInactive;
403
404 gIOHibernateDebugFlags = 0;
405 if (kIOLogHibernate & gIOKitDebug)
406 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
407
408 if (IOService::getPMRootDomain()->getHibernateSettings(
409 &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime))
410 {
411 if (kIOHibernateModeSleep & gIOHibernateMode)
412 // default to discard clean for safe sleep
413 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
414 | kIOHibernateModeDiscardCleanActive);
415 }
416
417 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
418 {
419 if ((str = OSDynamicCast(OSString, obj)))
420 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
421 sizeof(gIOHibernateFilename));
422 obj->release();
423 }
424
425 if (!gIOHibernateMode || !gIOHibernateFilename[0])
426 return (kIOReturnUnsupported);
427
428 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
429
430 vars = IONew(IOHibernateVars, 1);
431 if (!vars) return (kIOReturnNoMemory);
432 bzero(vars, sizeof(*vars));
433
434 IOLockLock(gFSLock);
435 if (!gIOHibernateTrimCalloutEntry)
436 {
437 gIOHibernateTrimCalloutEntry = thread_call_allocate(&IOHibernateSystemPostWakeTrim, &gFSLock);
438 }
439 IOHibernateSystemPostWakeTrim(NULL, NULL);
440 thread_call_cancel(gIOHibernateTrimCalloutEntry);
441 if (kFSIdle != gFSState)
442 {
443 HIBLOG("hibernate file busy\n");
444 IOLockUnlock(gFSLock);
445 IODelete(vars, IOHibernateVars, 1);
446 return (kIOReturnBusy);
447 }
448 gFSState = kFSOpening;
449 IOLockUnlock(gFSLock);
450
451 swapPinned = false;
452 do
453 {
454 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
455 2 * page_size + WKdm_SCRATCH_BUF_SIZE_INTERNAL, page_size);
456
457 vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
458 ptoa_64(gIOHibernateHandoffPageCount), page_size);
459
460 if (!vars->srcBuffer || !vars->handoffBuffer)
461 {
462 err = kIOReturnNoMemory;
463 break;
464 }
465
466 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey)))
467 {
468 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMinSize = num->unsigned64BitValue();
469 obj->release();
470 }
471 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey)))
472 {
473 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMaxSize = num->unsigned64BitValue();
474 obj->release();
475 }
476
477 boolean_t encryptedswap = true;
478 uint32_t pageCount;
479 AbsoluteTime startTime, endTime;
480 uint64_t nsec;
481
482 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
483 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
484 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
485
486 vmflush = ((kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
487 err = hibernate_alloc_page_lists(&vars->page_list,
488 &vars->page_list_wired,
489 &vars->page_list_pal);
490 if (KERN_SUCCESS != err) break;
491
492 err = hibernate_pin_swap(TRUE);
493 if (KERN_SUCCESS != err) break;
494 swapPinned = true;
495
496 if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode))
497 {
498 hibernate_page_list_setall(vars->page_list,
499 vars->page_list_wired,
500 vars->page_list_pal,
501 true /* preflight */,
502 vmflush /* discard */,
503 &pageCount);
504 PE_Video consoleInfo;
505 bzero(&consoleInfo, sizeof(consoleInfo));
506 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
507
508 // estimate: 6% increase in pages compressed
509 // screen preview 2 images compressed 0%
510 setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
511 + vars->page_list->list_size
512 + (consoleInfo.v_width * consoleInfo.v_height * 8);
513 enum { setFileRound = 1024*1024ULL };
514 setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
515
516 HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
517 pageCount, (100ULL * gIOHibernateCompression) >> 8,
518 setFileSize, vars->fileMinSize);
519
520 if (!(kIOHibernateModeFileResize & gIOHibernateMode)
521 && (setFileSize < vars->fileMinSize))
522 {
523 setFileSize = vars->fileMinSize;
524 }
525 }
526
527 // Invalidate the image file
528 if (gDebugImageLock) {
529 IOLockLock(gDebugImageLock);
530 if (gDebugImageFileVars != 0) {
531 IOSetBootImageNVRAM(0);
532 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
533 }
534 IOLockUnlock(gDebugImageLock);
535 }
536
537 err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, 0,
538 gIOHibernateCurrentHeader, sizeof(gIOHibernateCurrentHeader),
539 &vars->fileVars, &nvramData,
540 &vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
541
542 if (KERN_SUCCESS != err)
543 {
544 IOLockLock(gFSLock);
545 if (kFSOpening != gFSState) err = kIOReturnTimeout;
546 IOLockUnlock(gFSLock);
547 }
548
549 if (KERN_SUCCESS != err)
550 {
551 HIBLOG("IOPolledFileOpen(%x)\n", err);
552 break;
553 }
554
555 // write extents for debug data usage in EFI
556 IOWriteExtentsToFile(vars->fileVars, kIOHibernateHeaderOpenSignature);
557
558 err = IOPolledFilePollersSetup(vars->fileVars, kIOPolledPreflightState);
559 if (KERN_SUCCESS != err) break;
560
561 clock_get_uptime(&startTime);
562 err = hibernate_setup(gIOHibernateCurrentHeader,
563 vmflush,
564 vars->page_list, vars->page_list_wired, vars->page_list_pal);
565 clock_get_uptime(&endTime);
566 SUB_ABSOLUTETIME(&endTime, &startTime);
567 absolutetime_to_nanoseconds(endTime, &nsec);
568
569 boolean_t haveSwapPin, hibFileSSD;
570 haveSwapPin = vm_swap_files_pinned();
571
572 hibFileSSD = (kIOPolledFileSSD & vars->fileVars->flags);
573
574 HIBLOG("hibernate_setup(%d) took %qd ms, swapPin(%d) ssd(%d)\n",
575 err, nsec / 1000000ULL,
576 haveSwapPin, hibFileSSD);
577 if (KERN_SUCCESS != err) break;
578
579 gIOHibernateStandbyDisabled = ((!haveSwapPin || !hibFileSSD));
580
581 dsSSD = ((0 != (kIOPolledFileSSD & vars->fileVars->flags))
582 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
583
584 if (dsSSD) gIOHibernateCurrentHeader->options |= kIOHibernateOptionSSD | kIOHibernateOptionColor;
585 else gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
586
587
588 #if defined(__i386__) || defined(__x86_64__)
589 if (!uuid_is_null(vars->volumeCryptKey) &&
590 (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey)))
591 {
592 uintptr_t smcVars[2];
593 smcVars[0] = sizeof(vars->volumeCryptKey);
594 smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
595
596 IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
597 bzero(smcVars, sizeof(smcVars));
598 }
599 #endif
600
601
602 if (encryptedswap || !uuid_is_null(vars->volumeCryptKey))
603 gIOHibernateMode ^= kIOHibernateModeEncrypt;
604
605 if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
606 {
607 vars->videoAllocSize = kVideoMapSize;
608 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize, VM_KERN_MEMORY_IOKIT))
609 vars->videoMapping = 0;
610 }
611
612 // generate crypt keys
613 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
614 vars->wiredCryptKey[i] = random();
615 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
616 vars->cryptKey[i] = random();
617
618 // set nvram
619
620 IOSetBootImageNVRAM(nvramData);
621 nvramData->release();
622
623 #if defined(__i386__) || defined(__x86_64__)
624 {
625 struct AppleRTCHibernateVars
626 {
627 uint8_t signature[4];
628 uint32_t revision;
629 uint8_t booterSignature[20];
630 uint8_t wiredCryptKey[16];
631 };
632 AppleRTCHibernateVars rtcVars;
633 OSData * data;
634
635 rtcVars.signature[0] = 'A';
636 rtcVars.signature[1] = 'A';
637 rtcVars.signature[2] = 'P';
638 rtcVars.signature[3] = 'L';
639 rtcVars.revision = 1;
640 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
641
642 if (gIOChosenEntry
643 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateBootSignatureKey)))
644 && (sizeof(rtcVars.booterSignature) <= data->getLength()))
645 {
646 bcopy(data->getBytesNoCopy(), &rtcVars.booterSignature[0], sizeof(rtcVars.booterSignature));
647 }
648 else if (gIOHibernateBootSignature[0])
649 {
650 char c;
651 uint8_t value = 0;
652 uint32_t in, out, digits;
653 for (in = out = digits = 0;
654 (c = gIOHibernateBootSignature[in]) && (in < sizeof(gIOHibernateBootSignature));
655 in++)
656 {
657 if ((c >= 'a') && (c <= 'f')) c -= 'a' - 10;
658 else if ((c >= 'A') && (c <= 'F')) c -= 'A' - 10;
659 else if ((c >= '0') && (c <= '9')) c -= '0';
660 else
661 {
662 if (c == '=') out = digits = value = 0;
663 continue;
664 }
665 value = (value << 4) | c;
666 if (digits & 1)
667 {
668 rtcVars.booterSignature[out++] = value;
669 if (out >= sizeof(rtcVars.booterSignature)) break;
670 }
671 digits++;
672 }
673 }
674 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
675 if (data)
676 {
677 if (gIOHibernateRTCVariablesKey)
678 IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
679 data->release();
680 }
681 if (gIOChosenEntry && gIOOptionsEntry)
682 {
683 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
684 if (data) gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
685 // set BootNext
686 if (!gIOHibernateBoot0082Data)
687 {
688 OSData * fileData = 0;
689 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
690 if (data->getLength() >= 4) fileData = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-file-path"));
691 if (data)
692 {
693 // AppleNVRAM_EFI_LOAD_OPTION
694 struct {
695 uint32_t Attributes;
696 uint16_t FilePathLength;
697 uint16_t Desc;
698 } loadOptionHeader;
699 loadOptionHeader.Attributes = 1;
700 loadOptionHeader.FilePathLength = data->getLength();
701 loadOptionHeader.Desc = 0;
702 if (fileData)
703 {
704 loadOptionHeader.FilePathLength -= 4;
705 loadOptionHeader.FilePathLength += fileData->getLength();
706 }
707 gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
708 if (gIOHibernateBoot0082Data)
709 {
710 gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
711 if (fileData)
712 {
713 gIOHibernateBoot0082Data->appendBytes(data->getBytesNoCopy(), data->getLength() - 4);
714 gIOHibernateBoot0082Data->appendBytes(fileData);
715 }
716 else gIOHibernateBoot0082Data->appendBytes(data);
717 }
718 }
719 }
720 if (!gIOHibernateBootNextData)
721 {
722 uint16_t bits = 0x0082;
723 gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
724 }
725 if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData)
726 {
727 gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
728 gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
729 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
730 }
731 // BootNext
732 }
733 }
734 #endif /* !i386 && !x86_64 */
735 }
736 while (false);
737
738 if (swapPinned) hibernate_pin_swap(FALSE);
739
740 IOLockLock(gFSLock);
741 if ((kIOReturnSuccess == err) && (kFSOpening != gFSState))
742 {
743 HIBLOG("hibernate file close due timeout\n");
744 err = kIOReturnTimeout;
745 }
746 if (kIOReturnSuccess == err)
747 {
748 gFSState = kFSOpened;
749 gIOHibernateVars = *vars;
750 gFileVars = *vars->fileVars;
751 gFileVars.allocated = false;
752 gIOHibernateVars.fileVars = &gFileVars;
753 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
754 gIOHibernateState = kIOHibernateStateHibernating;
755 }
756 else
757 {
758 IOPolledFileIOVars * fileVars = vars->fileVars;
759 IOHibernateDone(vars);
760 IOPolledFileClose(&fileVars,
761 #if DISABLE_TRIM
762 0, NULL, 0, 0, 0);
763 #else
764 0, NULL, 0, sizeof(IOHibernateImageHeader), setFileSize);
765 #endif
766 gFSState = kFSIdle;
767 }
768 IOLockUnlock(gFSLock);
769
770 if (vars->fileVars) IODelete(vars->fileVars, IOPolledFileIOVars, 1);
771 IODelete(vars, IOHibernateVars, 1);
772
773 return (err);
774 }
775
776 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
777
778 static void
779 IOSetBootImageNVRAM(OSData * data)
780 {
781 IORegistryEntry * regEntry;
782
783 if (!gIOOptionsEntry)
784 {
785 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
786 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
787 if (regEntry && !gIOOptionsEntry)
788 regEntry->release();
789 }
790 if (gIOOptionsEntry && gIOHibernateBootImageKey)
791 {
792 if (data) gIOOptionsEntry->setProperty(gIOHibernateBootImageKey, data);
793 else
794 {
795 gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
796 gIOOptionsEntry->sync();
797 }
798 }
799 }
800
801 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
802 /*
803 * Writes header to disk with signature, block size and file extents data.
804 * If there are more than 2 extents, then they are written on second block.
805 */
806 static IOReturn
807 IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature)
808 {
809 IOHibernateImageHeader hdr;
810 IOItemCount count;
811 IOReturn err = kIOReturnSuccess;
812 int rc;
813 IOPolledFileExtent * fileExtents;
814
815 fileExtents = (typeof(fileExtents)) vars->fileExtents->getBytesNoCopy();
816
817 memset(&hdr, 0, sizeof(IOHibernateImageHeader));
818 count = vars->fileExtents->getLength();
819 if (count > sizeof(hdr.fileExtentMap))
820 {
821 hdr.fileExtentMapSize = count;
822 count = sizeof(hdr.fileExtentMap);
823 }
824 else
825 hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap);
826
827 bcopy(fileExtents, &hdr.fileExtentMap[0], count);
828
829 // copy file block extent list if larger than header
830 if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap))
831 {
832 count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap);
833 rc = kern_write_file(vars->fileRef, vars->blockSize,
834 (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)),
835 count, IO_SKIP_ENCRYPTION);
836 if (rc != 0) {
837 HIBLOG("kern_write_file returned %d\n", rc);
838 err = kIOReturnIOError;
839 goto exit;
840 }
841 }
842 hdr.signature = signature;
843 hdr.deviceBlockSize = vars->blockSize;
844
845 rc = kern_write_file(vars->fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION);
846 if (rc != 0) {
847 HIBLOG("kern_write_file returned %d\n", rc);
848 err = kIOReturnIOError;
849 goto exit;
850 }
851
852 exit:
853 return err;
854 }
855
856 extern "C" boolean_t root_is_CF_drive;
857
858 void
859 IOOpenDebugDataFile(const char *fname, uint64_t size)
860 {
861 IOReturn err;
862 OSData * imagePath = NULL;
863 uint64_t padding;
864
865 if (!gDebugImageLock) {
866 gDebugImageLock = IOLockAlloc();
867 }
868
869 if (root_is_CF_drive) return;
870
871 // Try to get a lock, but don't block for getting lock
872 if (!IOLockTryLock(gDebugImageLock)) {
873 HIBLOG("IOOpenDebugDataFile: Failed to get lock\n");
874 return;
875 }
876
877 if (gDebugImageFileVars || !fname || !size) {
878 HIBLOG("IOOpenDebugDataFile: conditions failed\n");
879 goto exit;
880 }
881
882 padding = (PAGE_SIZE*2); // allocate couple more pages for header and fileextents
883 err = IOPolledFileOpen(fname, size+padding, 32ULL*1024*1024*1024,
884 NULL, 0,
885 &gDebugImageFileVars, &imagePath, NULL, 0);
886
887 if ((kIOReturnSuccess == err) && imagePath)
888 {
889 if ((gDebugImageFileVars->fileSize < (size+padding)) ||
890 (gDebugImageFileVars->fileExtents->getLength() > PAGE_SIZE)) {
891 // Can't use the file
892 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
893 HIBLOG("IOOpenDebugDataFile: too many file extents\n");
894 goto exit;
895 }
896
897 // write extents for debug data usage in EFI
898 IOWriteExtentsToFile(gDebugImageFileVars, kIOHibernateHeaderOpenSignature);
899 IOSetBootImageNVRAM(imagePath);
900 }
901
902 exit:
903 IOLockUnlock(gDebugImageLock);
904
905 if (imagePath) imagePath->release();
906 return;
907 }
908
909 void
910 IOCloseDebugDataFile()
911 {
912 IOSetBootImageNVRAM(0);
913
914 if (gDebugImageLock) {
915 IOLockLock(gDebugImageLock);
916 if (gDebugImageFileVars != 0) {
917 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
918 }
919 IOLockUnlock(gDebugImageLock);
920 }
921
922
923 }
924
925 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
926
927 DECLARE_IOHIBERNATEPROGRESSALPHA
928
929 static void
930 ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
931 {
932 uint32_t rowBytes, pixelShift;
933 uint32_t x, y;
934 int32_t blob;
935 uint32_t alpha, in, color, result;
936 uint8_t * out;
937 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
938
939 rowBytes = display->rowBytes;
940 pixelShift = display->depth >> 4;
941 if (pixelShift < 1) return;
942
943 screen += ((display->width
944 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
945 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
946
947 for (y = 0; y < kIOHibernateProgressHeight; y++)
948 {
949 out = screen + y * rowBytes;
950 for (blob = 0; blob < kIOHibernateProgressCount; blob++)
951 {
952 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
953 for (x = 0; x < kIOHibernateProgressWidth; x++)
954 {
955 alpha = gIOHibernateProgressAlpha[y][x];
956 result = color;
957 if (alpha)
958 {
959 if (0xff != alpha)
960 {
961 if (1 == pixelShift)
962 {
963 in = *((uint16_t *)out) & 0x1f; // 16
964 in = (in << 3) | (in >> 2);
965 }
966 else
967 in = *((uint32_t *)out) & 0xff; // 32
968 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
969 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
970 }
971 if (1 == pixelShift)
972 {
973 result >>= 3;
974 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
975 }
976 else
977 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
978 }
979 out += (1 << pixelShift);
980 }
981 out += (kIOHibernateProgressSpacing << pixelShift);
982 }
983 }
984 }
985
986
987 static void
988 ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
989 {
990 uint32_t rowBytes, pixelShift;
991 uint32_t x, y;
992 int32_t blob, lastBlob;
993 uint32_t alpha, in, color, result;
994 uint8_t * out;
995 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
996
997 pixelShift = display->depth >> 4;
998 if (pixelShift < 1)
999 return;
1000
1001 rowBytes = display->rowBytes;
1002
1003 screen += ((display->width
1004 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1005 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1006
1007 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
1008
1009 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
1010
1011 for (y = 0; y < kIOHibernateProgressHeight; y++)
1012 {
1013 out = screen + y * rowBytes;
1014 for (blob = firstBlob; blob <= lastBlob; blob++)
1015 {
1016 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
1017 for (x = 0; x < kIOHibernateProgressWidth; x++)
1018 {
1019 alpha = gIOHibernateProgressAlpha[y][x];
1020 result = color;
1021 if (alpha)
1022 {
1023 if (0xff != alpha)
1024 {
1025 in = display->progressSaveUnder[blob][saveindex[blob]++];
1026 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
1027 }
1028 if (1 == pixelShift)
1029 {
1030 result >>= 3;
1031 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1032 }
1033 else
1034 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1035 }
1036 out += (1 << pixelShift);
1037 }
1038 out += (kIOHibernateProgressSpacing << pixelShift);
1039 }
1040 }
1041 }
1042
1043 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1044
1045 IOReturn
1046 IOHibernateIOKitSleep(void)
1047 {
1048 IOReturn ret = kIOReturnSuccess;
1049 IOLockLock(gFSLock);
1050 if (kFSOpening == gFSState)
1051 {
1052 gFSState = kFSTimedOut;
1053 HIBLOG("hibernate file open timed out\n");
1054 ret = kIOReturnTimeout;
1055 }
1056 IOLockUnlock(gFSLock);
1057 return (ret);
1058 }
1059
1060 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1061
1062 IOReturn
1063 IOHibernateSystemHasSlept(void)
1064 {
1065 IOReturn ret = kIOReturnSuccess;
1066 IOHibernateVars * vars = &gIOHibernateVars;
1067 OSObject * obj = 0;
1068 OSData * data;
1069
1070 IOLockLock(gFSLock);
1071 if ((kFSOpened != gFSState) && gIOHibernateMode)
1072 {
1073 ret = kIOReturnTimeout;
1074 }
1075 IOLockUnlock(gFSLock);
1076 if (kIOReturnSuccess != ret) return (ret);
1077
1078 if (gIOHibernateMode) obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
1079 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1080 if (obj && !vars->previewBuffer)
1081 obj->release();
1082
1083 vars->consoleMapping = NULL;
1084 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
1085 {
1086 vars->previewBuffer->release();
1087 vars->previewBuffer = 0;
1088 }
1089
1090 if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1091 && vars->previewBuffer
1092 && (data = OSDynamicCast(OSData,
1093 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
1094 {
1095 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
1096 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
1097
1098 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1099
1100 if (kIOHibernatePreviewUpdates & flags)
1101 {
1102 PE_Video consoleInfo;
1103 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1104
1105 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1106
1107 graphicsInfo->width = consoleInfo.v_width;
1108 graphicsInfo->height = consoleInfo.v_height;
1109 graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
1110 graphicsInfo->depth = consoleInfo.v_depth;
1111 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1112
1113 HIBPRINT("video %p %d %d %d\n",
1114 vars->consoleMapping, graphicsInfo->depth,
1115 graphicsInfo->width, graphicsInfo->height);
1116 if (vars->consoleMapping)
1117 ProgressInit(graphicsInfo, vars->consoleMapping,
1118 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1119 }
1120 }
1121
1122 if (gIOOptionsEntry)
1123 gIOOptionsEntry->sync();
1124
1125 return (ret);
1126 }
1127
1128 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1129
1130 static DeviceTreeNode *
1131 MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry)
1132 {
1133 DeviceTreeNodeProperty * prop;
1134 DeviceTreeNode * child;
1135 IORegistryEntry * childRegEntry;
1136 const char * nameProp;
1137 unsigned int propLen, idx;
1138
1139 prop = (DeviceTreeNodeProperty *) (entry + 1);
1140 for (idx = 0; idx < entry->nProperties; idx++)
1141 {
1142 if (regEntry && (0 != strcmp("name", prop->name)))
1143 {
1144 regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
1145 // HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
1146 }
1147 prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
1148 }
1149
1150 child = (DeviceTreeNode *) prop;
1151 for (idx = 0; idx < entry->nChildren; idx++)
1152 {
1153 if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen))
1154 panic("no name");
1155 childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
1156 // HIBPRINT("%s == %p\n", nameProp, childRegEntry);
1157 child = MergeDeviceTree(child, childRegEntry);
1158 }
1159 return (child);
1160 }
1161
1162 IOReturn
1163 IOHibernateSystemWake(void)
1164 {
1165 if (kFSOpened == gFSState)
1166 {
1167 IOPolledFilePollersClose(gIOHibernateVars.fileVars, kIOPolledPostflightState);
1168 IOHibernateDone(&gIOHibernateVars);
1169 }
1170 else
1171 {
1172 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1173 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1174 }
1175 return (kIOReturnSuccess);
1176 }
1177
1178 static IOReturn
1179 IOHibernateDone(IOHibernateVars * vars)
1180 {
1181 hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
1182
1183 if (vars->videoMapping)
1184 {
1185 if (vars->videoMapSize)
1186 // remove mappings
1187 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1188 if (vars->videoAllocSize)
1189 // dealloc range
1190 kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
1191 }
1192
1193 if (vars->previewBuffer)
1194 {
1195 vars->previewBuffer->release();
1196 vars->previewBuffer = 0;
1197 }
1198
1199 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1200 {
1201 IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
1202 gIOHibernateCurrentHeader->options, 32);
1203 }
1204 else
1205 {
1206 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1207 }
1208
1209 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1210 && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus))
1211 {
1212 IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
1213 &gIOHibernateGraphicsInfo->gfxStatus,
1214 sizeof(gIOHibernateGraphicsInfo->gfxStatus));
1215 }
1216 else
1217 {
1218 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1219 }
1220
1221 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1222
1223 #if defined(__i386__) || defined(__x86_64__)
1224 IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
1225 IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
1226
1227 /*
1228 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1229 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1230 */
1231 if (gIOOptionsEntry) {
1232
1233 if (gIOHibernateRTCVariablesKey) {
1234 if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
1235 gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
1236 }
1237 }
1238
1239 if (gIOHibernateBootNextKey)
1240 {
1241 if (gIOHibernateBootNextSave)
1242 {
1243 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
1244 gIOHibernateBootNextSave->release();
1245 gIOHibernateBootNextSave = NULL;
1246 }
1247 else
1248 gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
1249 }
1250 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) gIOOptionsEntry->sync();
1251 }
1252 #endif
1253
1254 if (vars->srcBuffer) vars->srcBuffer->release();
1255 bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
1256 if (vars->handoffBuffer)
1257 {
1258 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1259 {
1260 IOHibernateHandoff * handoff;
1261 bool done = false;
1262 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
1263 !done;
1264 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
1265 {
1266 HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
1267 uint8_t * data = &handoff->data[0];
1268 switch (handoff->type)
1269 {
1270 case kIOHibernateHandoffTypeEnd:
1271 done = true;
1272 break;
1273
1274 case kIOHibernateHandoffTypeDeviceTree:
1275 MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
1276 break;
1277
1278 case kIOHibernateHandoffTypeKeyStore:
1279 #if defined(__i386__) || defined(__x86_64__)
1280 {
1281 IOBufferMemoryDescriptor *
1282 md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
1283 if (md)
1284 {
1285 IOSetKeyStoreData(md);
1286 }
1287 }
1288 #endif
1289 break;
1290
1291 default:
1292 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
1293 break;
1294 }
1295 }
1296 }
1297 vars->handoffBuffer->release();
1298 }
1299
1300 bzero(vars, sizeof(*vars));
1301
1302 // gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1303
1304 return (kIOReturnSuccess);
1305 }
1306
1307 static void
1308 IOHibernateSystemPostWakeTrim(void * p1, void * p2)
1309 {
1310 // invalidate & close the image file
1311 if (p1) IOLockLock(gFSLock);
1312 if (kFSTrimDelay == gFSState)
1313 {
1314 IOPolledFileIOVars * vars = &gFileVars;
1315 IOPolledFileClose(&vars,
1316 #if DISABLE_TRIM
1317 0, NULL, 0, 0, 0);
1318 #else
1319 0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader),
1320 sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize);
1321 #endif
1322 gFSState = kFSIdle;
1323 }
1324 if (p1) IOLockUnlock(gFSLock);
1325 }
1326
1327 IOReturn
1328 IOHibernateSystemPostWake(bool now)
1329 {
1330 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1331 IOLockLock(gFSLock);
1332 if (kFSTrimDelay == gFSState)
1333 {
1334 thread_call_cancel(gIOHibernateTrimCalloutEntry);
1335 IOHibernateSystemPostWakeTrim(NULL, NULL);
1336 }
1337 else if (kFSOpened != gFSState) gFSState = kFSIdle;
1338 else
1339 {
1340 gFSState = kFSTrimDelay;
1341 if (now)
1342 {
1343 thread_call_cancel(gIOHibernateTrimCalloutEntry);
1344 IOHibernateSystemPostWakeTrim(NULL, NULL);
1345 }
1346 else
1347 {
1348 AbsoluteTime deadline;
1349 clock_interval_to_deadline(TRIM_DELAY, kMillisecondScale, &deadline );
1350 thread_call_enter1_delayed(gIOHibernateTrimCalloutEntry, NULL, deadline);
1351 }
1352 }
1353 IOLockUnlock(gFSLock);
1354
1355 return (kIOReturnSuccess);
1356 }
1357
1358 uint32_t IOHibernateWasScreenLocked(void)
1359 {
1360 uint32_t ret = 0;
1361 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry)
1362 {
1363 OSData *
1364 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
1365 if (data)
1366 {
1367 ret = ((uint32_t *)data->getBytesNoCopy())[0];
1368 gIOChosenEntry->setProperty(kIOBooterScreenLockStateKey, data);
1369 }
1370 }
1371 else gIOChosenEntry->removeProperty(kIOBooterScreenLockStateKey);
1372
1373 return (ret);
1374 }
1375
1376 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1377
1378 SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
1379 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1380 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
1381 SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
1382 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1383 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
1384 SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
1385 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1386 &gIOHibernateMode, 0, "");
1387 SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
1388 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1389 &_hibernateStats, hibernate_statistics_t, "");
1390
1391 SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
1392 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1393 &_hibernateStats.graphicsReadyTime, 0, "");
1394 SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
1395 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1396 &_hibernateStats.wakeNotificationTime, 0, "");
1397 SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
1398 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1399 &_hibernateStats.lockScreenReadyTime, 0, "");
1400 SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
1401 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1402 &_hibernateStats.hidReadyTime, 0, "");
1403
1404
1405 void
1406 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1407 {
1408 gIOHibernateBootImageKey = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1409
1410 #if defined(__i386__) || defined(__x86_64__)
1411 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1412 gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
1413 gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
1414 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1415 #endif /* defined(__i386__) || defined(__x86_64__) */
1416
1417 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
1418 if (data)
1419 {
1420 rootDomain->setProperty(kIOHibernateStateKey, data);
1421 data->release();
1422 }
1423
1424 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))
1425 gIOHibernateMode = kIOHibernateModeOn;
1426 else
1427 gIOHibernateFilename[0] = 0;
1428
1429 sysctl_register_oid(&sysctl__kern_hibernatefile);
1430 sysctl_register_oid(&sysctl__kern_bootsignature);
1431 sysctl_register_oid(&sysctl__kern_hibernatemode);
1432 sysctl_register_oid(&sysctl__kern_hibernatestatistics);
1433 sysctl_register_oid(&sysctl__kern_hibernategraphicsready);
1434 sysctl_register_oid(&sysctl__kern_hibernatewakenotification);
1435 sysctl_register_oid(&sysctl__kern_hibernatelockscreenready);
1436 sysctl_register_oid(&sysctl__kern_hibernatehidready);
1437
1438 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1439
1440 gFSLock = IOLockAlloc();
1441 }
1442
1443 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1444
1445 static IOReturn
1446 IOHibernatePolledFileWrite(IOPolledFileIOVars * vars,
1447 const uint8_t * bytes, IOByteCount size,
1448 IOPolledFileCryptVars * cryptvars)
1449 {
1450 IOReturn err;
1451
1452 err = IOPolledFileWrite(vars, bytes, size, cryptvars);
1453 if ((kIOReturnSuccess == err) && hibernate_should_abort()) err = kIOReturnAborted;
1454
1455 return (err);
1456 }
1457
1458 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1459
1460 extern "C" uint32_t
1461 hibernate_write_image(void)
1462 {
1463 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
1464 IOHibernateVars * vars = &gIOHibernateVars;
1465 IOPolledFileExtent * fileExtents;
1466
1467 _static_assert_1_arg(sizeof(IOHibernateImageHeader) == 512);
1468
1469 uint32_t pageCount, pagesDone;
1470 IOReturn err;
1471 vm_offset_t ppnum, page;
1472 IOItemCount count;
1473 uint8_t * src;
1474 uint8_t * data;
1475 uint8_t * compressed;
1476 uint8_t * scratch;
1477 IOByteCount pageCompressedSize;
1478 uint64_t compressedSize, uncompressedSize;
1479 uint64_t image1Size = 0;
1480 uint32_t bitmap_size;
1481 bool iterDone, pollerOpen, needEncrypt;
1482 uint32_t restore1Sum, sum, sum1, sum2;
1483 int wkresult;
1484 uint32_t tag;
1485 uint32_t pageType;
1486 uint32_t pageAndCount[2];
1487 addr64_t phys64;
1488 IOByteCount segLen;
1489 uintptr_t hibernateBase;
1490 uintptr_t hibernateEnd;
1491
1492 AbsoluteTime startTime, endTime;
1493 AbsoluteTime allTime, compTime;
1494 uint64_t compBytes;
1495 uint64_t nsec;
1496 uint32_t lastProgressStamp = 0;
1497 uint32_t progressStamp;
1498 uint32_t blob, lastBlob = (uint32_t) -1L;
1499
1500 uint32_t wiredPagesEncrypted;
1501 uint32_t dirtyPagesEncrypted;
1502 uint32_t wiredPagesClear;
1503 uint32_t svPageCount;
1504 uint32_t zvPageCount;
1505
1506 IOPolledFileCryptVars _cryptvars;
1507 IOPolledFileCryptVars * cryptvars = 0;
1508
1509 wiredPagesEncrypted = 0;
1510 dirtyPagesEncrypted = 0;
1511 wiredPagesClear = 0;
1512 svPageCount = 0;
1513 zvPageCount = 0;
1514
1515 if (!vars->fileVars
1516 || !vars->fileVars->pollers
1517 || !(kIOHibernateModeOn & gIOHibernateMode)) return (kIOHibernatePostWriteSleep);
1518
1519 if (kIOHibernateModeSleep & gIOHibernateMode)
1520 kdebug_enable = save_kdebug_enable;
1521
1522 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START);
1523 IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
1524
1525 restore1Sum = sum1 = sum2 = 0;
1526
1527 #if CRYPTO
1528 // encryption data. "iv" is the "initial vector".
1529 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1530 {
1531 static const unsigned char first_iv[AES_BLOCK_SIZE]
1532 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
1533 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
1534
1535 cryptvars = &gIOHibernateCryptWakeContext;
1536 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1537 aes_encrypt_key(vars->cryptKey,
1538 kIOHibernateAESKeySize,
1539 &cryptvars->ctx.encrypt);
1540 aes_decrypt_key(vars->cryptKey,
1541 kIOHibernateAESKeySize,
1542 &cryptvars->ctx.decrypt);
1543
1544 cryptvars = &_cryptvars;
1545 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1546 for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++)
1547 vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
1548 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
1549 aes_encrypt_key(vars->wiredCryptKey,
1550 kIOHibernateAESKeySize,
1551 &cryptvars->ctx.encrypt);
1552
1553 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1554 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1555 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
1556 }
1557 #endif /* CRYPTO */
1558
1559 hibernate_page_list_setall(vars->page_list,
1560 vars->page_list_wired,
1561 vars->page_list_pal,
1562 false /* !preflight */,
1563 /* discard_all */
1564 ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
1565 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
1566 &pageCount);
1567
1568 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
1569
1570 fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy();
1571
1572 #if 0
1573 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
1574 for (page = 0; page < count; page++)
1575 {
1576 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
1577 fileExtents[page].start, fileExtents[page].length,
1578 fileExtents[page].start + fileExtents[page].length);
1579 }
1580 #endif
1581
1582 needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
1583 AbsoluteTime_to_scalar(&compTime) = 0;
1584 compBytes = 0;
1585
1586 clock_get_uptime(&allTime);
1587 IOService::getPMRootDomain()->pmStatsRecordEvent(
1588 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
1589 do
1590 {
1591 compressedSize = 0;
1592 uncompressedSize = 0;
1593 svPageCount = 0;
1594 zvPageCount = 0;
1595
1596 IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
1597
1598 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
1599 ml_get_interrupts_enabled());
1600 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState,
1601 // abortable if not low battery
1602 !IOService::getPMRootDomain()->mustHibernate());
1603 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
1604 pollerOpen = (kIOReturnSuccess == err);
1605 if (!pollerOpen)
1606 break;
1607
1608 // copy file block extent list if larger than header
1609
1610 count = vars->fileVars->fileExtents->getLength();
1611 if (count > sizeof(header->fileExtentMap))
1612 {
1613 count -= sizeof(header->fileExtentMap);
1614 err = IOHibernatePolledFileWrite(vars->fileVars,
1615 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
1616 if (kIOReturnSuccess != err)
1617 break;
1618 }
1619
1620 hibernateBase = HIB_BASE; /* Defined in PAL headers */
1621 hibernateEnd = (segHIBB + segSizeHIB);
1622
1623 // copy out restore1 code
1624
1625 for (count = 0;
1626 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1627 count += segLen)
1628 {
1629 for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++)
1630 {
1631 gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
1632 }
1633 }
1634
1635 page = atop_32(kvtophys(hibernateBase));
1636 count = atop_32(round_page(hibernateEnd) - hibernateBase);
1637 header->restore1CodePhysPage = page;
1638 header->restore1CodeVirt = hibernateBase;
1639 header->restore1PageCount = count;
1640 header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
1641 header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
1642
1643 // sum __HIB seg, with zeros for the stack
1644 src = (uint8_t *) trunc_page(hibernateBase);
1645 for (page = 0; page < count; page++)
1646 {
1647 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
1648 restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
1649 else
1650 restore1Sum += 0x00000000;
1651 src += page_size;
1652 }
1653 sum1 = restore1Sum;
1654
1655 // write the __HIB seg, with zeros for the stack
1656
1657 src = (uint8_t *) trunc_page(hibernateBase);
1658 count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
1659 if (count)
1660 {
1661 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
1662 if (kIOReturnSuccess != err)
1663 break;
1664 }
1665 err = IOHibernatePolledFileWrite(vars->fileVars,
1666 (uint8_t *) 0,
1667 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
1668 cryptvars);
1669 if (kIOReturnSuccess != err)
1670 break;
1671 src = &gIOHibernateRestoreStackEnd[0];
1672 count = round_page(hibernateEnd) - ((uintptr_t) src);
1673 if (count)
1674 {
1675 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
1676 if (kIOReturnSuccess != err)
1677 break;
1678 }
1679
1680 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1681 {
1682 vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
1683 vars->fileVars->encryptEnd = UINT64_MAX;
1684 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1685 }
1686
1687 // write the preview buffer
1688
1689 if (vars->previewBuffer)
1690 {
1691 ppnum = 0;
1692 count = 0;
1693 do
1694 {
1695 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
1696 pageAndCount[0] = atop_64(phys64);
1697 pageAndCount[1] = atop_32(segLen);
1698 err = IOHibernatePolledFileWrite(vars->fileVars,
1699 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1700 cryptvars);
1701 if (kIOReturnSuccess != err)
1702 break;
1703 count += segLen;
1704 ppnum += sizeof(pageAndCount);
1705 }
1706 while (phys64);
1707 if (kIOReturnSuccess != err)
1708 break;
1709
1710 src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
1711
1712 ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
1713
1714 count = vars->previewBuffer->getLength();
1715
1716 header->previewPageListSize = ppnum;
1717 header->previewSize = count + ppnum;
1718
1719 for (page = 0; page < count; page += page_size)
1720 {
1721 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
1722 sum1 += hibernate_sum_page(src + page, atop_64(phys64));
1723 }
1724 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
1725 if (kIOReturnSuccess != err)
1726 break;
1727 }
1728
1729 // mark areas for no save
1730 IOMemoryDescriptor * ioBuffer;
1731 ioBuffer = IOPolledFileGetIOBuffer(vars->fileVars);
1732 for (count = 0;
1733 (phys64 = ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1734 count += segLen)
1735 {
1736 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1737 atop_64(phys64), atop_32(segLen),
1738 kIOHibernatePageStateFree);
1739 pageCount -= atop_32(segLen);
1740 }
1741
1742 for (count = 0;
1743 (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1744 count += segLen)
1745 {
1746 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1747 atop_64(phys64), atop_32(segLen),
1748 kIOHibernatePageStateFree);
1749 pageCount -= atop_32(segLen);
1750 }
1751
1752 // copy out bitmap of pages available for trashing during restore
1753
1754 bitmap_size = vars->page_list_wired->list_size;
1755 src = (uint8_t *) vars->page_list_wired;
1756 err = IOHibernatePolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
1757 if (kIOReturnSuccess != err)
1758 break;
1759
1760 // mark more areas for no save, but these are not available
1761 // for trashing during restore
1762
1763 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
1764
1765
1766 page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
1767 count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
1768 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1769 page, count,
1770 kIOHibernatePageStateFree);
1771 pageCount -= count;
1772
1773 if (vars->previewBuffer) for (count = 0;
1774 (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1775 count += segLen)
1776 {
1777 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1778 atop_64(phys64), atop_32(segLen),
1779 kIOHibernatePageStateFree);
1780 pageCount -= atop_32(segLen);
1781 }
1782
1783 for (count = 0;
1784 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1785 count += segLen)
1786 {
1787 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1788 atop_64(phys64), atop_32(segLen),
1789 kIOHibernatePageStateFree);
1790 pageCount -= atop_32(segLen);
1791 }
1792
1793 #if KASAN
1794 vm_size_t shadow_pages_free = atop_64(shadow_ptop) - atop_64(shadow_pnext);
1795
1796 /* no need to save unused shadow pages */
1797 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1798 atop_64(shadow_pnext),
1799 shadow_pages_free,
1800 kIOHibernatePageStateFree);
1801 #endif
1802
1803 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
1804 compressed = src + page_size;
1805 scratch = compressed + page_size;
1806
1807 pagesDone = 0;
1808 lastBlob = 0;
1809
1810 HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
1811 bitmap_size, header->previewSize,
1812 pageCount, vars->fileVars->position);
1813
1814 enum
1815 // pageType
1816 {
1817 kWired = 0x02,
1818 kEncrypt = 0x01,
1819 kWiredEncrypt = kWired | kEncrypt,
1820 kWiredClear = kWired,
1821 kUnwiredEncrypt = kEncrypt
1822 };
1823
1824 bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features()));
1825 #define _pmap_is_noencrypt(x) (cpuAES ? false : pmap_is_noencrypt((x)))
1826
1827 for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)
1828 {
1829 if (kUnwiredEncrypt == pageType)
1830 {
1831 // start unwired image
1832 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1833 {
1834 vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
1835 vars->fileVars->encryptEnd = UINT64_MAX;
1836 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1837 }
1838 bcopy(&cryptvars->aes_iv[0],
1839 &gIOHibernateCryptWakeContext.aes_iv[0],
1840 sizeof(cryptvars->aes_iv));
1841 cryptvars = &gIOHibernateCryptWakeContext;
1842 }
1843 for (iterDone = false, ppnum = 0; !iterDone; )
1844 {
1845 count = hibernate_page_list_iterate((kWired & pageType)
1846 ? vars->page_list_wired : vars->page_list,
1847 &ppnum);
1848 // kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
1849 iterDone = !count;
1850
1851 if (count && (kWired & pageType) && needEncrypt)
1852 {
1853 uint32_t checkIndex;
1854 for (checkIndex = 0;
1855 (checkIndex < count)
1856 && (((kEncrypt & pageType) == 0) == _pmap_is_noencrypt(ppnum + checkIndex));
1857 checkIndex++)
1858 {}
1859 if (!checkIndex)
1860 {
1861 ppnum++;
1862 continue;
1863 }
1864 count = checkIndex;
1865 }
1866
1867 switch (pageType)
1868 {
1869 case kWiredEncrypt: wiredPagesEncrypted += count; break;
1870 case kWiredClear: wiredPagesClear += count; break;
1871 case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
1872 }
1873
1874 if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */}
1875 else
1876 {
1877 pageAndCount[0] = ppnum;
1878 pageAndCount[1] = count;
1879 err = IOHibernatePolledFileWrite(vars->fileVars,
1880 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1881 cryptvars);
1882 if (kIOReturnSuccess != err)
1883 break;
1884 }
1885
1886 for (page = ppnum; page < (ppnum + count); page++)
1887 {
1888 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
1889 if (err)
1890 {
1891 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
1892 break;
1893 }
1894
1895 sum = hibernate_sum_page(src, page);
1896 if (kWired & pageType)
1897 sum1 += sum;
1898 else
1899 sum2 += sum;
1900
1901 clock_get_uptime(&startTime);
1902 wkresult = WKdm_compress_new((const WK_word*) src,
1903 (WK_word*) compressed,
1904 (WK_word*) scratch,
1905 page_size - 4);
1906
1907 clock_get_uptime(&endTime);
1908 ADD_ABSOLUTETIME(&compTime, &endTime);
1909 SUB_ABSOLUTETIME(&compTime, &startTime);
1910
1911 compBytes += page_size;
1912 pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
1913
1914 if (pageCompressedSize == 0)
1915 {
1916 pageCompressedSize = 4;
1917 data = src;
1918
1919 if (*(uint32_t *)src)
1920 svPageCount++;
1921 else
1922 zvPageCount++;
1923 }
1924 else
1925 {
1926 if (pageCompressedSize != page_size)
1927 data = compressed;
1928 else
1929 data = src;
1930 }
1931
1932 tag = pageCompressedSize | kIOHibernateTagSignature;
1933 err = IOHibernatePolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
1934 if (kIOReturnSuccess != err)
1935 break;
1936
1937 err = IOHibernatePolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
1938 if (kIOReturnSuccess != err)
1939 break;
1940
1941 compressedSize += pageCompressedSize;
1942 uncompressedSize += page_size;
1943 pagesDone++;
1944
1945 if (vars->consoleMapping && (0 == (1023 & pagesDone)))
1946 {
1947 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
1948 if (blob != lastBlob)
1949 {
1950 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
1951 lastBlob = blob;
1952 }
1953 }
1954 if (0 == (8191 & pagesDone))
1955 {
1956 clock_get_uptime(&endTime);
1957 SUB_ABSOLUTETIME(&endTime, &allTime);
1958 absolutetime_to_nanoseconds(endTime, &nsec);
1959 progressStamp = nsec / 750000000ULL;
1960 if (progressStamp != lastProgressStamp)
1961 {
1962 lastProgressStamp = progressStamp;
1963 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
1964 }
1965 }
1966 }
1967 if (kIOReturnSuccess != err)
1968 break;
1969 ppnum = page;
1970 }
1971
1972 if (kIOReturnSuccess != err)
1973 break;
1974
1975 if ((kEncrypt & pageType) && vars->fileVars->encryptStart)
1976 {
1977 vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
1978 HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
1979 }
1980
1981 if (kWiredEncrypt != pageType)
1982 {
1983 // end of image1/2 - fill to next block
1984 err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
1985 if (kIOReturnSuccess != err)
1986 break;
1987 }
1988 if (kWiredClear == pageType)
1989 {
1990 // enlarge wired image for test
1991 // err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
1992
1993 // end wired image
1994 header->encryptStart = vars->fileVars->encryptStart;
1995 header->encryptEnd = vars->fileVars->encryptEnd;
1996 image1Size = vars->fileVars->position;
1997 HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
1998 image1Size, header->encryptStart, header->encryptEnd);
1999 }
2000 }
2001 if (kIOReturnSuccess != err)
2002 {
2003 if (kIOReturnOverrun == err)
2004 {
2005 // update actual compression ratio on not enough space (for retry)
2006 gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
2007 }
2008
2009 // update partial amount written (for IOPolledFileClose cleanup/unmap)
2010 header->imageSize = vars->fileVars->position;
2011 break;
2012 }
2013
2014 // Header:
2015
2016 header->imageSize = vars->fileVars->position;
2017 header->image1Size = image1Size;
2018 header->bitmapSize = bitmap_size;
2019 header->pageCount = pageCount;
2020
2021 header->restore1Sum = restore1Sum;
2022 header->image1Sum = sum1;
2023 header->image2Sum = sum2;
2024 header->sleepTime = gIOLastSleepTime.tv_sec;
2025
2026 header->compression = (compressedSize << 8) / uncompressedSize;
2027 gIOHibernateCompression = header->compression;
2028
2029 count = vars->fileVars->fileExtents->getLength();
2030 if (count > sizeof(header->fileExtentMap))
2031 {
2032 header->fileExtentMapSize = count;
2033 count = sizeof(header->fileExtentMap);
2034 }
2035 else
2036 header->fileExtentMapSize = sizeof(header->fileExtentMap);
2037 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
2038
2039 header->deviceBase = vars->fileVars->block0;
2040 header->deviceBlockSize = vars->fileVars->blockSize;
2041
2042 IOPolledFileSeek(vars->fileVars, 0);
2043 err = IOHibernatePolledFileWrite(vars->fileVars,
2044 (uint8_t *) header, sizeof(IOHibernateImageHeader),
2045 cryptvars);
2046 if (kIOReturnSuccess != err)
2047 break;
2048 err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2049 }
2050 while (false);
2051
2052 clock_get_uptime(&endTime);
2053
2054 IOService::getPMRootDomain()->pmStatsRecordEvent(
2055 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
2056
2057 SUB_ABSOLUTETIME(&endTime, &allTime);
2058 absolutetime_to_nanoseconds(endTime, &nsec);
2059 HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
2060
2061 absolutetime_to_nanoseconds(compTime, &nsec);
2062 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2063 compBytes,
2064 nsec / 1000000ULL,
2065 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2066
2067 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2068 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
2069 vars->fileVars->cryptBytes,
2070 nsec / 1000000ULL,
2071 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2072
2073 HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
2074 header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
2075 uncompressedSize, atop_32(uncompressedSize), compressedSize,
2076 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
2077 sum1, sum2);
2078
2079 HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
2080 svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
2081
2082 if (pollerOpen)
2083 IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted );
2084
2085 if (vars->consoleMapping)
2086 ProgressUpdate(gIOHibernateGraphicsInfo,
2087 vars->consoleMapping, 0, kIOHibernateProgressCount);
2088
2089 HIBLOG("hibernate_write_image done(%x)\n", err);
2090
2091 // should we come back via regular wake, set the state in memory.
2092 gIOHibernateState = kIOHibernateStateInactive;
2093
2094 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, wiredPagesEncrypted,
2095 wiredPagesClear, dirtyPagesEncrypted);
2096
2097 if (kIOReturnSuccess == err)
2098 {
2099 if (kIOHibernateModeSleep & gIOHibernateMode)
2100 {
2101 return (kIOHibernatePostWriteSleep);
2102 }
2103 else if(kIOHibernateModeRestart & gIOHibernateMode)
2104 {
2105 return (kIOHibernatePostWriteRestart);
2106 }
2107 else
2108 {
2109 /* by default, power down */
2110 return (kIOHibernatePostWriteHalt);
2111 }
2112 }
2113 else if (kIOReturnAborted == err)
2114 {
2115 return (kIOHibernatePostWriteWake);
2116 }
2117 else
2118 {
2119 /* on error, sleep */
2120 return (kIOHibernatePostWriteSleep);
2121 }
2122 }
2123
2124 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2125
2126 extern "C" void
2127 hibernate_machine_init(void)
2128 {
2129 IOReturn err;
2130 uint32_t sum;
2131 uint32_t pagesDone;
2132 uint32_t pagesRead = 0;
2133 AbsoluteTime startTime, compTime;
2134 AbsoluteTime allTime, endTime;
2135 AbsoluteTime startIOTime, endIOTime;
2136 uint64_t nsec, nsecIO;
2137 uint64_t compBytes;
2138 uint32_t lastProgressStamp = 0;
2139 uint32_t progressStamp;
2140 IOPolledFileCryptVars * cryptvars = 0;
2141
2142 IOHibernateVars * vars = &gIOHibernateVars;
2143 bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
2144
2145 if (!vars->fileVars || !vars->fileVars->pollers)
2146 return;
2147
2148 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2149 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2150
2151 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
2152 {
2153 HIBLOG("regular wake\n");
2154 return;
2155 }
2156
2157 HIBPRINT("diag %x %x %x %x\n",
2158 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
2159 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
2160
2161 #define t40ms(x) (tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)
2162 #define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
2163 tStat(booterStart, booterStart);
2164 gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart;
2165 tStat(booterDuration0, booterTime0);
2166 tStat(booterDuration1, booterTime1);
2167 tStat(booterDuration2, booterTime2);
2168 tStat(booterDuration, booterTime);
2169 tStat(booterConnectDisplayDuration, connectDisplayTime);
2170 tStat(booterSplashDuration, splashTime);
2171 tStat(trampolineDuration, trampolineTime);
2172
2173 gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size;
2174 gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize;
2175 gIOHibernateStats->image1Pages = pagesDone;
2176
2177 /* HIBERNATE_stats */
2178 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 14), gIOHibernateStats->smcStart,
2179 gIOHibernateStats->booterStart, gIOHibernateStats->booterDuration,
2180 gIOHibernateStats->trampolineDuration);
2181
2182 HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
2183 gIOHibernateStats->booterStart,
2184 gIOHibernateStats->smcStart,
2185 gIOHibernateStats->booterDuration0,
2186 gIOHibernateStats->booterDuration1,
2187 gIOHibernateStats->booterDuration2,
2188 gIOHibernateStats->booterDuration,
2189 gIOHibernateStats->booterConnectDisplayDuration,
2190 gIOHibernateStats->booterSplashDuration,
2191 gIOHibernateStats->trampolineDuration);
2192
2193 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
2194 gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
2195 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
2196
2197 if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
2198 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)))
2199 {
2200 hibernate_page_list_discard(vars->page_list);
2201 }
2202
2203 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
2204
2205 if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount)
2206 panic("handoff overflow");
2207
2208 IOHibernateHandoff * handoff;
2209 bool done = false;
2210 bool foundCryptData = false;
2211
2212 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
2213 !done;
2214 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
2215 {
2216 // HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
2217 uint8_t * data = &handoff->data[0];
2218 switch (handoff->type)
2219 {
2220 case kIOHibernateHandoffTypeEnd:
2221 done = true;
2222 break;
2223
2224 case kIOHibernateHandoffTypeGraphicsInfo:
2225 if (handoff->bytecount == sizeof(*gIOHibernateGraphicsInfo))
2226 {
2227 bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
2228 }
2229 break;
2230
2231 case kIOHibernateHandoffTypeCryptVars:
2232 if (cryptvars)
2233 {
2234 hibernate_cryptwakevars_t *
2235 wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
2236 bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
2237 }
2238 foundCryptData = true;
2239 bzero(data, handoff->bytecount);
2240 break;
2241
2242 case kIOHibernateHandoffTypeMemoryMap:
2243
2244 clock_get_uptime(&allTime);
2245
2246 hibernate_newruntime_map(data, handoff->bytecount,
2247 gIOHibernateCurrentHeader->systemTableOffset);
2248
2249 clock_get_uptime(&endTime);
2250
2251 SUB_ABSOLUTETIME(&endTime, &allTime);
2252 absolutetime_to_nanoseconds(endTime, &nsec);
2253
2254 HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
2255
2256 break;
2257
2258 case kIOHibernateHandoffTypeDeviceTree:
2259 {
2260 // DTEntry chosen = NULL;
2261 // HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen));
2262 }
2263 break;
2264
2265 default:
2266 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
2267 break;
2268 }
2269 }
2270 if (cryptvars && !foundCryptData)
2271 panic("hibernate handoff");
2272
2273 HIBPRINT("video 0x%llx %d %d %d status %x\n",
2274 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2275 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
2276
2277 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)
2278 {
2279 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2280 * gIOHibernateGraphicsInfo->rowBytes);
2281 if (vars->videoMapSize > vars->videoAllocSize) vars->videoMapSize = 0;
2282 else
2283 {
2284 IOMapPages(kernel_map,
2285 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2286 vars->videoMapSize, kIOMapInhibitCache );
2287 }
2288 }
2289
2290 if (vars->videoMapSize)
2291 ProgressUpdate(gIOHibernateGraphicsInfo,
2292 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2293
2294 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2295 uint8_t * compressed = src + page_size;
2296 uint8_t * scratch = compressed + page_size;
2297 uint32_t decoOffset;
2298
2299 clock_get_uptime(&allTime);
2300 AbsoluteTime_to_scalar(&compTime) = 0;
2301 compBytes = 0;
2302
2303 HIBLOG("IOPolledFilePollersOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2304 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledAfterSleepState, false);
2305 clock_get_uptime(&startIOTime);
2306 endTime = startIOTime;
2307 SUB_ABSOLUTETIME(&endTime, &allTime);
2308 absolutetime_to_nanoseconds(endTime, &nsec);
2309 HIBLOG("IOPolledFilePollersOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
2310
2311 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2312
2313 // kick off the read ahead
2314 vars->fileVars->bufferHalf = 0;
2315 vars->fileVars->bufferLimit = 0;
2316 vars->fileVars->lastRead = 0;
2317 vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
2318 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2319 vars->fileVars->cryptBytes = 0;
2320 AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
2321
2322 err = IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
2323 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2324 // --
2325
2326 HIBLOG("hibernate_machine_init reading\n");
2327
2328 uint32_t * header = (uint32_t *) src;
2329 sum = 0;
2330
2331 while (kIOReturnSuccess == err)
2332 {
2333 unsigned int count;
2334 unsigned int page;
2335 uint32_t tag;
2336 vm_offset_t ppnum, compressedSize;
2337
2338 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2339 if (kIOReturnSuccess != err)
2340 break;
2341
2342 ppnum = header[0];
2343 count = header[1];
2344
2345 // HIBPRINT("(%x, %x)\n", ppnum, count);
2346
2347 if (!count)
2348 break;
2349
2350 for (page = 0; page < count; page++)
2351 {
2352 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2353 if (kIOReturnSuccess != err)
2354 break;
2355
2356 compressedSize = kIOHibernateTagLength & tag;
2357 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength))
2358 {
2359 err = kIOReturnIPCError;
2360 break;
2361 }
2362
2363 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2364 if (kIOReturnSuccess != err) break;
2365
2366 if (compressedSize < page_size)
2367 {
2368 decoOffset = page_size;
2369 clock_get_uptime(&startTime);
2370
2371 if (compressedSize == 4) {
2372 int i;
2373 uint32_t *s, *d;
2374
2375 s = (uint32_t *)src;
2376 d = (uint32_t *)(uintptr_t)compressed;
2377
2378 for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++)
2379 *d++ = *s;
2380 }
2381 else
2382 WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, compressedSize);
2383 clock_get_uptime(&endTime);
2384 ADD_ABSOLUTETIME(&compTime, &endTime);
2385 SUB_ABSOLUTETIME(&compTime, &startTime);
2386 compBytes += page_size;
2387 }
2388 else decoOffset = 0;
2389
2390 sum += hibernate_sum_page((src + decoOffset), ppnum);
2391 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2392 if (err)
2393 {
2394 HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
2395 break;
2396 }
2397
2398 ppnum++;
2399 pagesDone++;
2400 pagesRead++;
2401
2402 if (0 == (8191 & pagesDone))
2403 {
2404 clock_get_uptime(&endTime);
2405 SUB_ABSOLUTETIME(&endTime, &allTime);
2406 absolutetime_to_nanoseconds(endTime, &nsec);
2407 progressStamp = nsec / 750000000ULL;
2408 if (progressStamp != lastProgressStamp)
2409 {
2410 lastProgressStamp = progressStamp;
2411 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2412 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2413 }
2414 }
2415 }
2416 }
2417 if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages))
2418 err = kIOReturnLockedRead;
2419
2420 if (kIOReturnSuccess != err)
2421 panic("Hibernate restore error %x", err);
2422
2423 gIOHibernateCurrentHeader->actualImage2Sum = sum;
2424 gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
2425
2426 clock_get_uptime(&endIOTime);
2427
2428 err = IOPolledFilePollersClose(vars->fileVars, kIOPolledAfterSleepState);
2429
2430 clock_get_uptime(&endTime);
2431
2432 IOService::getPMRootDomain()->pmStatsRecordEvent(
2433 kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
2434 IOService::getPMRootDomain()->pmStatsRecordEvent(
2435 kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
2436
2437 SUB_ABSOLUTETIME(&endTime, &allTime);
2438 absolutetime_to_nanoseconds(endTime, &nsec);
2439
2440 SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
2441 absolutetime_to_nanoseconds(endIOTime, &nsecIO);
2442
2443 gIOHibernateStats->kernelImageReadDuration = nsec / 1000000ULL;
2444 gIOHibernateStats->imagePages = pagesDone;
2445
2446 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
2447 pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
2448 nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
2449
2450 absolutetime_to_nanoseconds(compTime, &nsec);
2451 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2452 compBytes,
2453 nsec / 1000000ULL,
2454 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2455
2456 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2457 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
2458 vars->fileVars->cryptBytes,
2459 nsec / 1000000ULL,
2460 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2461
2462 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 2), pagesRead, pagesDone);
2463 }
2464
2465 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2466
2467 void IOHibernateSetWakeCapabilities(uint32_t capability)
2468 {
2469 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
2470 {
2471 gIOHibernateStats->wakeCapability = capability;
2472
2473 if (kIOPMSystemCapabilityGraphics & capability)
2474 {
2475 vm_compressor_do_warmup();
2476 }
2477 }
2478 }
2479
2480 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2481
2482 void IOHibernateSystemRestart(void)
2483 {
2484 static uint8_t noteStore[32] __attribute__((aligned(32)));
2485 IORegistryEntry * regEntry;
2486 const OSSymbol * sym;
2487 OSData * noteProp;
2488 OSData * data;
2489 uintptr_t * smcVars;
2490 uint8_t * smcBytes;
2491 size_t len;
2492 addr64_t element;
2493
2494 data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
2495 if (!data) return;
2496
2497 smcVars = (typeof(smcVars)) data->getBytesNoCopy();
2498 smcBytes = (typeof(smcBytes)) smcVars[1];
2499 len = smcVars[0];
2500 if (len > sizeof(noteStore)) len = sizeof(noteStore);
2501 noteProp = OSData::withCapacity(3 * sizeof(element));
2502 if (!noteProp) return;
2503 element = len;
2504 noteProp->appendBytes(&element, sizeof(element));
2505 element = crc32(0, smcBytes, len);
2506 noteProp->appendBytes(&element, sizeof(element));
2507
2508 bcopy(smcBytes, noteStore, len);
2509 element = (addr64_t) &noteStore[0];
2510 element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
2511 noteProp->appendBytes(&element, sizeof(element));
2512
2513 if (!gIOOptionsEntry)
2514 {
2515 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
2516 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
2517 if (regEntry && !gIOOptionsEntry)
2518 regEntry->release();
2519 }
2520
2521 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
2522 if (gIOOptionsEntry && sym) gIOOptionsEntry->setProperty(sym, noteProp);
2523 if (noteProp) noteProp->release();
2524 if (sym) sym->release();
2525 }
2526
2527
2528
mirror of XNU