]> git.saurik.com Git - apple/xnu.git/blob - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blob
? search:


56a77f89de70087117b556f5d38c4cf682568b13
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
1 /*
2 * Copyright (c) 2004-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30
31 Sleep:
32
33 - PMRootDomain calls IOHibernateSystemSleep() before system sleep
34 (devices awake, normal execution context)
35 - IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
36 grabs its extents and searches for a polling driver willing to work with that IOMedia.
37 The BSD code makes an ioctl to the storage driver to get the partition base offset to
38 the disk, and other ioctls to get the transfer constraints
39 If successful, the file is written to make sure its initially not bootable (in case of
40 later failure) and nvram set to point to the first block of the file. (Has to be done
41 here so blocking is possible in nvram support).
42 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
43 page out any pages it wants to (currently zero, but probably some percentage of memory).
44 Its assumed just allocating pages will cause the VM system to naturally select the best
45 pages for eviction. It also copies processor flags needed for the restore path and sets
46 a flag in the boot processor proc info.
47 gIOHibernateState = kIOHibernateStateHibernating.
48 - Regular sleep progresses - some drivers may inspect the root domain property
49 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
50 as usual but leaves motherboard I/O on.
51 - Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
52 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
53 all ppc RC bits out of the hash table and caches into the mapping structures.
54 - hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
55 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
56 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
57 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
58 The image header and block list are written. The header includes the second file extent so
59 only the header block is needed to read the file, regardless of filesystem.
60 The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
61 (only) is used to decompress the image during wake/boot.
62 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
63 The bitmaps are written to the image.
64 More areas are removed from the bitmaps (after they have been written to the image) - the
65 segment "__HIB" pages and interrupt stack.
66 Each wired page is compressed and written and then each non-wired page. Compression and
67 disk writes are in parallel.
68 The image header is written to the start of the file and the polling driver closed.
69 The machine powers down (or sleeps).
70
71 Boot/Wake:
72
73 - BootX sees the boot-image nvram variable containing the device and block number of the image,
74 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
75 - BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
76 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
77 that is in the kernel's __HIB section.
78 - BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
79 hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
80 only code & data in that section is safe to call since all the other wired pages are still
81 compressed in the image.
82 - hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
83 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
84 location directly, and copies those that can't to interim free pages. When the image has been
85 completed, the copies are uncompressed, overwriting the wired image pages.
86 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
87 is used to get pages into place for 64bit.
88 - the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
89 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
90 are removed from the software strutures, and the hash table is reinitialized.
91 - After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
92 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
93 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
94 for the remaining non wired pages.
95 - The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
96 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
97 is closed via bsd.
98
99 Polled Mode I/O:
100
101 IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
102 registry, specifying an object of calls IOPolledInterface.
103
104 Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
105 partition) that the image is going to live, looking for polled interface properties. If it finds
106 one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
107 interfaces found are kept in an ordered list.
108
109 There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
110 few different contexts things happen in:
111
112 - there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
113 up and running) and after wake - this is safe to allocate memory and do anything. The device
114 ignores sleep requests from that point since its a waste of time if it goes to sleep and
115 immediately wakes back up for the image write.
116
117 - there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
118 immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
119 from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
120 used to flush and set the disk to sleep.
121
122 - there is an Open/Close (AfterSleep) pair made around the image read operations that happen
123 immediately after sleep. These can't block or allocate memory. This is happening after the platform
124 expert has woken the low level bits of the system, but most of the I/O system has not. There is only
125 one thread running.
126
127 For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
128 (prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
129 the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
130 Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
131 that is called for the hardware to check for events, and complete the I/O via the callback.
132 The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
133 to restrict I/O ops.
134 */
135
136 #include <sys/systm.h>
137
138 #include <IOKit/IOWorkLoop.h>
139 #include <IOKit/IOCommandGate.h>
140 #include <IOKit/IOTimerEventSource.h>
141 #include <IOKit/IOPlatformExpert.h>
142 #include <IOKit/IOKitDebug.h>
143 #include <IOKit/IOTimeStamp.h>
144 #include <IOKit/pwr_mgt/RootDomain.h>
145 #include <IOKit/pwr_mgt/IOPMPrivate.h>
146 #include <IOKit/IOMessage.h>
147 #include <IOKit/IODeviceTreeSupport.h>
148 #include <IOKit/IOBSD.h>
149 #include "RootDomainUserClient.h"
150 #include <IOKit/pwr_mgt/IOPowerConnection.h>
151 #include "IOPMPowerStateQueue.h"
152 #include <IOKit/IOBufferMemoryDescriptor.h>
153 #include <IOKit/AppleKeyStoreInterface.h>
154 #include <libkern/crypto/aes.h>
155
156 #include <sys/uio.h>
157 #include <sys/conf.h>
158 #include <sys/stat.h>
159 #include <sys/fcntl.h> // (FWRITE, ...)
160 #include <sys/sysctl.h>
161 #include <sys/kdebug.h>
162 #include <stdint.h>
163
164 #include <IOKit/IOHibernatePrivate.h>
165 #include <IOKit/IOPolledInterface.h>
166 #include <IOKit/IONVRAM.h>
167 #include "IOHibernateInternal.h"
168 #include <vm/WKdm_new.h>
169 #include <vm/vm_protos.h>
170 #include "IOKitKernelInternal.h"
171 #include <pexpert/device_tree.h>
172
173 #include <machine/pal_routines.h>
174 #include <machine/pal_hibernate.h>
175 #include <i386/tsc.h>
176 #include <i386/cpuid.h>
177
178 extern "C" addr64_t kvtophys(vm_offset_t va);
179 extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
180
181 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
182
183 #define DISABLE_TRIM 0
184 #define TRIM_DELAY 25000
185
186 extern unsigned int save_kdebug_enable;
187 extern uint32_t gIOHibernateState;
188 uint32_t gIOHibernateMode;
189 static char gIOHibernateBootSignature[256+1];
190 static char gIOHibernateFilename[MAXPATHLEN+1];
191 static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
192 uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
193 static uint64_t gIOHibernateCompression = 0x80; // default compression 50%
194 boolean_t gIOHibernateStandbyDisabled;
195
196 static IODTNVRAM * gIOOptionsEntry;
197 static IORegistryEntry * gIOChosenEntry;
198
199 static const OSSymbol * gIOHibernateBootImageKey;
200
201 #if defined(__i386__) || defined(__x86_64__)
202
203 static const OSSymbol * gIOHibernateRTCVariablesKey;
204 static const OSSymbol * gIOHibernateBoot0082Key;
205 static const OSSymbol * gIOHibernateBootNextKey;
206 static OSData * gIOHibernateBoot0082Data;
207 static OSData * gIOHibernateBootNextData;
208 static OSObject * gIOHibernateBootNextSave;
209
210 static IOPolledFileIOVars * gDebugImageFileVars;
211 static IOLock * gDebugImageLock;
212
213 #endif /* defined(__i386__) || defined(__x86_64__) */
214
215 static IOLock * gFSLock;
216 static uint32_t gFSState;
217 static IOPolledFileIOVars gFileVars;
218 static IOHibernateVars gIOHibernateVars;
219 static IOPolledFileCryptVars gIOHibernateCryptWakeContext;
220 static hibernate_graphics_t _hibernateGraphics;
221 static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
222 static hibernate_statistics_t _hibernateStats;
223 static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats;
224
225 enum
226 {
227 kFSIdle = 0,
228 kFSOpening = 2,
229 kFSOpened = 3,
230 kFSTimedOut = 4,
231 };
232
233 static IOReturn IOHibernateDone(IOHibernateVars * vars);
234 static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature);
235 static void IOSetBootImageNVRAM(OSData * data);
236
237 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
238
239 enum { kDefaultIOSize = 128 * 1024 };
240 enum { kVideoMapSize = 80 * 1024 * 1024 };
241
242 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
243
244 // copy from phys addr to MD
245
246 static IOReturn
247 IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
248 IOByteCount offset, addr64_t bytes, IOByteCount length)
249 {
250 addr64_t srcAddr = bytes;
251 IOByteCount remaining;
252
253 remaining = length = min(length, md->getLength() - offset);
254 while (remaining) { // (process another target segment?)
255 addr64_t dstAddr64;
256 IOByteCount dstLen;
257
258 dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
259 if (!dstAddr64)
260 break;
261
262 // Clip segment length to remaining
263 if (dstLen > remaining)
264 dstLen = remaining;
265
266 #if 1
267 bcopy_phys(srcAddr, dstAddr64, dstLen);
268 #else
269 copypv(srcAddr, dstAddr64, dstLen,
270 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
271 #endif
272 srcAddr += dstLen;
273 offset += dstLen;
274 remaining -= dstLen;
275 }
276
277 assert(!remaining);
278
279 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
280 }
281
282 // copy from MD to phys addr
283
284 static IOReturn
285 IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
286 IOByteCount offset, addr64_t bytes, IOByteCount length)
287 {
288 addr64_t dstAddr = bytes;
289 IOByteCount remaining;
290
291 remaining = length = min(length, md->getLength() - offset);
292 while (remaining) { // (process another target segment?)
293 addr64_t srcAddr64;
294 IOByteCount dstLen;
295
296 srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
297 if (!srcAddr64)
298 break;
299
300 // Clip segment length to remaining
301 if (dstLen > remaining)
302 dstLen = remaining;
303
304 #if 1
305 bcopy_phys(srcAddr64, dstAddr, dstLen);
306 #else
307 copypv(srcAddr, dstAddr64, dstLen,
308 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
309 #endif
310 dstAddr += dstLen;
311 offset += dstLen;
312 remaining -= dstLen;
313 }
314
315 assert(!remaining);
316
317 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
318 }
319
320 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
321
322 void
323 hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
324 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
325 {
326 count += ppnum;
327 switch (kind)
328 {
329 case kIOHibernatePageStateUnwiredSave:
330 // unwired save
331 for (; ppnum < count; ppnum++)
332 {
333 hibernate_page_bitset(page_list, FALSE, ppnum);
334 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
335 }
336 break;
337 case kIOHibernatePageStateWiredSave:
338 // wired save
339 for (; ppnum < count; ppnum++)
340 {
341 hibernate_page_bitset(page_list, FALSE, ppnum);
342 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
343 }
344 break;
345 case kIOHibernatePageStateFree:
346 // free page
347 for (; ppnum < count; ppnum++)
348 {
349 hibernate_page_bitset(page_list, TRUE, ppnum);
350 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
351 }
352 break;
353 default:
354 panic("hibernate_set_page_state");
355 }
356 }
357
358 static vm_offset_t
359 hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
360 {
361 uint32_t page = *pPage;
362 uint32_t count;
363 hibernate_bitmap_t * bitmap;
364
365 while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
366 {
367 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
368 if (!count)
369 break;
370 page += count;
371 if (page <= bitmap->last_page)
372 break;
373 }
374
375 *pPage = page;
376 if (bitmap)
377 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
378 else
379 count = 0;
380
381 return (count);
382 }
383
384 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
385
386 IOReturn
387 IOHibernateSystemSleep(void)
388 {
389 IOReturn err;
390 OSData * nvramData;
391 OSObject * obj;
392 OSString * str;
393 OSNumber * num;
394 bool dsSSD, vmflush, swapPinned;
395 IOHibernateVars * vars;
396 uint64_t setFileSize = 0;
397
398 gIOHibernateState = kIOHibernateStateInactive;
399
400 gIOHibernateDebugFlags = 0;
401 if (kIOLogHibernate & gIOKitDebug)
402 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
403
404 if (IOService::getPMRootDomain()->getHibernateSettings(
405 &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime))
406 {
407 if (kIOHibernateModeSleep & gIOHibernateMode)
408 // default to discard clean for safe sleep
409 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
410 | kIOHibernateModeDiscardCleanActive);
411 }
412
413 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
414 {
415 if ((str = OSDynamicCast(OSString, obj)))
416 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
417 sizeof(gIOHibernateFilename));
418 obj->release();
419 }
420
421 if (!gIOHibernateMode || !gIOHibernateFilename[0])
422 return (kIOReturnUnsupported);
423
424 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
425
426 vars = IONew(IOHibernateVars, 1);
427 if (!vars) return (kIOReturnNoMemory);
428 bzero(vars, sizeof(*vars));
429
430 IOLockLock(gFSLock);
431 if (kFSIdle != gFSState)
432 {
433 HIBLOG("hibernate file busy\n");
434 IOLockUnlock(gFSLock);
435 IODelete(vars, IOHibernateVars, 1);
436 return (kIOReturnBusy);
437 }
438 gFSState = kFSOpening;
439 IOLockUnlock(gFSLock);
440
441 swapPinned = false;
442 do
443 {
444 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
445 2 * page_size + WKdm_SCRATCH_BUF_SIZE_INTERNAL, page_size);
446
447 vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
448 ptoa_64(gIOHibernateHandoffPageCount), page_size);
449
450 if (!vars->srcBuffer || !vars->handoffBuffer)
451 {
452 err = kIOReturnNoMemory;
453 break;
454 }
455
456 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey)))
457 {
458 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMinSize = num->unsigned64BitValue();
459 obj->release();
460 }
461 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey)))
462 {
463 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMaxSize = num->unsigned64BitValue();
464 obj->release();
465 }
466
467 boolean_t encryptedswap = true;
468 uint32_t pageCount;
469 AbsoluteTime startTime, endTime;
470 uint64_t nsec;
471
472 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
473 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
474 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
475
476 vmflush = ((kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
477 err = hibernate_alloc_page_lists(&vars->page_list,
478 &vars->page_list_wired,
479 &vars->page_list_pal);
480 if (KERN_SUCCESS != err) break;
481
482 err = hibernate_pin_swap(TRUE);
483 if (KERN_SUCCESS != err) break;
484 swapPinned = true;
485
486 if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode))
487 {
488 hibernate_page_list_setall(vars->page_list,
489 vars->page_list_wired,
490 vars->page_list_pal,
491 true /* preflight */,
492 vmflush /* discard */,
493 &pageCount);
494 PE_Video consoleInfo;
495 bzero(&consoleInfo, sizeof(consoleInfo));
496 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
497
498 // estimate: 6% increase in pages compressed
499 // screen preview 2 images compressed 0%
500 setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
501 + vars->page_list->list_size
502 + (consoleInfo.v_width * consoleInfo.v_height * 8);
503 enum { setFileRound = 1024*1024ULL };
504 setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
505
506 HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
507 pageCount, (100ULL * gIOHibernateCompression) >> 8,
508 setFileSize, vars->fileMinSize);
509
510 if (!(kIOHibernateModeFileResize & gIOHibernateMode)
511 && (setFileSize < vars->fileMinSize))
512 {
513 setFileSize = vars->fileMinSize;
514 }
515 }
516
517 // Invalidate the image file
518 if (gDebugImageLock) {
519 IOLockLock(gDebugImageLock);
520 if (gDebugImageFileVars != 0) {
521 kprintf("IOHIBSystemSleep: Closing debugdata file\n");
522 IOSetBootImageNVRAM(0);
523 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
524 }
525 IOLockUnlock(gDebugImageLock);
526 }
527
528 err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, 0,
529 gIOHibernateCurrentHeader, sizeof(gIOHibernateCurrentHeader),
530 &vars->fileVars, &nvramData,
531 &vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
532
533 if (KERN_SUCCESS != err)
534 {
535 IOLockLock(gFSLock);
536 if (kFSOpening != gFSState) err = kIOReturnTimeout;
537 IOLockUnlock(gFSLock);
538 }
539
540 if (KERN_SUCCESS != err)
541 {
542 HIBLOG("IOPolledFileOpen(%x)\n", err);
543 break;
544 }
545
546 // write extents for debug data usage in EFI
547 IOWriteExtentsToFile(vars->fileVars, kIOHibernateHeaderOpenSignature);
548
549 err = IOPolledFilePollersSetup(vars->fileVars, kIOPolledPreflightState);
550 if (KERN_SUCCESS != err) break;
551
552 clock_get_uptime(&startTime);
553 err = hibernate_setup(gIOHibernateCurrentHeader,
554 vmflush,
555 vars->page_list, vars->page_list_wired, vars->page_list_pal);
556 clock_get_uptime(&endTime);
557 SUB_ABSOLUTETIME(&endTime, &startTime);
558 absolutetime_to_nanoseconds(endTime, &nsec);
559
560 boolean_t haveSwapPin, hibFileSSD;
561 haveSwapPin = vm_swap_files_pinned();
562
563 hibFileSSD = (kIOPolledFileSSD & vars->fileVars->flags);
564
565 HIBLOG("hibernate_setup(%d) took %qd ms, swapPin(%d) ssd(%d)\n",
566 err, nsec / 1000000ULL,
567 haveSwapPin, hibFileSSD);
568 if (KERN_SUCCESS != err) break;
569
570 gIOHibernateStandbyDisabled = ((!haveSwapPin || !hibFileSSD));
571
572 dsSSD = ((0 != (kIOPolledFileSSD & vars->fileVars->flags))
573 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
574
575 if (dsSSD) gIOHibernateCurrentHeader->options |= kIOHibernateOptionSSD | kIOHibernateOptionColor;
576 else gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
577
578
579 #if defined(__i386__) || defined(__x86_64__)
580 if (!uuid_is_null(vars->volumeCryptKey) &&
581 (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey)))
582 {
583 uintptr_t smcVars[2];
584 smcVars[0] = sizeof(vars->volumeCryptKey);
585 smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
586
587 IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
588 bzero(smcVars, sizeof(smcVars));
589 }
590 #endif
591
592
593 if (encryptedswap || !uuid_is_null(vars->volumeCryptKey))
594 gIOHibernateMode ^= kIOHibernateModeEncrypt;
595
596 if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
597 {
598 vars->videoAllocSize = kVideoMapSize;
599 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize, VM_KERN_MEMORY_IOKIT))
600 vars->videoMapping = 0;
601 }
602
603 // generate crypt keys
604 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
605 vars->wiredCryptKey[i] = random();
606 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
607 vars->cryptKey[i] = random();
608
609 // set nvram
610
611 IOSetBootImageNVRAM(nvramData);
612 nvramData->release();
613
614 #if defined(__i386__) || defined(__x86_64__)
615 {
616 struct AppleRTCHibernateVars
617 {
618 uint8_t signature[4];
619 uint32_t revision;
620 uint8_t booterSignature[20];
621 uint8_t wiredCryptKey[16];
622 };
623 AppleRTCHibernateVars rtcVars;
624 OSData * data;
625
626 rtcVars.signature[0] = 'A';
627 rtcVars.signature[1] = 'A';
628 rtcVars.signature[2] = 'P';
629 rtcVars.signature[3] = 'L';
630 rtcVars.revision = 1;
631 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
632 if (gIOHibernateBootSignature[0])
633 {
634 char c;
635 uint8_t value = 0;
636 for (uint32_t i = 0;
637 (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
638 i++)
639 {
640 if (c >= 'a') c -= 'a' - 10;
641 else if (c >= 'A') c -= 'A' - 10;
642 else if (c >= '0') c -= '0';
643 else continue;
644 value = (value << 4) | c;
645 if (i & 1) rtcVars.booterSignature[i >> 1] = value;
646 }
647 }
648 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
649 if (data)
650 {
651 if (gIOHibernateRTCVariablesKey)
652 IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
653 data->release();
654 }
655 if (gIOChosenEntry)
656 {
657 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
658 if (data) gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
659 // set BootNext
660 if (!gIOHibernateBoot0082Data)
661 {
662 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
663 if (data)
664 {
665 // AppleNVRAM_EFI_LOAD_OPTION
666 struct {
667 uint32_t Attributes;
668 uint16_t FilePathLength;
669 uint16_t Desc;
670 } loadOptionHeader;
671 loadOptionHeader.Attributes = 1;
672 loadOptionHeader.FilePathLength = data->getLength();
673 loadOptionHeader.Desc = 0;
674 gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
675 if (gIOHibernateBoot0082Data)
676 {
677 gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
678 gIOHibernateBoot0082Data->appendBytes(data);
679 }
680 }
681 }
682 if (!gIOHibernateBootNextData)
683 {
684 uint16_t bits = 0x0082;
685 gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
686 }
687 if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData)
688 {
689 gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
690 gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
691 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
692 }
693 // BootNext
694 }
695 }
696 #endif /* !i386 && !x86_64 */
697 }
698 while (false);
699
700 if (swapPinned) hibernate_pin_swap(FALSE);
701
702 IOLockLock(gFSLock);
703 if ((kIOReturnSuccess == err) && (kFSOpening != gFSState))
704 {
705 HIBLOG("hibernate file close due timeout\n");
706 err = kIOReturnTimeout;
707 }
708 if (kIOReturnSuccess == err)
709 {
710 gFSState = kFSOpened;
711 gIOHibernateVars = *vars;
712 gFileVars = *vars->fileVars;
713 gFileVars.allocated = false;
714 gIOHibernateVars.fileVars = &gFileVars;
715 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
716 gIOHibernateState = kIOHibernateStateHibernating;
717 }
718 else
719 {
720 IOPolledFileIOVars * fileVars = vars->fileVars;
721 IOHibernateDone(vars);
722 IOPolledFileClose(&fileVars,
723 #if DISABLE_TRIM
724 0, NULL, 0, 0, 0);
725 #else
726 0, NULL, 0, sizeof(IOHibernateImageHeader), setFileSize);
727 #endif
728 gFSState = kFSIdle;
729 }
730 IOLockUnlock(gFSLock);
731
732 if (vars->fileVars) IODelete(vars->fileVars, IOPolledFileIOVars, 1);
733 IODelete(vars, IOHibernateVars, 1);
734
735 return (err);
736 }
737
738 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
739
740 static void
741 IOSetBootImageNVRAM(OSData * data)
742 {
743 IORegistryEntry * regEntry;
744
745 if (!gIOOptionsEntry)
746 {
747 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
748 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
749 if (regEntry && !gIOOptionsEntry)
750 regEntry->release();
751 }
752 if (gIOOptionsEntry && gIOHibernateBootImageKey)
753 {
754 if (data) gIOOptionsEntry->setProperty(gIOHibernateBootImageKey, data);
755 else
756 {
757 gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
758 gIOOptionsEntry->sync();
759 }
760 }
761 }
762
763 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
764 /*
765 * Writes header to disk with signature, block size and file extents data.
766 * If there are more than 2 extents, then they are written on second block.
767 */
768 static IOReturn
769 IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature)
770 {
771 IOHibernateImageHeader hdr;
772 IOItemCount count;
773 IOReturn err = kIOReturnSuccess;
774 int rc;
775 IOPolledFileExtent * fileExtents;
776
777 fileExtents = (typeof(fileExtents)) vars->fileExtents->getBytesNoCopy();
778
779 memset(&hdr, 0, sizeof(IOHibernateImageHeader));
780 count = vars->fileExtents->getLength();
781 if (count > sizeof(hdr.fileExtentMap))
782 {
783 hdr.fileExtentMapSize = count;
784 count = sizeof(hdr.fileExtentMap);
785 }
786 else
787 hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap);
788
789 bcopy(fileExtents, &hdr.fileExtentMap[0], count);
790
791 // copy file block extent list if larger than header
792 if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap))
793 {
794 count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap);
795 rc = kern_write_file(vars->fileRef, vars->blockSize,
796 (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)),
797 count, IO_SKIP_ENCRYPTION);
798 if (rc != 0) {
799 HIBLOG("kern_write_file returned %d\n", rc);
800 err = kIOReturnIOError;
801 goto exit;
802 }
803 }
804 hdr.signature = signature;
805 hdr.deviceBlockSize = vars->blockSize;
806
807 rc = kern_write_file(vars->fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION);
808 if (rc != 0) {
809 HIBLOG("kern_write_file returned %d\n", rc);
810 err = kIOReturnIOError;
811 goto exit;
812 }
813
814 exit:
815 return err;
816 }
817
818 extern "C" boolean_t root_is_CF_drive;
819
820 void
821 IOOpenDebugDataFile(const char *fname, uint64_t size)
822 {
823 IOReturn err;
824 OSData * imagePath = NULL;
825 uint64_t padding;
826
827 if (!gDebugImageLock) {
828 gDebugImageLock = IOLockAlloc();
829 }
830
831 if (root_is_CF_drive) return;
832
833 // Try to get a lock, but don't block for getting lock
834 if (!IOLockTryLock(gDebugImageLock)) {
835 HIBLOG("IOOpenDebugDataFile: Failed to get lock\n");
836 return;
837 }
838
839 if (gDebugImageFileVars || !fname || !size) {
840 HIBLOG("IOOpenDebugDataFile: conditions failed\n");
841 goto exit;
842 }
843
844 padding = (PAGE_SIZE*2); // allocate couple more pages for header and fileextents
845 err = IOPolledFileOpen(fname, size+padding, 32ULL*1024*1024*1024,
846 NULL, 0,
847 &gDebugImageFileVars, &imagePath, NULL, 0);
848
849 if ((kIOReturnSuccess == err) && imagePath)
850 {
851 if ((gDebugImageFileVars->fileSize < (size+padding)) ||
852 (gDebugImageFileVars->fileExtents->getLength() > PAGE_SIZE)) {
853 // Can't use the file
854 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
855 HIBLOG("IOOpenDebugDataFile: too many file extents\n");
856 goto exit;
857 }
858
859 // write extents for debug data usage in EFI
860 IOWriteExtentsToFile(gDebugImageFileVars, kIOHibernateHeaderOpenSignature);
861 IOSetBootImageNVRAM(imagePath);
862 kprintf("IOOpenDebugDataFile: opened debugdata file\n");
863 }
864
865 exit:
866 IOLockUnlock(gDebugImageLock);
867
868 if (imagePath) imagePath->release();
869 return;
870 }
871
872 void
873 IOCloseDebugDataFile()
874 {
875 IOSetBootImageNVRAM(0);
876
877 if (gDebugImageLock) {
878 IOLockLock(gDebugImageLock);
879 if (gDebugImageFileVars != 0) {
880 kprintf("IOHibernateSystemPostWake: Closing debugdata file\n");
881 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
882 }
883 IOLockUnlock(gDebugImageLock);
884 }
885
886
887 }
888
889 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
890
891 DECLARE_IOHIBERNATEPROGRESSALPHA
892
893 static void
894 ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
895 {
896 uint32_t rowBytes, pixelShift;
897 uint32_t x, y;
898 int32_t blob;
899 uint32_t alpha, in, color, result;
900 uint8_t * out;
901 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
902
903 rowBytes = display->rowBytes;
904 pixelShift = display->depth >> 4;
905 if (pixelShift < 1) return;
906
907 screen += ((display->width
908 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
909 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
910
911 for (y = 0; y < kIOHibernateProgressHeight; y++)
912 {
913 out = screen + y * rowBytes;
914 for (blob = 0; blob < kIOHibernateProgressCount; blob++)
915 {
916 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
917 for (x = 0; x < kIOHibernateProgressWidth; x++)
918 {
919 alpha = gIOHibernateProgressAlpha[y][x];
920 result = color;
921 if (alpha)
922 {
923 if (0xff != alpha)
924 {
925 if (1 == pixelShift)
926 {
927 in = *((uint16_t *)out) & 0x1f; // 16
928 in = (in << 3) | (in >> 2);
929 }
930 else
931 in = *((uint32_t *)out) & 0xff; // 32
932 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
933 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
934 }
935 if (1 == pixelShift)
936 {
937 result >>= 3;
938 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
939 }
940 else
941 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
942 }
943 out += (1 << pixelShift);
944 }
945 out += (kIOHibernateProgressSpacing << pixelShift);
946 }
947 }
948 }
949
950
951 static void
952 ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
953 {
954 uint32_t rowBytes, pixelShift;
955 uint32_t x, y;
956 int32_t blob, lastBlob;
957 uint32_t alpha, in, color, result;
958 uint8_t * out;
959 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
960
961 pixelShift = display->depth >> 4;
962 if (pixelShift < 1)
963 return;
964
965 rowBytes = display->rowBytes;
966
967 screen += ((display->width
968 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
969 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
970
971 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
972
973 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
974
975 for (y = 0; y < kIOHibernateProgressHeight; y++)
976 {
977 out = screen + y * rowBytes;
978 for (blob = firstBlob; blob <= lastBlob; blob++)
979 {
980 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
981 for (x = 0; x < kIOHibernateProgressWidth; x++)
982 {
983 alpha = gIOHibernateProgressAlpha[y][x];
984 result = color;
985 if (alpha)
986 {
987 if (0xff != alpha)
988 {
989 in = display->progressSaveUnder[blob][saveindex[blob]++];
990 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
991 }
992 if (1 == pixelShift)
993 {
994 result >>= 3;
995 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
996 }
997 else
998 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
999 }
1000 out += (1 << pixelShift);
1001 }
1002 out += (kIOHibernateProgressSpacing << pixelShift);
1003 }
1004 }
1005 }
1006
1007 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1008
1009 IOReturn
1010 IOHibernateIOKitSleep(void)
1011 {
1012 IOReturn ret = kIOReturnSuccess;
1013 IOLockLock(gFSLock);
1014 if (kFSOpening == gFSState)
1015 {
1016 gFSState = kFSTimedOut;
1017 HIBLOG("hibernate file open timed out\n");
1018 ret = kIOReturnTimeout;
1019 }
1020 IOLockUnlock(gFSLock);
1021 return (ret);
1022 }
1023
1024 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1025
1026 IOReturn
1027 IOHibernateSystemHasSlept(void)
1028 {
1029 IOReturn ret = kIOReturnSuccess;
1030 IOHibernateVars * vars = &gIOHibernateVars;
1031 OSObject * obj = 0;
1032 OSData * data;
1033
1034 IOLockLock(gFSLock);
1035 if ((kFSOpened != gFSState) && gIOHibernateMode)
1036 {
1037 ret = kIOReturnTimeout;
1038 }
1039 IOLockUnlock(gFSLock);
1040 if (kIOReturnSuccess != ret) return (ret);
1041
1042 if (gIOHibernateMode) obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
1043 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1044 if (obj && !vars->previewBuffer)
1045 obj->release();
1046
1047 vars->consoleMapping = NULL;
1048 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
1049 {
1050 vars->previewBuffer->release();
1051 vars->previewBuffer = 0;
1052 }
1053
1054 if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1055 && vars->previewBuffer
1056 && (data = OSDynamicCast(OSData,
1057 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
1058 {
1059 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
1060 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
1061
1062 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1063
1064 if (kIOHibernatePreviewUpdates & flags)
1065 {
1066 PE_Video consoleInfo;
1067 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1068
1069 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1070
1071 graphicsInfo->width = consoleInfo.v_width;
1072 graphicsInfo->height = consoleInfo.v_height;
1073 graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
1074 graphicsInfo->depth = consoleInfo.v_depth;
1075 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1076
1077 HIBPRINT("video %p %d %d %d\n",
1078 vars->consoleMapping, graphicsInfo->depth,
1079 graphicsInfo->width, graphicsInfo->height);
1080 if (vars->consoleMapping)
1081 ProgressInit(graphicsInfo, vars->consoleMapping,
1082 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1083 }
1084 }
1085
1086 if (gIOOptionsEntry)
1087 gIOOptionsEntry->sync();
1088
1089 return (ret);
1090 }
1091
1092 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1093
1094 static DeviceTreeNode *
1095 MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry)
1096 {
1097 DeviceTreeNodeProperty * prop;
1098 DeviceTreeNode * child;
1099 IORegistryEntry * childRegEntry;
1100 const char * nameProp;
1101 unsigned int propLen, idx;
1102
1103 prop = (DeviceTreeNodeProperty *) (entry + 1);
1104 for (idx = 0; idx < entry->nProperties; idx++)
1105 {
1106 if (regEntry && (0 != strcmp("name", prop->name)))
1107 {
1108 regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
1109 // HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
1110 }
1111 prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
1112 }
1113
1114 child = (DeviceTreeNode *) prop;
1115 for (idx = 0; idx < entry->nChildren; idx++)
1116 {
1117 if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen))
1118 panic("no name");
1119 childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
1120 // HIBPRINT("%s == %p\n", nameProp, childRegEntry);
1121 child = MergeDeviceTree(child, childRegEntry);
1122 }
1123 return (child);
1124 }
1125
1126 IOReturn
1127 IOHibernateSystemWake(void)
1128 {
1129 if (kFSOpened == gFSState)
1130 {
1131 IOPolledFilePollersClose(gIOHibernateVars.fileVars, kIOPolledPostflightState);
1132 IOHibernateDone(&gIOHibernateVars);
1133 }
1134 else
1135 {
1136 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1137 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1138 }
1139 return (kIOReturnSuccess);
1140 }
1141
1142 static IOReturn
1143 IOHibernateDone(IOHibernateVars * vars)
1144 {
1145 hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
1146
1147 if (vars->videoMapping)
1148 {
1149 if (vars->videoMapSize)
1150 // remove mappings
1151 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1152 if (vars->videoAllocSize)
1153 // dealloc range
1154 kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
1155 }
1156
1157 if (vars->previewBuffer)
1158 {
1159 vars->previewBuffer->release();
1160 vars->previewBuffer = 0;
1161 }
1162
1163 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1164 {
1165 IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
1166 gIOHibernateCurrentHeader->options, 32);
1167 }
1168 else
1169 {
1170 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1171 }
1172
1173 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1174 && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus))
1175 {
1176 IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
1177 &gIOHibernateGraphicsInfo->gfxStatus,
1178 sizeof(gIOHibernateGraphicsInfo->gfxStatus));
1179 }
1180 else
1181 {
1182 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1183 }
1184
1185 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1186
1187 #if defined(__i386__) || defined(__x86_64__)
1188 IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
1189 IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
1190
1191 /*
1192 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1193 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1194 */
1195 if (gIOOptionsEntry) {
1196
1197 if (gIOHibernateRTCVariablesKey) {
1198 if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
1199 gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
1200 }
1201 }
1202
1203 if (gIOHibernateBootNextKey)
1204 {
1205 if (gIOHibernateBootNextSave)
1206 {
1207 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
1208 gIOHibernateBootNextSave->release();
1209 gIOHibernateBootNextSave = NULL;
1210 }
1211 else
1212 gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
1213 }
1214 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) gIOOptionsEntry->sync();
1215 }
1216 #endif
1217
1218 if (vars->srcBuffer) vars->srcBuffer->release();
1219 bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
1220 if (vars->handoffBuffer)
1221 {
1222 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1223 {
1224 IOHibernateHandoff * handoff;
1225 bool done = false;
1226 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
1227 !done;
1228 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
1229 {
1230 HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
1231 uint8_t * data = &handoff->data[0];
1232 switch (handoff->type)
1233 {
1234 case kIOHibernateHandoffTypeEnd:
1235 done = true;
1236 break;
1237
1238 case kIOHibernateHandoffTypeDeviceTree:
1239 MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
1240 break;
1241
1242 case kIOHibernateHandoffTypeKeyStore:
1243 #if defined(__i386__) || defined(__x86_64__)
1244 {
1245 IOBufferMemoryDescriptor *
1246 md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
1247 if (md)
1248 {
1249 IOSetKeyStoreData(md);
1250 }
1251 }
1252 #endif
1253 break;
1254
1255 default:
1256 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
1257 break;
1258 }
1259 }
1260 }
1261 vars->handoffBuffer->release();
1262 }
1263
1264 bzero(vars, sizeof(*vars));
1265
1266 // gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1267
1268 return (kIOReturnSuccess);
1269 }
1270
1271 IOReturn
1272 IOHibernateSystemPostWake(void)
1273 {
1274 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1275 IOLockLock(gFSLock);
1276 if (kFSOpened == gFSState)
1277 {
1278 // invalidate & close the image file
1279 IOSleep(TRIM_DELAY);
1280 IOPolledFileIOVars * vars = &gFileVars;
1281 IOPolledFileClose(&vars,
1282 #if DISABLE_TRIM
1283 0, NULL, 0, 0, 0);
1284 #else
1285 0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader),
1286 sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize);
1287 #endif
1288 }
1289 gFSState = kFSIdle;
1290
1291 IOLockUnlock(gFSLock);
1292
1293 // IOCloseDebugDataFile() calls IOSetBootImageNVRAM() unconditionally
1294 IOCloseDebugDataFile( );
1295 return (kIOReturnSuccess);
1296 }
1297
1298 bool IOHibernateWasScreenLocked(void)
1299 {
1300 bool ret = false;
1301 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry)
1302 {
1303 OSData *
1304 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
1305 if (data) switch (*((uint32_t *)data->getBytesNoCopy()))
1306 {
1307 case kIOScreenLockLocked:
1308 case kIOScreenLockFileVaultDialog:
1309 ret = true;
1310 break;
1311 case kIOScreenLockNoLock:
1312 case kIOScreenLockUnlocked:
1313 default:
1314 ret = false;
1315 break;
1316 }
1317 }
1318 return (ret);
1319 }
1320
1321 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1322
1323 SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
1324 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1325 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
1326 SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
1327 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1328 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
1329 SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
1330 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1331 &gIOHibernateMode, 0, "");
1332 SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
1333 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1334 &_hibernateStats, hibernate_statistics_t, "");
1335
1336 SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
1337 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1338 &_hibernateStats.graphicsReadyTime, 0, "");
1339 SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
1340 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1341 &_hibernateStats.wakeNotificationTime, 0, "");
1342 SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
1343 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1344 &_hibernateStats.lockScreenReadyTime, 0, "");
1345 SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
1346 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1347 &_hibernateStats.hidReadyTime, 0, "");
1348
1349
1350 void
1351 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1352 {
1353 gIOHibernateBootImageKey = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1354
1355 #if defined(__i386__) || defined(__x86_64__)
1356 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1357 gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
1358 gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
1359 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1360 #endif /* defined(__i386__) || defined(__x86_64__) */
1361
1362 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
1363 if (data)
1364 {
1365 rootDomain->setProperty(kIOHibernateStateKey, data);
1366 data->release();
1367 }
1368
1369 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))
1370 gIOHibernateMode = kIOHibernateModeOn;
1371 else
1372 gIOHibernateFilename[0] = 0;
1373
1374 sysctl_register_oid(&sysctl__kern_hibernatefile);
1375 sysctl_register_oid(&sysctl__kern_bootsignature);
1376 sysctl_register_oid(&sysctl__kern_hibernatemode);
1377 sysctl_register_oid(&sysctl__kern_hibernatestatistics);
1378 sysctl_register_oid(&sysctl__kern_hibernategraphicsready);
1379 sysctl_register_oid(&sysctl__kern_hibernatewakenotification);
1380 sysctl_register_oid(&sysctl__kern_hibernatelockscreenready);
1381 sysctl_register_oid(&sysctl__kern_hibernatehidready);
1382
1383 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1384
1385 gFSLock = IOLockAlloc();
1386 }
1387
1388 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1389
1390 static IOReturn
1391 IOHibernatePolledFileWrite(IOPolledFileIOVars * vars,
1392 const uint8_t * bytes, IOByteCount size,
1393 IOPolledFileCryptVars * cryptvars)
1394 {
1395 IOReturn err;
1396
1397 err = IOPolledFileWrite(vars, bytes, size, cryptvars);
1398 if ((kIOReturnSuccess == err) && hibernate_should_abort()) err = kIOReturnAborted;
1399
1400 return (err);
1401 }
1402
1403 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1404
1405 extern "C" uint32_t
1406 hibernate_write_image(void)
1407 {
1408 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
1409 IOHibernateVars * vars = &gIOHibernateVars;
1410 IOPolledFileExtent * fileExtents;
1411
1412 _static_assert_1_arg(sizeof(IOHibernateImageHeader) == 512);
1413
1414 uint32_t pageCount, pagesDone;
1415 IOReturn err;
1416 vm_offset_t ppnum, page;
1417 IOItemCount count;
1418 uint8_t * src;
1419 uint8_t * data;
1420 uint8_t * compressed;
1421 uint8_t * scratch;
1422 IOByteCount pageCompressedSize;
1423 uint64_t compressedSize, uncompressedSize;
1424 uint64_t image1Size = 0;
1425 uint32_t bitmap_size;
1426 bool iterDone, pollerOpen, needEncrypt;
1427 uint32_t restore1Sum, sum, sum1, sum2;
1428 int wkresult;
1429 uint32_t tag;
1430 uint32_t pageType;
1431 uint32_t pageAndCount[2];
1432 addr64_t phys64;
1433 IOByteCount segLen;
1434 uintptr_t hibernateBase;
1435 uintptr_t hibernateEnd;
1436
1437 AbsoluteTime startTime, endTime;
1438 AbsoluteTime allTime, compTime;
1439 uint64_t compBytes;
1440 uint64_t nsec;
1441 uint32_t lastProgressStamp = 0;
1442 uint32_t progressStamp;
1443 uint32_t blob, lastBlob = (uint32_t) -1L;
1444
1445 uint32_t wiredPagesEncrypted;
1446 uint32_t dirtyPagesEncrypted;
1447 uint32_t wiredPagesClear;
1448 uint32_t svPageCount;
1449 uint32_t zvPageCount;
1450
1451 IOPolledFileCryptVars _cryptvars;
1452 IOPolledFileCryptVars * cryptvars = 0;
1453
1454 wiredPagesEncrypted = 0;
1455 dirtyPagesEncrypted = 0;
1456 wiredPagesClear = 0;
1457 svPageCount = 0;
1458 zvPageCount = 0;
1459
1460 if (!vars->fileVars
1461 || !vars->fileVars->pollers
1462 || !(kIOHibernateModeOn & gIOHibernateMode)) return (kIOHibernatePostWriteSleep);
1463
1464 if (kIOHibernateModeSleep & gIOHibernateMode)
1465 kdebug_enable = save_kdebug_enable;
1466
1467 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START, 0, 0, 0, 0, 0);
1468 IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
1469
1470 restore1Sum = sum1 = sum2 = 0;
1471
1472 #if CRYPTO
1473 // encryption data. "iv" is the "initial vector".
1474 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1475 {
1476 static const unsigned char first_iv[AES_BLOCK_SIZE]
1477 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
1478 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
1479
1480 cryptvars = &gIOHibernateCryptWakeContext;
1481 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1482 aes_encrypt_key(vars->cryptKey,
1483 kIOHibernateAESKeySize,
1484 &cryptvars->ctx.encrypt);
1485 aes_decrypt_key(vars->cryptKey,
1486 kIOHibernateAESKeySize,
1487 &cryptvars->ctx.decrypt);
1488
1489 cryptvars = &_cryptvars;
1490 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1491 for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++)
1492 vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
1493 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
1494 aes_encrypt_key(vars->wiredCryptKey,
1495 kIOHibernateAESKeySize,
1496 &cryptvars->ctx.encrypt);
1497
1498 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1499 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1500 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
1501 }
1502 #endif /* CRYPTO */
1503
1504 hibernate_page_list_setall(vars->page_list,
1505 vars->page_list_wired,
1506 vars->page_list_pal,
1507 false /* !preflight */,
1508 /* discard_all */
1509 ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
1510 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
1511 &pageCount);
1512
1513 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
1514
1515 fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy();
1516
1517 #if 0
1518 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
1519 for (page = 0; page < count; page++)
1520 {
1521 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
1522 fileExtents[page].start, fileExtents[page].length,
1523 fileExtents[page].start + fileExtents[page].length);
1524 }
1525 #endif
1526
1527 needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
1528 AbsoluteTime_to_scalar(&compTime) = 0;
1529 compBytes = 0;
1530
1531 clock_get_uptime(&allTime);
1532 IOService::getPMRootDomain()->pmStatsRecordEvent(
1533 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
1534 do
1535 {
1536 compressedSize = 0;
1537 uncompressedSize = 0;
1538 svPageCount = 0;
1539 zvPageCount = 0;
1540
1541 IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
1542
1543 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
1544 ml_get_interrupts_enabled());
1545 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState, true);
1546 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
1547 pollerOpen = (kIOReturnSuccess == err);
1548 if (!pollerOpen)
1549 break;
1550
1551 // copy file block extent list if larger than header
1552
1553 count = vars->fileVars->fileExtents->getLength();
1554 if (count > sizeof(header->fileExtentMap))
1555 {
1556 count -= sizeof(header->fileExtentMap);
1557 err = IOHibernatePolledFileWrite(vars->fileVars,
1558 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
1559 if (kIOReturnSuccess != err)
1560 break;
1561 }
1562
1563 hibernateBase = HIB_BASE; /* Defined in PAL headers */
1564 hibernateEnd = (segHIBB + segSizeHIB);
1565
1566 // copy out restore1 code
1567
1568 for (count = 0;
1569 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1570 count += segLen)
1571 {
1572 for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++)
1573 {
1574 gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
1575 }
1576 }
1577
1578 page = atop_32(kvtophys(hibernateBase));
1579 count = atop_32(round_page(hibernateEnd) - hibernateBase);
1580 header->restore1CodePhysPage = page;
1581 header->restore1CodeVirt = hibernateBase;
1582 header->restore1PageCount = count;
1583 header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
1584 header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
1585
1586 // sum __HIB seg, with zeros for the stack
1587 src = (uint8_t *) trunc_page(hibernateBase);
1588 for (page = 0; page < count; page++)
1589 {
1590 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
1591 restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
1592 else
1593 restore1Sum += 0x00000000;
1594 src += page_size;
1595 }
1596 sum1 = restore1Sum;
1597
1598 // write the __HIB seg, with zeros for the stack
1599
1600 src = (uint8_t *) trunc_page(hibernateBase);
1601 count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
1602 if (count)
1603 {
1604 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
1605 if (kIOReturnSuccess != err)
1606 break;
1607 }
1608 err = IOHibernatePolledFileWrite(vars->fileVars,
1609 (uint8_t *) 0,
1610 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
1611 cryptvars);
1612 if (kIOReturnSuccess != err)
1613 break;
1614 src = &gIOHibernateRestoreStackEnd[0];
1615 count = round_page(hibernateEnd) - ((uintptr_t) src);
1616 if (count)
1617 {
1618 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
1619 if (kIOReturnSuccess != err)
1620 break;
1621 }
1622
1623 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1624 {
1625 vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
1626 vars->fileVars->encryptEnd = UINT64_MAX;
1627 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1628 }
1629
1630 // write the preview buffer
1631
1632 if (vars->previewBuffer)
1633 {
1634 ppnum = 0;
1635 count = 0;
1636 do
1637 {
1638 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
1639 pageAndCount[0] = atop_64(phys64);
1640 pageAndCount[1] = atop_32(segLen);
1641 err = IOHibernatePolledFileWrite(vars->fileVars,
1642 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1643 cryptvars);
1644 if (kIOReturnSuccess != err)
1645 break;
1646 count += segLen;
1647 ppnum += sizeof(pageAndCount);
1648 }
1649 while (phys64);
1650 if (kIOReturnSuccess != err)
1651 break;
1652
1653 src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
1654
1655 ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
1656
1657 count = vars->previewBuffer->getLength();
1658
1659 header->previewPageListSize = ppnum;
1660 header->previewSize = count + ppnum;
1661
1662 for (page = 0; page < count; page += page_size)
1663 {
1664 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
1665 sum1 += hibernate_sum_page(src + page, atop_64(phys64));
1666 }
1667 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
1668 if (kIOReturnSuccess != err)
1669 break;
1670 }
1671
1672 // mark areas for no save
1673 IOMemoryDescriptor * ioBuffer;
1674 ioBuffer = IOPolledFileGetIOBuffer(vars->fileVars);
1675 for (count = 0;
1676 (phys64 = ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1677 count += segLen)
1678 {
1679 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1680 atop_64(phys64), atop_32(segLen),
1681 kIOHibernatePageStateFree);
1682 pageCount -= atop_32(segLen);
1683 }
1684
1685 for (count = 0;
1686 (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1687 count += segLen)
1688 {
1689 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1690 atop_64(phys64), atop_32(segLen),
1691 kIOHibernatePageStateFree);
1692 pageCount -= atop_32(segLen);
1693 }
1694
1695 // copy out bitmap of pages available for trashing during restore
1696
1697 bitmap_size = vars->page_list_wired->list_size;
1698 src = (uint8_t *) vars->page_list_wired;
1699 err = IOHibernatePolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
1700 if (kIOReturnSuccess != err)
1701 break;
1702
1703 // mark more areas for no save, but these are not available
1704 // for trashing during restore
1705
1706 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
1707
1708
1709 page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
1710 count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
1711 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1712 page, count,
1713 kIOHibernatePageStateFree);
1714 pageCount -= count;
1715
1716 if (vars->previewBuffer) for (count = 0;
1717 (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1718 count += segLen)
1719 {
1720 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1721 atop_64(phys64), atop_32(segLen),
1722 kIOHibernatePageStateFree);
1723 pageCount -= atop_32(segLen);
1724 }
1725
1726 for (count = 0;
1727 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1728 count += segLen)
1729 {
1730 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1731 atop_64(phys64), atop_32(segLen),
1732 kIOHibernatePageStateFree);
1733 pageCount -= atop_32(segLen);
1734 }
1735
1736 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
1737 compressed = src + page_size;
1738 scratch = compressed + page_size;
1739
1740 pagesDone = 0;
1741 lastBlob = 0;
1742
1743 HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
1744 bitmap_size, header->previewSize,
1745 pageCount, vars->fileVars->position);
1746
1747 enum
1748 // pageType
1749 {
1750 kWired = 0x02,
1751 kEncrypt = 0x01,
1752 kWiredEncrypt = kWired | kEncrypt,
1753 kWiredClear = kWired,
1754 kUnwiredEncrypt = kEncrypt
1755 };
1756
1757 bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features()));
1758 #define _pmap_is_noencrypt(x) (cpuAES ? false : pmap_is_noencrypt((x)))
1759
1760 for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)
1761 {
1762 if (kUnwiredEncrypt == pageType)
1763 {
1764 // start unwired image
1765 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1766 {
1767 vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
1768 vars->fileVars->encryptEnd = UINT64_MAX;
1769 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1770 }
1771 bcopy(&cryptvars->aes_iv[0],
1772 &gIOHibernateCryptWakeContext.aes_iv[0],
1773 sizeof(cryptvars->aes_iv));
1774 cryptvars = &gIOHibernateCryptWakeContext;
1775 }
1776 for (iterDone = false, ppnum = 0; !iterDone; )
1777 {
1778 count = hibernate_page_list_iterate((kWired & pageType)
1779 ? vars->page_list_wired : vars->page_list,
1780 &ppnum);
1781 // kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
1782 iterDone = !count;
1783
1784 if (count && (kWired & pageType) && needEncrypt)
1785 {
1786 uint32_t checkIndex;
1787 for (checkIndex = 0;
1788 (checkIndex < count)
1789 && (((kEncrypt & pageType) == 0) == _pmap_is_noencrypt(ppnum + checkIndex));
1790 checkIndex++)
1791 {}
1792 if (!checkIndex)
1793 {
1794 ppnum++;
1795 continue;
1796 }
1797 count = checkIndex;
1798 }
1799
1800 switch (pageType)
1801 {
1802 case kWiredEncrypt: wiredPagesEncrypted += count; break;
1803 case kWiredClear: wiredPagesClear += count; break;
1804 case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
1805 }
1806
1807 if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */}
1808 else
1809 {
1810 pageAndCount[0] = ppnum;
1811 pageAndCount[1] = count;
1812 err = IOHibernatePolledFileWrite(vars->fileVars,
1813 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1814 cryptvars);
1815 if (kIOReturnSuccess != err)
1816 break;
1817 }
1818
1819 for (page = ppnum; page < (ppnum + count); page++)
1820 {
1821 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
1822 if (err)
1823 {
1824 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
1825 break;
1826 }
1827
1828 sum = hibernate_sum_page(src, page);
1829 if (kWired & pageType)
1830 sum1 += sum;
1831 else
1832 sum2 += sum;
1833
1834 clock_get_uptime(&startTime);
1835 wkresult = WKdm_compress_new((const WK_word*) src,
1836 (WK_word*) compressed,
1837 (WK_word*) scratch,
1838 page_size - 4);
1839
1840 clock_get_uptime(&endTime);
1841 ADD_ABSOLUTETIME(&compTime, &endTime);
1842 SUB_ABSOLUTETIME(&compTime, &startTime);
1843
1844 compBytes += page_size;
1845 pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
1846
1847 if (pageCompressedSize == 0)
1848 {
1849 pageCompressedSize = 4;
1850 data = src;
1851
1852 if (*(uint32_t *)src)
1853 svPageCount++;
1854 else
1855 zvPageCount++;
1856 }
1857 else
1858 {
1859 if (pageCompressedSize != page_size)
1860 data = compressed;
1861 else
1862 data = src;
1863 }
1864
1865 tag = pageCompressedSize | kIOHibernateTagSignature;
1866 err = IOHibernatePolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
1867 if (kIOReturnSuccess != err)
1868 break;
1869
1870 err = IOHibernatePolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
1871 if (kIOReturnSuccess != err)
1872 break;
1873
1874 compressedSize += pageCompressedSize;
1875 uncompressedSize += page_size;
1876 pagesDone++;
1877
1878 if (vars->consoleMapping && (0 == (1023 & pagesDone)))
1879 {
1880 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
1881 if (blob != lastBlob)
1882 {
1883 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
1884 lastBlob = blob;
1885 }
1886 }
1887 if (0 == (8191 & pagesDone))
1888 {
1889 clock_get_uptime(&endTime);
1890 SUB_ABSOLUTETIME(&endTime, &allTime);
1891 absolutetime_to_nanoseconds(endTime, &nsec);
1892 progressStamp = nsec / 750000000ULL;
1893 if (progressStamp != lastProgressStamp)
1894 {
1895 lastProgressStamp = progressStamp;
1896 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
1897 }
1898 }
1899 }
1900 if (kIOReturnSuccess != err)
1901 break;
1902 ppnum = page;
1903 }
1904
1905 if (kIOReturnSuccess != err)
1906 break;
1907
1908 if ((kEncrypt & pageType) && vars->fileVars->encryptStart)
1909 {
1910 vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
1911 HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
1912 }
1913
1914 if (kWiredEncrypt != pageType)
1915 {
1916 // end of image1/2 - fill to next block
1917 err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
1918 if (kIOReturnSuccess != err)
1919 break;
1920 }
1921 if (kWiredClear == pageType)
1922 {
1923 // enlarge wired image for test
1924 // err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
1925
1926 // end wired image
1927 header->encryptStart = vars->fileVars->encryptStart;
1928 header->encryptEnd = vars->fileVars->encryptEnd;
1929 image1Size = vars->fileVars->position;
1930 HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
1931 image1Size, header->encryptStart, header->encryptEnd);
1932 }
1933 }
1934 if (kIOReturnSuccess != err)
1935 {
1936 if (kIOReturnOverrun == err)
1937 {
1938 // update actual compression ratio on not enough space (for retry)
1939 gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
1940 }
1941
1942 // update partial amount written (for IOPolledFileClose cleanup/unmap)
1943 header->imageSize = vars->fileVars->position;
1944 break;
1945 }
1946
1947 // Header:
1948
1949 header->imageSize = vars->fileVars->position;
1950 header->image1Size = image1Size;
1951 header->bitmapSize = bitmap_size;
1952 header->pageCount = pageCount;
1953
1954 header->restore1Sum = restore1Sum;
1955 header->image1Sum = sum1;
1956 header->image2Sum = sum2;
1957 header->sleepTime = gIOLastSleepTime.tv_sec;
1958
1959 header->compression = (compressedSize << 8) / uncompressedSize;
1960 gIOHibernateCompression = header->compression;
1961
1962 count = vars->fileVars->fileExtents->getLength();
1963 if (count > sizeof(header->fileExtentMap))
1964 {
1965 header->fileExtentMapSize = count;
1966 count = sizeof(header->fileExtentMap);
1967 }
1968 else
1969 header->fileExtentMapSize = sizeof(header->fileExtentMap);
1970 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
1971
1972 header->deviceBase = vars->fileVars->block0;
1973 header->deviceBlockSize = vars->fileVars->blockSize;
1974
1975 IOPolledFileSeek(vars->fileVars, 0);
1976 err = IOHibernatePolledFileWrite(vars->fileVars,
1977 (uint8_t *) header, sizeof(IOHibernateImageHeader),
1978 cryptvars);
1979 if (kIOReturnSuccess != err)
1980 break;
1981 err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
1982 }
1983 while (false);
1984
1985 clock_get_uptime(&endTime);
1986
1987 IOService::getPMRootDomain()->pmStatsRecordEvent(
1988 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
1989
1990 SUB_ABSOLUTETIME(&endTime, &allTime);
1991 absolutetime_to_nanoseconds(endTime, &nsec);
1992 HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
1993
1994 absolutetime_to_nanoseconds(compTime, &nsec);
1995 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
1996 compBytes,
1997 nsec / 1000000ULL,
1998 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
1999
2000 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2001 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
2002 vars->fileVars->cryptBytes,
2003 nsec / 1000000ULL,
2004 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2005
2006 HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
2007 header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
2008 uncompressedSize, atop_32(uncompressedSize), compressedSize,
2009 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
2010 sum1, sum2);
2011
2012 HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
2013 svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
2014
2015 if (pollerOpen)
2016 IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted );
2017
2018 if (vars->consoleMapping)
2019 ProgressUpdate(gIOHibernateGraphicsInfo,
2020 vars->consoleMapping, 0, kIOHibernateProgressCount);
2021
2022 HIBLOG("hibernate_write_image done(%x)\n", err);
2023
2024 // should we come back via regular wake, set the state in memory.
2025 gIOHibernateState = kIOHibernateStateInactive;
2026
2027 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END,
2028 wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted, 0, 0);
2029
2030 if (kIOReturnSuccess == err)
2031 {
2032 if (kIOHibernateModeSleep & gIOHibernateMode)
2033 {
2034 return (kIOHibernatePostWriteSleep);
2035 }
2036 else if(kIOHibernateModeRestart & gIOHibernateMode)
2037 {
2038 return (kIOHibernatePostWriteRestart);
2039 }
2040 else
2041 {
2042 /* by default, power down */
2043 return (kIOHibernatePostWriteHalt);
2044 }
2045 }
2046 else if (kIOReturnAborted == err)
2047 {
2048 return (kIOHibernatePostWriteWake);
2049 }
2050 else
2051 {
2052 /* on error, sleep */
2053 return (kIOHibernatePostWriteSleep);
2054 }
2055 }
2056
2057 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2058
2059 extern "C" void
2060 hibernate_machine_init(void)
2061 {
2062 IOReturn err;
2063 uint32_t sum;
2064 uint32_t pagesDone;
2065 uint32_t pagesRead = 0;
2066 AbsoluteTime startTime, compTime;
2067 AbsoluteTime allTime, endTime;
2068 AbsoluteTime startIOTime, endIOTime;
2069 uint64_t nsec, nsecIO;
2070 uint64_t compBytes;
2071 uint32_t lastProgressStamp = 0;
2072 uint32_t progressStamp;
2073 IOPolledFileCryptVars * cryptvars = 0;
2074
2075 IOHibernateVars * vars = &gIOHibernateVars;
2076 bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
2077
2078 if (!vars->fileVars || !vars->fileVars->pollers)
2079 return;
2080
2081 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2082 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2083
2084 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
2085 {
2086 HIBLOG("regular wake\n");
2087 return;
2088 }
2089
2090 HIBPRINT("diag %x %x %x %x\n",
2091 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
2092 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
2093
2094 #define t40ms(x) (tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)
2095 #define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
2096 tStat(booterStart, booterStart);
2097 gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart;
2098 tStat(booterDuration0, booterTime0);
2099 tStat(booterDuration1, booterTime1);
2100 tStat(booterDuration2, booterTime2);
2101 tStat(booterDuration, booterTime);
2102 tStat(booterConnectDisplayDuration, connectDisplayTime);
2103 tStat(booterSplashDuration, splashTime);
2104 tStat(trampolineDuration, trampolineTime);
2105
2106 gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size;
2107 gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize;
2108 gIOHibernateStats->image1Pages = pagesDone;
2109
2110 HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
2111 gIOHibernateStats->booterStart,
2112 gIOHibernateStats->smcStart,
2113 gIOHibernateStats->booterDuration0,
2114 gIOHibernateStats->booterDuration1,
2115 gIOHibernateStats->booterDuration2,
2116 gIOHibernateStats->booterDuration,
2117 gIOHibernateStats->booterConnectDisplayDuration,
2118 gIOHibernateStats->booterSplashDuration,
2119 gIOHibernateStats->trampolineDuration);
2120
2121 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
2122 gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
2123 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
2124
2125 if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
2126 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)))
2127 {
2128 hibernate_page_list_discard(vars->page_list);
2129 }
2130
2131 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
2132
2133 if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount)
2134 panic("handoff overflow");
2135
2136 IOHibernateHandoff * handoff;
2137 bool done = false;
2138 bool foundCryptData = false;
2139
2140 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
2141 !done;
2142 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
2143 {
2144 // HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
2145 uint8_t * data = &handoff->data[0];
2146 switch (handoff->type)
2147 {
2148 case kIOHibernateHandoffTypeEnd:
2149 done = true;
2150 break;
2151
2152 case kIOHibernateHandoffTypeGraphicsInfo:
2153 if (handoff->bytecount == sizeof(*gIOHibernateGraphicsInfo))
2154 {
2155 bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
2156 }
2157 break;
2158
2159 case kIOHibernateHandoffTypeCryptVars:
2160 if (cryptvars)
2161 {
2162 hibernate_cryptwakevars_t *
2163 wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
2164 bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
2165 }
2166 foundCryptData = true;
2167 bzero(data, handoff->bytecount);
2168 break;
2169
2170 case kIOHibernateHandoffTypeMemoryMap:
2171
2172 clock_get_uptime(&allTime);
2173
2174 hibernate_newruntime_map(data, handoff->bytecount,
2175 gIOHibernateCurrentHeader->systemTableOffset);
2176
2177 clock_get_uptime(&endTime);
2178
2179 SUB_ABSOLUTETIME(&endTime, &allTime);
2180 absolutetime_to_nanoseconds(endTime, &nsec);
2181
2182 HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
2183
2184 break;
2185
2186 case kIOHibernateHandoffTypeDeviceTree:
2187 {
2188 // DTEntry chosen = NULL;
2189 // HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen));
2190 }
2191 break;
2192
2193 default:
2194 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
2195 break;
2196 }
2197 }
2198 if (cryptvars && !foundCryptData)
2199 panic("hibernate handoff");
2200
2201 HIBPRINT("video 0x%llx %d %d %d status %x\n",
2202 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2203 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
2204
2205 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)
2206 {
2207 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2208 * gIOHibernateGraphicsInfo->rowBytes);
2209 if (vars->videoMapSize > vars->videoAllocSize) vars->videoMapSize = 0;
2210 else
2211 {
2212 IOMapPages(kernel_map,
2213 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2214 vars->videoMapSize, kIOMapInhibitCache );
2215 }
2216 }
2217
2218 if (vars->videoMapSize)
2219 ProgressUpdate(gIOHibernateGraphicsInfo,
2220 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2221
2222 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2223 uint8_t * compressed = src + page_size;
2224 uint8_t * scratch = compressed + page_size;
2225 uint32_t decoOffset;
2226
2227 clock_get_uptime(&allTime);
2228 AbsoluteTime_to_scalar(&compTime) = 0;
2229 compBytes = 0;
2230
2231 HIBLOG("IOPolledFilePollersOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2232 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledAfterSleepState, false);
2233 clock_get_uptime(&startIOTime);
2234 endTime = startIOTime;
2235 SUB_ABSOLUTETIME(&endTime, &allTime);
2236 absolutetime_to_nanoseconds(endTime, &nsec);
2237 HIBLOG("IOPolledFilePollersOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
2238
2239 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2240
2241 // kick off the read ahead
2242 vars->fileVars->bufferHalf = 0;
2243 vars->fileVars->bufferLimit = 0;
2244 vars->fileVars->lastRead = 0;
2245 vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
2246 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2247 vars->fileVars->cryptBytes = 0;
2248 AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
2249
2250 err = IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
2251 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2252 // --
2253
2254 HIBLOG("hibernate_machine_init reading\n");
2255
2256 uint32_t * header = (uint32_t *) src;
2257 sum = 0;
2258
2259 while (kIOReturnSuccess == err)
2260 {
2261 unsigned int count;
2262 unsigned int page;
2263 uint32_t tag;
2264 vm_offset_t ppnum, compressedSize;
2265
2266 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2267 if (kIOReturnSuccess != err)
2268 break;
2269
2270 ppnum = header[0];
2271 count = header[1];
2272
2273 // HIBPRINT("(%x, %x)\n", ppnum, count);
2274
2275 if (!count)
2276 break;
2277
2278 for (page = 0; page < count; page++)
2279 {
2280 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2281 if (kIOReturnSuccess != err)
2282 break;
2283
2284 compressedSize = kIOHibernateTagLength & tag;
2285 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength))
2286 {
2287 err = kIOReturnIPCError;
2288 break;
2289 }
2290
2291 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2292 if (kIOReturnSuccess != err) break;
2293
2294 if (compressedSize < page_size)
2295 {
2296 decoOffset = page_size;
2297 clock_get_uptime(&startTime);
2298
2299 if (compressedSize == 4) {
2300 int i;
2301 uint32_t *s, *d;
2302
2303 s = (uint32_t *)src;
2304 d = (uint32_t *)(uintptr_t)compressed;
2305
2306 for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++)
2307 *d++ = *s;
2308 }
2309 else
2310 WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, compressedSize);
2311 clock_get_uptime(&endTime);
2312 ADD_ABSOLUTETIME(&compTime, &endTime);
2313 SUB_ABSOLUTETIME(&compTime, &startTime);
2314 compBytes += page_size;
2315 }
2316 else decoOffset = 0;
2317
2318 sum += hibernate_sum_page((src + decoOffset), ppnum);
2319 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2320 if (err)
2321 {
2322 HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
2323 break;
2324 }
2325
2326 ppnum++;
2327 pagesDone++;
2328 pagesRead++;
2329
2330 if (0 == (8191 & pagesDone))
2331 {
2332 clock_get_uptime(&endTime);
2333 SUB_ABSOLUTETIME(&endTime, &allTime);
2334 absolutetime_to_nanoseconds(endTime, &nsec);
2335 progressStamp = nsec / 750000000ULL;
2336 if (progressStamp != lastProgressStamp)
2337 {
2338 lastProgressStamp = progressStamp;
2339 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2340 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2341 }
2342 }
2343 }
2344 }
2345 if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages))
2346 err = kIOReturnLockedRead;
2347
2348 if (kIOReturnSuccess != err)
2349 panic("Hibernate restore error %x", err);
2350
2351 gIOHibernateCurrentHeader->actualImage2Sum = sum;
2352 gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
2353
2354 clock_get_uptime(&endIOTime);
2355
2356 err = IOPolledFilePollersClose(vars->fileVars, kIOPolledAfterSleepState);
2357
2358 clock_get_uptime(&endTime);
2359
2360 IOService::getPMRootDomain()->pmStatsRecordEvent(
2361 kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
2362 IOService::getPMRootDomain()->pmStatsRecordEvent(
2363 kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
2364
2365 SUB_ABSOLUTETIME(&endTime, &allTime);
2366 absolutetime_to_nanoseconds(endTime, &nsec);
2367
2368 SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
2369 absolutetime_to_nanoseconds(endIOTime, &nsecIO);
2370
2371 gIOHibernateStats->kernelImageReadDuration = nsec / 1000000ULL;
2372 gIOHibernateStats->imagePages = pagesDone;
2373
2374 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
2375 pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
2376 nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
2377
2378 absolutetime_to_nanoseconds(compTime, &nsec);
2379 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2380 compBytes,
2381 nsec / 1000000ULL,
2382 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2383
2384 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2385 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
2386 vars->fileVars->cryptBytes,
2387 nsec / 1000000ULL,
2388 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2389
2390 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_NONE, pagesRead, pagesDone, 0, 0, 0);
2391 }
2392
2393 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2394
2395 void IOHibernateSetWakeCapabilities(uint32_t capability)
2396 {
2397 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
2398 {
2399 gIOHibernateStats->wakeCapability = capability;
2400
2401 if (kIOPMSystemCapabilityGraphics & capability)
2402 {
2403 vm_compressor_do_warmup();
2404 }
2405 }
2406 }
2407
2408 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2409
2410 void IOHibernateSystemRestart(void)
2411 {
2412 static uint8_t noteStore[32] __attribute__((aligned(32)));
2413 IORegistryEntry * regEntry;
2414 const OSSymbol * sym;
2415 OSData * noteProp;
2416 OSData * data;
2417 uintptr_t * smcVars;
2418 uint8_t * smcBytes;
2419 size_t len;
2420 addr64_t element;
2421
2422 data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
2423 if (!data) return;
2424
2425 smcVars = (typeof(smcVars)) data->getBytesNoCopy();
2426 smcBytes = (typeof(smcBytes)) smcVars[1];
2427 len = smcVars[0];
2428 if (len > sizeof(noteStore)) len = sizeof(noteStore);
2429 noteProp = OSData::withCapacity(3 * sizeof(element));
2430 if (!noteProp) return;
2431 element = len;
2432 noteProp->appendBytes(&element, sizeof(element));
2433 element = crc32(0, smcBytes, len);
2434 noteProp->appendBytes(&element, sizeof(element));
2435
2436 bcopy(smcBytes, noteStore, len);
2437 element = (addr64_t) &noteStore[0];
2438 element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
2439 noteProp->appendBytes(&element, sizeof(element));
2440
2441 if (!gIOOptionsEntry)
2442 {
2443 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
2444 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
2445 if (regEntry && !gIOOptionsEntry)
2446 regEntry->release();
2447 }
2448
2449 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
2450 if (gIOOptionsEntry && sym) gIOOptionsEntry->setProperty(sym, noteProp);
2451 if (noteProp) noteProp->release();
2452 if (sym) sym->release();
2453 }
2454
2455
2456
mirror of XNU