]> git.saurik.com Git - apple/xnu.git/blob - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blob
? search:


2ae11096b7a9253709d2f3b41790a9830f2e72f7
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
1 /*
2 * Copyright (c) 2004-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30
31 Sleep:
32
33 - PMRootDomain calls IOHibernateSystemSleep() before system sleep
34 (devices awake, normal execution context)
35 - IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
36 grabs its extents and searches for a polling driver willing to work with that IOMedia.
37 The BSD code makes an ioctl to the storage driver to get the partition base offset to
38 the disk, and other ioctls to get the transfer constraints
39 If successful, the file is written to make sure its initially not bootable (in case of
40 later failure) and nvram set to point to the first block of the file. (Has to be done
41 here so blocking is possible in nvram support).
42 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
43 page out any pages it wants to (currently zero, but probably some percentage of memory).
44 Its assumed just allocating pages will cause the VM system to naturally select the best
45 pages for eviction. It also copies processor flags needed for the restore path and sets
46 a flag in the boot processor proc info.
47 gIOHibernateState = kIOHibernateStateHibernating.
48 - Regular sleep progresses - some drivers may inspect the root domain property
49 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
50 as usual but leaves motherboard I/O on.
51 - Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
52 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
53 all ppc RC bits out of the hash table and caches into the mapping structures.
54 - hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
55 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
56 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
57 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
58 The image header and block list are written. The header includes the second file extent so
59 only the header block is needed to read the file, regardless of filesystem.
60 The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
61 (only) is used to decompress the image during wake/boot.
62 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
63 The bitmaps are written to the image.
64 More areas are removed from the bitmaps (after they have been written to the image) - the
65 segment "__HIB" pages and interrupt stack.
66 Each wired page is compressed and written and then each non-wired page. Compression and
67 disk writes are in parallel.
68 The image header is written to the start of the file and the polling driver closed.
69 The machine powers down (or sleeps).
70
71 Boot/Wake:
72
73 - BootX sees the boot-image nvram variable containing the device and block number of the image,
74 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
75 - BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
76 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
77 that is in the kernel's __HIB section.
78 - BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
79 hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
80 only code & data in that section is safe to call since all the other wired pages are still
81 compressed in the image.
82 - hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
83 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
84 location directly, and copies those that can't to interim free pages. When the image has been
85 completed, the copies are uncompressed, overwriting the wired image pages.
86 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
87 is used to get pages into place for 64bit.
88 - the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
89 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
90 are removed from the software strutures, and the hash table is reinitialized.
91 - After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
92 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
93 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
94 for the remaining non wired pages.
95 - The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
96 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
97 is closed via bsd.
98
99 Polled Mode I/O:
100
101 IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
102 registry, specifying an object of calls IOPolledInterface.
103
104 Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
105 partition) that the image is going to live, looking for polled interface properties. If it finds
106 one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
107 interfaces found are kept in an ordered list.
108
109 There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
110 few different contexts things happen in:
111
112 - there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
113 up and running) and after wake - this is safe to allocate memory and do anything. The device
114 ignores sleep requests from that point since its a waste of time if it goes to sleep and
115 immediately wakes back up for the image write.
116
117 - there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
118 immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
119 from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
120 used to flush and set the disk to sleep.
121
122 - there is an Open/Close (AfterSleep) pair made around the image read operations that happen
123 immediately after sleep. These can't block or allocate memory. This is happening after the platform
124 expert has woken the low level bits of the system, but most of the I/O system has not. There is only
125 one thread running.
126
127 For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
128 (prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
129 the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
130 Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
131 that is called for the hardware to check for events, and complete the I/O via the callback.
132 The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
133 to restrict I/O ops.
134 */
135
136 #include <sys/systm.h>
137
138 #include <IOKit/IOWorkLoop.h>
139 #include <IOKit/IOCommandGate.h>
140 #include <IOKit/IOTimerEventSource.h>
141 #include <IOKit/IOPlatformExpert.h>
142 #include <IOKit/IOKitDebug.h>
143 #include <IOKit/IOTimeStamp.h>
144 #include <IOKit/pwr_mgt/RootDomain.h>
145 #include <IOKit/pwr_mgt/IOPMPrivate.h>
146 #include <IOKit/IOMessage.h>
147 #include <IOKit/IODeviceTreeSupport.h>
148 #include <IOKit/IOBSD.h>
149 #include "RootDomainUserClient.h"
150 #include <IOKit/pwr_mgt/IOPowerConnection.h>
151 #include "IOPMPowerStateQueue.h"
152 #include <IOKit/IOBufferMemoryDescriptor.h>
153 #include <IOKit/AppleKeyStoreInterface.h>
154 #include <libkern/crypto/aes.h>
155
156 #include <sys/uio.h>
157 #include <sys/conf.h>
158 #include <sys/stat.h>
159 #include <sys/fcntl.h> // (FWRITE, ...)
160 #include <sys/sysctl.h>
161 #include <sys/kdebug.h>
162 #include <stdint.h>
163
164 #include <IOKit/IOHibernatePrivate.h>
165 #include <IOKit/IOPolledInterface.h>
166 #include <IOKit/IONVRAM.h>
167 #include "IOHibernateInternal.h"
168 #include <vm/WKdm_new.h>
169 #include <vm/vm_protos.h>
170 #include "IOKitKernelInternal.h"
171 #include <pexpert/device_tree.h>
172
173 #include <machine/pal_routines.h>
174 #include <machine/pal_hibernate.h>
175 #include <i386/tsc.h>
176 #include <i386/cpuid.h>
177
178 extern "C" addr64_t kvtophys(vm_offset_t va);
179 extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
180
181 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
182
183 #define DISABLE_TRIM 0
184 #define TRIM_DELAY 25000
185
186 extern unsigned int save_kdebug_enable;
187 extern uint32_t gIOHibernateState;
188 uint32_t gIOHibernateMode;
189 static char gIOHibernateBootSignature[256+1];
190 static char gIOHibernateFilename[MAXPATHLEN+1];
191 static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
192 uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
193 static uint64_t gIOHibernateCompression = 0x80; // default compression 50%
194 boolean_t gIOHibernateStandbyDisabled;
195
196 static IODTNVRAM * gIOOptionsEntry;
197 static IORegistryEntry * gIOChosenEntry;
198
199 static const OSSymbol * gIOHibernateBootImageKey;
200
201 #if defined(__i386__) || defined(__x86_64__)
202
203 static const OSSymbol * gIOHibernateRTCVariablesKey;
204 static const OSSymbol * gIOHibernateBoot0082Key;
205 static const OSSymbol * gIOHibernateBootNextKey;
206 static OSData * gIOHibernateBoot0082Data;
207 static OSData * gIOHibernateBootNextData;
208 static OSObject * gIOHibernateBootNextSave;
209
210 static IOPolledFileIOVars * gDebugImageFileVars;
211 static IOLock * gDebugImageLock;
212
213 #endif /* defined(__i386__) || defined(__x86_64__) */
214
215 static IOLock * gFSLock;
216 static uint32_t gFSState;
217 static IOPolledFileIOVars gFileVars;
218 static IOHibernateVars gIOHibernateVars;
219 static IOPolledFileCryptVars gIOHibernateCryptWakeContext;
220 static hibernate_graphics_t _hibernateGraphics;
221 static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
222 static hibernate_statistics_t _hibernateStats;
223 static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats;
224
225 enum
226 {
227 kFSIdle = 0,
228 kFSOpening = 2,
229 kFSOpened = 3,
230 kFSTimedOut = 4,
231 };
232
233 static IOReturn IOHibernateDone(IOHibernateVars * vars);
234 static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature);
235 static void IOSetBootImageNVRAM(OSData * data);
236
237 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
238
239 enum { kDefaultIOSize = 128 * 1024 };
240 enum { kVideoMapSize = 80 * 1024 * 1024 };
241
242 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
243
244 // copy from phys addr to MD
245
246 static IOReturn
247 IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
248 IOByteCount offset, addr64_t bytes, IOByteCount length)
249 {
250 addr64_t srcAddr = bytes;
251 IOByteCount remaining;
252
253 remaining = length = min(length, md->getLength() - offset);
254 while (remaining) { // (process another target segment?)
255 addr64_t dstAddr64;
256 IOByteCount dstLen;
257
258 dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
259 if (!dstAddr64)
260 break;
261
262 // Clip segment length to remaining
263 if (dstLen > remaining)
264 dstLen = remaining;
265
266 #if 1
267 bcopy_phys(srcAddr, dstAddr64, dstLen);
268 #else
269 copypv(srcAddr, dstAddr64, dstLen,
270 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
271 #endif
272 srcAddr += dstLen;
273 offset += dstLen;
274 remaining -= dstLen;
275 }
276
277 assert(!remaining);
278
279 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
280 }
281
282 // copy from MD to phys addr
283
284 static IOReturn
285 IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
286 IOByteCount offset, addr64_t bytes, IOByteCount length)
287 {
288 addr64_t dstAddr = bytes;
289 IOByteCount remaining;
290
291 remaining = length = min(length, md->getLength() - offset);
292 while (remaining) { // (process another target segment?)
293 addr64_t srcAddr64;
294 IOByteCount dstLen;
295
296 srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
297 if (!srcAddr64)
298 break;
299
300 // Clip segment length to remaining
301 if (dstLen > remaining)
302 dstLen = remaining;
303
304 #if 1
305 bcopy_phys(srcAddr64, dstAddr, dstLen);
306 #else
307 copypv(srcAddr, dstAddr64, dstLen,
308 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
309 #endif
310 dstAddr += dstLen;
311 offset += dstLen;
312 remaining -= dstLen;
313 }
314
315 assert(!remaining);
316
317 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
318 }
319
320 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
321
322 void
323 hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
324 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
325 {
326 count += ppnum;
327 switch (kind)
328 {
329 case kIOHibernatePageStateUnwiredSave:
330 // unwired save
331 for (; ppnum < count; ppnum++)
332 {
333 hibernate_page_bitset(page_list, FALSE, ppnum);
334 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
335 }
336 break;
337 case kIOHibernatePageStateWiredSave:
338 // wired save
339 for (; ppnum < count; ppnum++)
340 {
341 hibernate_page_bitset(page_list, FALSE, ppnum);
342 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
343 }
344 break;
345 case kIOHibernatePageStateFree:
346 // free page
347 for (; ppnum < count; ppnum++)
348 {
349 hibernate_page_bitset(page_list, TRUE, ppnum);
350 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
351 }
352 break;
353 default:
354 panic("hibernate_set_page_state");
355 }
356 }
357
358 static vm_offset_t
359 hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
360 {
361 uint32_t page = *pPage;
362 uint32_t count;
363 hibernate_bitmap_t * bitmap;
364
365 while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
366 {
367 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
368 if (!count)
369 break;
370 page += count;
371 if (page <= bitmap->last_page)
372 break;
373 }
374
375 *pPage = page;
376 if (bitmap)
377 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
378 else
379 count = 0;
380
381 return (count);
382 }
383
384 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
385
386 IOReturn
387 IOHibernateSystemSleep(void)
388 {
389 IOReturn err;
390 OSData * nvramData;
391 OSObject * obj;
392 OSString * str;
393 OSNumber * num;
394 bool dsSSD, vmflush, swapPinned;
395 IOHibernateVars * vars;
396 uint64_t setFileSize = 0;
397
398 gIOHibernateState = kIOHibernateStateInactive;
399
400 gIOHibernateDebugFlags = 0;
401 if (kIOLogHibernate & gIOKitDebug)
402 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
403
404 if (IOService::getPMRootDomain()->getHibernateSettings(
405 &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime))
406 {
407 if (kIOHibernateModeSleep & gIOHibernateMode)
408 // default to discard clean for safe sleep
409 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
410 | kIOHibernateModeDiscardCleanActive);
411 }
412
413 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
414 {
415 if ((str = OSDynamicCast(OSString, obj)))
416 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
417 sizeof(gIOHibernateFilename));
418 obj->release();
419 }
420
421 if (!gIOHibernateMode || !gIOHibernateFilename[0])
422 return (kIOReturnUnsupported);
423
424 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
425
426 vars = IONew(IOHibernateVars, 1);
427 if (!vars) return (kIOReturnNoMemory);
428 bzero(vars, sizeof(*vars));
429
430 IOLockLock(gFSLock);
431 if (kFSIdle != gFSState)
432 {
433 HIBLOG("hibernate file busy\n");
434 IOLockUnlock(gFSLock);
435 IODelete(vars, IOHibernateVars, 1);
436 return (kIOReturnBusy);
437 }
438 gFSState = kFSOpening;
439 IOLockUnlock(gFSLock);
440
441 swapPinned = false;
442 do
443 {
444 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
445 2 * page_size + WKdm_SCRATCH_BUF_SIZE_INTERNAL, page_size);
446
447 vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
448 ptoa_64(gIOHibernateHandoffPageCount), page_size);
449
450 if (!vars->srcBuffer || !vars->handoffBuffer)
451 {
452 err = kIOReturnNoMemory;
453 break;
454 }
455
456 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey)))
457 {
458 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMinSize = num->unsigned64BitValue();
459 obj->release();
460 }
461 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey)))
462 {
463 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMaxSize = num->unsigned64BitValue();
464 obj->release();
465 }
466
467 boolean_t encryptedswap = true;
468 uint32_t pageCount;
469 AbsoluteTime startTime, endTime;
470 uint64_t nsec;
471
472 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
473 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
474 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
475
476 vmflush = ((kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
477 err = hibernate_alloc_page_lists(&vars->page_list,
478 &vars->page_list_wired,
479 &vars->page_list_pal);
480 if (KERN_SUCCESS != err) break;
481
482 err = hibernate_pin_swap(TRUE);
483 if (KERN_SUCCESS != err) break;
484 swapPinned = true;
485
486 if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode))
487 {
488 hibernate_page_list_setall(vars->page_list,
489 vars->page_list_wired,
490 vars->page_list_pal,
491 true /* preflight */,
492 vmflush /* discard */,
493 &pageCount);
494 PE_Video consoleInfo;
495 bzero(&consoleInfo, sizeof(consoleInfo));
496 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
497
498 // estimate: 6% increase in pages compressed
499 // screen preview 2 images compressed 0%
500 setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
501 + vars->page_list->list_size
502 + (consoleInfo.v_width * consoleInfo.v_height * 8);
503 enum { setFileRound = 1024*1024ULL };
504 setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
505
506 HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
507 pageCount, (100ULL * gIOHibernateCompression) >> 8,
508 setFileSize, vars->fileMinSize);
509
510 if (!(kIOHibernateModeFileResize & gIOHibernateMode)
511 && (setFileSize < vars->fileMinSize))
512 {
513 setFileSize = vars->fileMinSize;
514 }
515 }
516
517 // Invalidate the image file
518 if (gDebugImageLock) {
519 IOLockLock(gDebugImageLock);
520 if (gDebugImageFileVars != 0) {
521 kprintf("IOHIBSystemSleep: Closing debugdata file\n");
522 IOSetBootImageNVRAM(0);
523 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
524 }
525 IOLockUnlock(gDebugImageLock);
526 }
527
528 err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, 0,
529 gIOHibernateCurrentHeader, sizeof(gIOHibernateCurrentHeader),
530 &vars->fileVars, &nvramData,
531 &vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
532
533 if (KERN_SUCCESS != err)
534 {
535 IOLockLock(gFSLock);
536 if (kFSOpening != gFSState) err = kIOReturnTimeout;
537 IOLockUnlock(gFSLock);
538 }
539
540 if (KERN_SUCCESS != err)
541 {
542 HIBLOG("IOPolledFileOpen(%x)\n", err);
543 break;
544 }
545
546 // write extents for debug data usage in EFI
547 IOWriteExtentsToFile(vars->fileVars, kIOHibernateHeaderOpenSignature);
548
549 err = IOPolledFilePollersSetup(vars->fileVars, kIOPolledPreflightState);
550 if (KERN_SUCCESS != err) break;
551
552 clock_get_uptime(&startTime);
553 err = hibernate_setup(gIOHibernateCurrentHeader,
554 vmflush,
555 vars->page_list, vars->page_list_wired, vars->page_list_pal);
556 clock_get_uptime(&endTime);
557 SUB_ABSOLUTETIME(&endTime, &startTime);
558 absolutetime_to_nanoseconds(endTime, &nsec);
559
560 boolean_t haveSwapPin, hibFileSSD;
561 haveSwapPin = vm_swap_files_pinned();
562
563 hibFileSSD = (kIOPolledFileSSD & vars->fileVars->flags);
564
565 HIBLOG("hibernate_setup(%d) took %qd ms, swapPin(%d) ssd(%d)\n",
566 err, nsec / 1000000ULL,
567 haveSwapPin, hibFileSSD);
568 if (KERN_SUCCESS != err) break;
569
570 gIOHibernateStandbyDisabled = ((!haveSwapPin || !hibFileSSD));
571
572 dsSSD = ((0 != (kIOPolledFileSSD & vars->fileVars->flags))
573 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
574
575 if (dsSSD) gIOHibernateCurrentHeader->options |= kIOHibernateOptionSSD | kIOHibernateOptionColor;
576 else gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
577
578
579 #if defined(__i386__) || defined(__x86_64__)
580 if (!uuid_is_null(vars->volumeCryptKey) &&
581 (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey)))
582 {
583 uintptr_t smcVars[2];
584 smcVars[0] = sizeof(vars->volumeCryptKey);
585 smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
586
587 IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
588 bzero(smcVars, sizeof(smcVars));
589 }
590 #endif
591
592
593 if (encryptedswap || !uuid_is_null(vars->volumeCryptKey))
594 gIOHibernateMode ^= kIOHibernateModeEncrypt;
595
596 if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
597 {
598 vars->videoAllocSize = kVideoMapSize;
599 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize, VM_KERN_MEMORY_IOKIT))
600 vars->videoMapping = 0;
601 }
602
603 // generate crypt keys
604 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
605 vars->wiredCryptKey[i] = random();
606 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
607 vars->cryptKey[i] = random();
608
609 // set nvram
610
611 IOSetBootImageNVRAM(nvramData);
612 nvramData->release();
613
614 #if defined(__i386__) || defined(__x86_64__)
615 {
616 struct AppleRTCHibernateVars
617 {
618 uint8_t signature[4];
619 uint32_t revision;
620 uint8_t booterSignature[20];
621 uint8_t wiredCryptKey[16];
622 };
623 AppleRTCHibernateVars rtcVars;
624 OSData * data;
625
626 rtcVars.signature[0] = 'A';
627 rtcVars.signature[1] = 'A';
628 rtcVars.signature[2] = 'P';
629 rtcVars.signature[3] = 'L';
630 rtcVars.revision = 1;
631 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
632 if (gIOHibernateBootSignature[0])
633 {
634 char c;
635 uint8_t value = 0;
636 for (uint32_t i = 0;
637 (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
638 i++)
639 {
640 if (c >= 'a') c -= 'a' - 10;
641 else if (c >= 'A') c -= 'A' - 10;
642 else if (c >= '0') c -= '0';
643 else continue;
644 value = (value << 4) | c;
645 if (i & 1) rtcVars.booterSignature[i >> 1] = value;
646 }
647 }
648 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
649 if (data)
650 {
651 if (gIOHibernateRTCVariablesKey)
652 IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
653 data->release();
654 }
655 if (gIOChosenEntry)
656 {
657 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
658 if (data) gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
659 // set BootNext
660 if (!gIOHibernateBoot0082Data)
661 {
662 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
663 if (data)
664 {
665 // AppleNVRAM_EFI_LOAD_OPTION
666 struct {
667 uint32_t Attributes;
668 uint16_t FilePathLength;
669 uint16_t Desc;
670 } loadOptionHeader;
671 loadOptionHeader.Attributes = 1;
672 loadOptionHeader.FilePathLength = data->getLength();
673 loadOptionHeader.Desc = 0;
674 gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
675 if (gIOHibernateBoot0082Data)
676 {
677 gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
678 gIOHibernateBoot0082Data->appendBytes(data);
679 }
680 }
681 }
682 if (!gIOHibernateBootNextData)
683 {
684 uint16_t bits = 0x0082;
685 gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
686 }
687 if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData)
688 {
689 gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
690 gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
691 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
692 }
693 // BootNext
694 }
695 }
696 #endif /* !i386 && !x86_64 */
697 }
698 while (false);
699
700 if (swapPinned) hibernate_pin_swap(FALSE);
701
702 IOLockLock(gFSLock);
703 if ((kIOReturnSuccess == err) && (kFSOpening != gFSState))
704 {
705 HIBLOG("hibernate file close due timeout\n");
706 err = kIOReturnTimeout;
707 }
708 if (kIOReturnSuccess == err)
709 {
710 gFSState = kFSOpened;
711 gIOHibernateVars = *vars;
712 gFileVars = *vars->fileVars;
713 gFileVars.allocated = false;
714 gIOHibernateVars.fileVars = &gFileVars;
715 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
716 gIOHibernateState = kIOHibernateStateHibernating;
717 }
718 else
719 {
720 IOPolledFileIOVars * fileVars = vars->fileVars;
721 IOHibernateDone(vars);
722 IOPolledFileClose(&fileVars,
723 #if DISABLE_TRIM
724 0, NULL, 0, 0, 0);
725 #else
726 0, NULL, 0, sizeof(IOHibernateImageHeader), setFileSize);
727 #endif
728 gFSState = kFSIdle;
729 }
730 IOLockUnlock(gFSLock);
731
732 if (vars->fileVars) IODelete(vars->fileVars, IOPolledFileIOVars, 1);
733 IODelete(vars, IOHibernateVars, 1);
734
735 return (err);
736 }
737
738 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
739
740 static void
741 IOSetBootImageNVRAM(OSData * data)
742 {
743 IORegistryEntry * regEntry;
744
745 if (!gIOOptionsEntry)
746 {
747 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
748 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
749 if (regEntry && !gIOOptionsEntry)
750 regEntry->release();
751 }
752 if (gIOOptionsEntry && gIOHibernateBootImageKey)
753 {
754 if (data) gIOOptionsEntry->setProperty(gIOHibernateBootImageKey, data);
755 else
756 {
757 gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
758 gIOOptionsEntry->sync();
759 }
760 }
761 }
762
763 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
764 /*
765 * Writes header to disk with signature, block size and file extents data.
766 * If there are more than 2 extents, then they are written on second block.
767 */
768 static IOReturn
769 IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature)
770 {
771 IOHibernateImageHeader hdr;
772 IOItemCount count;
773 IOReturn err = kIOReturnSuccess;
774 int rc;
775 IOPolledFileExtent * fileExtents;
776
777 fileExtents = (typeof(fileExtents)) vars->fileExtents->getBytesNoCopy();
778
779 memset(&hdr, 0, sizeof(IOHibernateImageHeader));
780 count = vars->fileExtents->getLength();
781 if (count > sizeof(hdr.fileExtentMap))
782 {
783 hdr.fileExtentMapSize = count;
784 count = sizeof(hdr.fileExtentMap);
785 }
786 else
787 hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap);
788
789 bcopy(fileExtents, &hdr.fileExtentMap[0], count);
790
791 // copy file block extent list if larger than header
792 if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap))
793 {
794 count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap);
795 rc = kern_write_file(vars->fileRef, vars->blockSize,
796 (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)),
797 count, IO_SKIP_ENCRYPTION);
798 if (rc != 0) {
799 HIBLOG("kern_write_file returned %d\n", rc);
800 err = kIOReturnIOError;
801 goto exit;
802 }
803 }
804 hdr.signature = signature;
805 hdr.deviceBlockSize = vars->blockSize;
806
807 rc = kern_write_file(vars->fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION);
808 if (rc != 0) {
809 HIBLOG("kern_write_file returned %d\n", rc);
810 err = kIOReturnIOError;
811 goto exit;
812 }
813
814 exit:
815 return err;
816 }
817
818 extern "C" boolean_t root_is_CF_drive;
819
820 void
821 IOOpenDebugDataFile(const char *fname, uint64_t size)
822 {
823 IOReturn err;
824 OSData * imagePath = NULL;
825 uint64_t padding;
826
827 if (!gDebugImageLock) {
828 gDebugImageLock = IOLockAlloc();
829 }
830
831 if (root_is_CF_drive) return;
832
833 // Try to get a lock, but don't block for getting lock
834 if (!IOLockTryLock(gDebugImageLock)) {
835 HIBLOG("IOOpenDebugDataFile: Failed to get lock\n");
836 return;
837 }
838
839 if (gDebugImageFileVars || !fname || !size) {
840 HIBLOG("IOOpenDebugDataFile: conditions failed\n");
841 goto exit;
842 }
843
844 padding = (PAGE_SIZE*2); // allocate couple more pages for header and fileextents
845 err = IOPolledFileOpen(fname, size+padding, 32ULL*1024*1024*1024,
846 NULL, 0,
847 &gDebugImageFileVars, &imagePath, NULL, 0);
848
849 if ((kIOReturnSuccess == err) && imagePath)
850 {
851 if ((gDebugImageFileVars->fileSize < (size+padding)) ||
852 (gDebugImageFileVars->fileExtents->getLength() > PAGE_SIZE)) {
853 // Can't use the file
854 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
855 HIBLOG("IOOpenDebugDataFile: too many file extents\n");
856 goto exit;
857 }
858
859 // write extents for debug data usage in EFI
860 IOWriteExtentsToFile(gDebugImageFileVars, kIOHibernateHeaderOpenSignature);
861 IOSetBootImageNVRAM(imagePath);
862 kprintf("IOOpenDebugDataFile: opened debugdata file\n");
863 }
864
865 exit:
866 IOLockUnlock(gDebugImageLock);
867
868 if (imagePath) imagePath->release();
869 return;
870 }
871
872 void
873 IOCloseDebugDataFile()
874 {
875 IOSetBootImageNVRAM(0);
876
877 if (gDebugImageLock) {
878 IOLockLock(gDebugImageLock);
879 if (gDebugImageFileVars != 0) {
880 kprintf("IOHibernateSystemPostWake: Closing debugdata file\n");
881 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
882 }
883 IOLockUnlock(gDebugImageLock);
884 }
885
886
887 }
888
889 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
890
891 DECLARE_IOHIBERNATEPROGRESSALPHA
892
893 static void
894 ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
895 {
896 uint32_t rowBytes, pixelShift;
897 uint32_t x, y;
898 int32_t blob;
899 uint32_t alpha, in, color, result;
900 uint8_t * out;
901 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
902
903 rowBytes = display->rowBytes;
904 pixelShift = display->depth >> 4;
905 if (pixelShift < 1) return;
906
907 screen += ((display->width
908 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
909 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
910
911 for (y = 0; y < kIOHibernateProgressHeight; y++)
912 {
913 out = screen + y * rowBytes;
914 for (blob = 0; blob < kIOHibernateProgressCount; blob++)
915 {
916 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
917 for (x = 0; x < kIOHibernateProgressWidth; x++)
918 {
919 alpha = gIOHibernateProgressAlpha[y][x];
920 result = color;
921 if (alpha)
922 {
923 if (0xff != alpha)
924 {
925 if (1 == pixelShift)
926 {
927 in = *((uint16_t *)out) & 0x1f; // 16
928 in = (in << 3) | (in >> 2);
929 }
930 else
931 in = *((uint32_t *)out) & 0xff; // 32
932 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
933 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
934 }
935 if (1 == pixelShift)
936 {
937 result >>= 3;
938 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
939 }
940 else
941 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
942 }
943 out += (1 << pixelShift);
944 }
945 out += (kIOHibernateProgressSpacing << pixelShift);
946 }
947 }
948 }
949
950
951 static void
952 ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
953 {
954 uint32_t rowBytes, pixelShift;
955 uint32_t x, y;
956 int32_t blob, lastBlob;
957 uint32_t alpha, in, color, result;
958 uint8_t * out;
959 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
960
961 pixelShift = display->depth >> 4;
962 if (pixelShift < 1)
963 return;
964
965 rowBytes = display->rowBytes;
966
967 screen += ((display->width
968 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
969 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
970
971 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
972
973 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
974
975 for (y = 0; y < kIOHibernateProgressHeight; y++)
976 {
977 out = screen + y * rowBytes;
978 for (blob = firstBlob; blob <= lastBlob; blob++)
979 {
980 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
981 for (x = 0; x < kIOHibernateProgressWidth; x++)
982 {
983 alpha = gIOHibernateProgressAlpha[y][x];
984 result = color;
985 if (alpha)
986 {
987 if (0xff != alpha)
988 {
989 in = display->progressSaveUnder[blob][saveindex[blob]++];
990 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
991 }
992 if (1 == pixelShift)
993 {
994 result >>= 3;
995 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
996 }
997 else
998 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
999 }
1000 out += (1 << pixelShift);
1001 }
1002 out += (kIOHibernateProgressSpacing << pixelShift);
1003 }
1004 }
1005 }
1006
1007 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1008
1009 IOReturn
1010 IOHibernateIOKitSleep(void)
1011 {
1012 IOReturn ret = kIOReturnSuccess;
1013 IOLockLock(gFSLock);
1014 if (kFSOpening == gFSState)
1015 {
1016 gFSState = kFSTimedOut;
1017 HIBLOG("hibernate file open timed out\n");
1018 ret = kIOReturnTimeout;
1019 }
1020 IOLockUnlock(gFSLock);
1021 return (ret);
1022 }
1023
1024 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1025
1026 IOReturn
1027 IOHibernateSystemHasSlept(void)
1028 {
1029 IOReturn ret = kIOReturnSuccess;
1030 IOHibernateVars * vars = &gIOHibernateVars;
1031 OSObject * obj = 0;
1032 OSData * data;
1033
1034 IOLockLock(gFSLock);
1035 if ((kFSOpened != gFSState) && gIOHibernateMode)
1036 {
1037 ret = kIOReturnTimeout;
1038 }
1039 IOLockUnlock(gFSLock);
1040 if (kIOReturnSuccess != ret) return (ret);
1041
1042 if (gIOHibernateMode) obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
1043 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1044 if (obj && !vars->previewBuffer)
1045 obj->release();
1046
1047 vars->consoleMapping = NULL;
1048 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
1049 {
1050 vars->previewBuffer->release();
1051 vars->previewBuffer = 0;
1052 }
1053
1054 if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1055 && vars->previewBuffer
1056 && (data = OSDynamicCast(OSData,
1057 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
1058 {
1059 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
1060 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
1061
1062 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1063
1064 if (kIOHibernatePreviewUpdates & flags)
1065 {
1066 PE_Video consoleInfo;
1067 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1068
1069 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1070
1071 graphicsInfo->width = consoleInfo.v_width;
1072 graphicsInfo->height = consoleInfo.v_height;
1073 graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
1074 graphicsInfo->depth = consoleInfo.v_depth;
1075 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1076
1077 HIBPRINT("video %p %d %d %d\n",
1078 vars->consoleMapping, graphicsInfo->depth,
1079 graphicsInfo->width, graphicsInfo->height);
1080 if (vars->consoleMapping)
1081 ProgressInit(graphicsInfo, vars->consoleMapping,
1082 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1083 }
1084 }
1085
1086 if (gIOOptionsEntry)
1087 gIOOptionsEntry->sync();
1088
1089 return (ret);
1090 }
1091
1092 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1093
1094 static DeviceTreeNode *
1095 MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry)
1096 {
1097 DeviceTreeNodeProperty * prop;
1098 DeviceTreeNode * child;
1099 IORegistryEntry * childRegEntry;
1100 const char * nameProp;
1101 unsigned int propLen, idx;
1102
1103 prop = (DeviceTreeNodeProperty *) (entry + 1);
1104 for (idx = 0; idx < entry->nProperties; idx++)
1105 {
1106 if (regEntry && (0 != strcmp("name", prop->name)))
1107 {
1108 regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
1109 // HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
1110 }
1111 prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
1112 }
1113
1114 child = (DeviceTreeNode *) prop;
1115 for (idx = 0; idx < entry->nChildren; idx++)
1116 {
1117 if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen))
1118 panic("no name");
1119 childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
1120 // HIBPRINT("%s == %p\n", nameProp, childRegEntry);
1121 child = MergeDeviceTree(child, childRegEntry);
1122 }
1123 return (child);
1124 }
1125
1126 IOReturn
1127 IOHibernateSystemWake(void)
1128 {
1129 if (kFSOpened == gFSState)
1130 {
1131 IOPolledFilePollersClose(gIOHibernateVars.fileVars, kIOPolledPostflightState);
1132 IOHibernateDone(&gIOHibernateVars);
1133 }
1134 else
1135 {
1136 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1137 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1138 }
1139 return (kIOReturnSuccess);
1140 }
1141
1142 static IOReturn
1143 IOHibernateDone(IOHibernateVars * vars)
1144 {
1145 hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
1146
1147 if (vars->videoMapping)
1148 {
1149 if (vars->videoMapSize)
1150 // remove mappings
1151 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1152 if (vars->videoAllocSize)
1153 // dealloc range
1154 kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
1155 }
1156
1157 if (vars->previewBuffer)
1158 {
1159 vars->previewBuffer->release();
1160 vars->previewBuffer = 0;
1161 }
1162
1163 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1164 {
1165 IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
1166 gIOHibernateCurrentHeader->options, 32);
1167 }
1168 else
1169 {
1170 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1171 }
1172
1173 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1174 && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus))
1175 {
1176 IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
1177 &gIOHibernateGraphicsInfo->gfxStatus,
1178 sizeof(gIOHibernateGraphicsInfo->gfxStatus));
1179 }
1180 else
1181 {
1182 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1183 }
1184
1185 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1186
1187 #if defined(__i386__) || defined(__x86_64__)
1188 IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
1189 IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
1190
1191 /*
1192 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1193 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1194 */
1195 if (gIOOptionsEntry) {
1196
1197 if (gIOHibernateRTCVariablesKey) {
1198 if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
1199 gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
1200 }
1201 }
1202
1203 if (gIOHibernateBootNextKey)
1204 {
1205 if (gIOHibernateBootNextSave)
1206 {
1207 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
1208 gIOHibernateBootNextSave->release();
1209 gIOHibernateBootNextSave = NULL;
1210 }
1211 else
1212 gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
1213 }
1214 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) gIOOptionsEntry->sync();
1215 }
1216 #endif
1217
1218 if (vars->srcBuffer) vars->srcBuffer->release();
1219 bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
1220 if (vars->handoffBuffer)
1221 {
1222 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1223 {
1224 IOHibernateHandoff * handoff;
1225 bool done = false;
1226 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
1227 !done;
1228 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
1229 {
1230 HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
1231 uint8_t * data = &handoff->data[0];
1232 switch (handoff->type)
1233 {
1234 case kIOHibernateHandoffTypeEnd:
1235 done = true;
1236 break;
1237
1238 case kIOHibernateHandoffTypeDeviceTree:
1239 MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
1240 break;
1241
1242 case kIOHibernateHandoffTypeKeyStore:
1243 #if defined(__i386__) || defined(__x86_64__)
1244 {
1245 IOBufferMemoryDescriptor *
1246 md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
1247 if (md)
1248 {
1249 IOSetKeyStoreData(md);
1250 }
1251 }
1252 #endif
1253 break;
1254
1255 default:
1256 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
1257 break;
1258 }
1259 }
1260 }
1261 vars->handoffBuffer->release();
1262 }
1263
1264 bzero(vars, sizeof(*vars));
1265
1266 // gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1267
1268 return (kIOReturnSuccess);
1269 }
1270
1271 IOReturn
1272 IOHibernateSystemPostWake(void)
1273 {
1274 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1275 IOLockLock(gFSLock);
1276 if (kFSOpened == gFSState)
1277 {
1278 // invalidate & close the image file
1279 IOSleep(TRIM_DELAY);
1280 IOPolledFileIOVars * vars = &gFileVars;
1281 IOPolledFileClose(&vars,
1282 #if DISABLE_TRIM
1283 0, NULL, 0, 0, 0);
1284 #else
1285 0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader),
1286 sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize);
1287 #endif
1288 }
1289 gFSState = kFSIdle;
1290
1291 IOLockUnlock(gFSLock);
1292
1293 // IOCloseDebugDataFile() calls IOSetBootImageNVRAM() unconditionally
1294 IOCloseDebugDataFile( );
1295 return (kIOReturnSuccess);
1296 }
1297
1298 uint32_t IOHibernateWasScreenLocked(void)
1299 {
1300 uint32_t ret = 0;
1301 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry)
1302 {
1303 OSData *
1304 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
1305 if (data)
1306 {
1307 ret = ((uint32_t *)data->getBytesNoCopy())[0];
1308 gIOChosenEntry->setProperty(kIOBooterScreenLockStateKey, data);
1309 }
1310 }
1311 else gIOChosenEntry->removeProperty(kIOBooterScreenLockStateKey);
1312
1313 return (ret);
1314 }
1315
1316 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1317
1318 SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
1319 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1320 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
1321 SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
1322 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1323 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
1324 SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
1325 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1326 &gIOHibernateMode, 0, "");
1327 SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
1328 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1329 &_hibernateStats, hibernate_statistics_t, "");
1330
1331 SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
1332 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1333 &_hibernateStats.graphicsReadyTime, 0, "");
1334 SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
1335 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1336 &_hibernateStats.wakeNotificationTime, 0, "");
1337 SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
1338 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1339 &_hibernateStats.lockScreenReadyTime, 0, "");
1340 SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
1341 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1342 &_hibernateStats.hidReadyTime, 0, "");
1343
1344
1345 void
1346 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1347 {
1348 gIOHibernateBootImageKey = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1349
1350 #if defined(__i386__) || defined(__x86_64__)
1351 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1352 gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
1353 gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
1354 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1355 #endif /* defined(__i386__) || defined(__x86_64__) */
1356
1357 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
1358 if (data)
1359 {
1360 rootDomain->setProperty(kIOHibernateStateKey, data);
1361 data->release();
1362 }
1363
1364 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))
1365 gIOHibernateMode = kIOHibernateModeOn;
1366 else
1367 gIOHibernateFilename[0] = 0;
1368
1369 sysctl_register_oid(&sysctl__kern_hibernatefile);
1370 sysctl_register_oid(&sysctl__kern_bootsignature);
1371 sysctl_register_oid(&sysctl__kern_hibernatemode);
1372 sysctl_register_oid(&sysctl__kern_hibernatestatistics);
1373 sysctl_register_oid(&sysctl__kern_hibernategraphicsready);
1374 sysctl_register_oid(&sysctl__kern_hibernatewakenotification);
1375 sysctl_register_oid(&sysctl__kern_hibernatelockscreenready);
1376 sysctl_register_oid(&sysctl__kern_hibernatehidready);
1377
1378 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1379
1380 gFSLock = IOLockAlloc();
1381 }
1382
1383 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1384
1385 static IOReturn
1386 IOHibernatePolledFileWrite(IOPolledFileIOVars * vars,
1387 const uint8_t * bytes, IOByteCount size,
1388 IOPolledFileCryptVars * cryptvars)
1389 {
1390 IOReturn err;
1391
1392 err = IOPolledFileWrite(vars, bytes, size, cryptvars);
1393 if ((kIOReturnSuccess == err) && hibernate_should_abort()) err = kIOReturnAborted;
1394
1395 return (err);
1396 }
1397
1398 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1399
1400 extern "C" uint32_t
1401 hibernate_write_image(void)
1402 {
1403 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
1404 IOHibernateVars * vars = &gIOHibernateVars;
1405 IOPolledFileExtent * fileExtents;
1406
1407 _static_assert_1_arg(sizeof(IOHibernateImageHeader) == 512);
1408
1409 uint32_t pageCount, pagesDone;
1410 IOReturn err;
1411 vm_offset_t ppnum, page;
1412 IOItemCount count;
1413 uint8_t * src;
1414 uint8_t * data;
1415 uint8_t * compressed;
1416 uint8_t * scratch;
1417 IOByteCount pageCompressedSize;
1418 uint64_t compressedSize, uncompressedSize;
1419 uint64_t image1Size = 0;
1420 uint32_t bitmap_size;
1421 bool iterDone, pollerOpen, needEncrypt;
1422 uint32_t restore1Sum, sum, sum1, sum2;
1423 int wkresult;
1424 uint32_t tag;
1425 uint32_t pageType;
1426 uint32_t pageAndCount[2];
1427 addr64_t phys64;
1428 IOByteCount segLen;
1429 uintptr_t hibernateBase;
1430 uintptr_t hibernateEnd;
1431
1432 AbsoluteTime startTime, endTime;
1433 AbsoluteTime allTime, compTime;
1434 uint64_t compBytes;
1435 uint64_t nsec;
1436 uint32_t lastProgressStamp = 0;
1437 uint32_t progressStamp;
1438 uint32_t blob, lastBlob = (uint32_t) -1L;
1439
1440 uint32_t wiredPagesEncrypted;
1441 uint32_t dirtyPagesEncrypted;
1442 uint32_t wiredPagesClear;
1443 uint32_t svPageCount;
1444 uint32_t zvPageCount;
1445
1446 IOPolledFileCryptVars _cryptvars;
1447 IOPolledFileCryptVars * cryptvars = 0;
1448
1449 wiredPagesEncrypted = 0;
1450 dirtyPagesEncrypted = 0;
1451 wiredPagesClear = 0;
1452 svPageCount = 0;
1453 zvPageCount = 0;
1454
1455 if (!vars->fileVars
1456 || !vars->fileVars->pollers
1457 || !(kIOHibernateModeOn & gIOHibernateMode)) return (kIOHibernatePostWriteSleep);
1458
1459 if (kIOHibernateModeSleep & gIOHibernateMode)
1460 kdebug_enable = save_kdebug_enable;
1461
1462 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START, 0, 0, 0, 0, 0);
1463 IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
1464
1465 restore1Sum = sum1 = sum2 = 0;
1466
1467 #if CRYPTO
1468 // encryption data. "iv" is the "initial vector".
1469 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1470 {
1471 static const unsigned char first_iv[AES_BLOCK_SIZE]
1472 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
1473 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
1474
1475 cryptvars = &gIOHibernateCryptWakeContext;
1476 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1477 aes_encrypt_key(vars->cryptKey,
1478 kIOHibernateAESKeySize,
1479 &cryptvars->ctx.encrypt);
1480 aes_decrypt_key(vars->cryptKey,
1481 kIOHibernateAESKeySize,
1482 &cryptvars->ctx.decrypt);
1483
1484 cryptvars = &_cryptvars;
1485 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1486 for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++)
1487 vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
1488 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
1489 aes_encrypt_key(vars->wiredCryptKey,
1490 kIOHibernateAESKeySize,
1491 &cryptvars->ctx.encrypt);
1492
1493 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1494 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1495 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
1496 }
1497 #endif /* CRYPTO */
1498
1499 hibernate_page_list_setall(vars->page_list,
1500 vars->page_list_wired,
1501 vars->page_list_pal,
1502 false /* !preflight */,
1503 /* discard_all */
1504 ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
1505 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
1506 &pageCount);
1507
1508 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
1509
1510 fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy();
1511
1512 #if 0
1513 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
1514 for (page = 0; page < count; page++)
1515 {
1516 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
1517 fileExtents[page].start, fileExtents[page].length,
1518 fileExtents[page].start + fileExtents[page].length);
1519 }
1520 #endif
1521
1522 needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
1523 AbsoluteTime_to_scalar(&compTime) = 0;
1524 compBytes = 0;
1525
1526 clock_get_uptime(&allTime);
1527 IOService::getPMRootDomain()->pmStatsRecordEvent(
1528 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
1529 do
1530 {
1531 compressedSize = 0;
1532 uncompressedSize = 0;
1533 svPageCount = 0;
1534 zvPageCount = 0;
1535
1536 IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
1537
1538 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
1539 ml_get_interrupts_enabled());
1540 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState, true);
1541 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
1542 pollerOpen = (kIOReturnSuccess == err);
1543 if (!pollerOpen)
1544 break;
1545
1546 // copy file block extent list if larger than header
1547
1548 count = vars->fileVars->fileExtents->getLength();
1549 if (count > sizeof(header->fileExtentMap))
1550 {
1551 count -= sizeof(header->fileExtentMap);
1552 err = IOHibernatePolledFileWrite(vars->fileVars,
1553 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
1554 if (kIOReturnSuccess != err)
1555 break;
1556 }
1557
1558 hibernateBase = HIB_BASE; /* Defined in PAL headers */
1559 hibernateEnd = (segHIBB + segSizeHIB);
1560
1561 // copy out restore1 code
1562
1563 for (count = 0;
1564 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1565 count += segLen)
1566 {
1567 for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++)
1568 {
1569 gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
1570 }
1571 }
1572
1573 page = atop_32(kvtophys(hibernateBase));
1574 count = atop_32(round_page(hibernateEnd) - hibernateBase);
1575 header->restore1CodePhysPage = page;
1576 header->restore1CodeVirt = hibernateBase;
1577 header->restore1PageCount = count;
1578 header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
1579 header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
1580
1581 // sum __HIB seg, with zeros for the stack
1582 src = (uint8_t *) trunc_page(hibernateBase);
1583 for (page = 0; page < count; page++)
1584 {
1585 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
1586 restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
1587 else
1588 restore1Sum += 0x00000000;
1589 src += page_size;
1590 }
1591 sum1 = restore1Sum;
1592
1593 // write the __HIB seg, with zeros for the stack
1594
1595 src = (uint8_t *) trunc_page(hibernateBase);
1596 count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
1597 if (count)
1598 {
1599 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
1600 if (kIOReturnSuccess != err)
1601 break;
1602 }
1603 err = IOHibernatePolledFileWrite(vars->fileVars,
1604 (uint8_t *) 0,
1605 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
1606 cryptvars);
1607 if (kIOReturnSuccess != err)
1608 break;
1609 src = &gIOHibernateRestoreStackEnd[0];
1610 count = round_page(hibernateEnd) - ((uintptr_t) src);
1611 if (count)
1612 {
1613 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
1614 if (kIOReturnSuccess != err)
1615 break;
1616 }
1617
1618 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1619 {
1620 vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
1621 vars->fileVars->encryptEnd = UINT64_MAX;
1622 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1623 }
1624
1625 // write the preview buffer
1626
1627 if (vars->previewBuffer)
1628 {
1629 ppnum = 0;
1630 count = 0;
1631 do
1632 {
1633 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
1634 pageAndCount[0] = atop_64(phys64);
1635 pageAndCount[1] = atop_32(segLen);
1636 err = IOHibernatePolledFileWrite(vars->fileVars,
1637 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1638 cryptvars);
1639 if (kIOReturnSuccess != err)
1640 break;
1641 count += segLen;
1642 ppnum += sizeof(pageAndCount);
1643 }
1644 while (phys64);
1645 if (kIOReturnSuccess != err)
1646 break;
1647
1648 src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
1649
1650 ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
1651
1652 count = vars->previewBuffer->getLength();
1653
1654 header->previewPageListSize = ppnum;
1655 header->previewSize = count + ppnum;
1656
1657 for (page = 0; page < count; page += page_size)
1658 {
1659 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
1660 sum1 += hibernate_sum_page(src + page, atop_64(phys64));
1661 }
1662 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
1663 if (kIOReturnSuccess != err)
1664 break;
1665 }
1666
1667 // mark areas for no save
1668 IOMemoryDescriptor * ioBuffer;
1669 ioBuffer = IOPolledFileGetIOBuffer(vars->fileVars);
1670 for (count = 0;
1671 (phys64 = ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1672 count += segLen)
1673 {
1674 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1675 atop_64(phys64), atop_32(segLen),
1676 kIOHibernatePageStateFree);
1677 pageCount -= atop_32(segLen);
1678 }
1679
1680 for (count = 0;
1681 (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1682 count += segLen)
1683 {
1684 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1685 atop_64(phys64), atop_32(segLen),
1686 kIOHibernatePageStateFree);
1687 pageCount -= atop_32(segLen);
1688 }
1689
1690 // copy out bitmap of pages available for trashing during restore
1691
1692 bitmap_size = vars->page_list_wired->list_size;
1693 src = (uint8_t *) vars->page_list_wired;
1694 err = IOHibernatePolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
1695 if (kIOReturnSuccess != err)
1696 break;
1697
1698 // mark more areas for no save, but these are not available
1699 // for trashing during restore
1700
1701 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
1702
1703
1704 page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
1705 count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
1706 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1707 page, count,
1708 kIOHibernatePageStateFree);
1709 pageCount -= count;
1710
1711 if (vars->previewBuffer) for (count = 0;
1712 (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1713 count += segLen)
1714 {
1715 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1716 atop_64(phys64), atop_32(segLen),
1717 kIOHibernatePageStateFree);
1718 pageCount -= atop_32(segLen);
1719 }
1720
1721 for (count = 0;
1722 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1723 count += segLen)
1724 {
1725 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1726 atop_64(phys64), atop_32(segLen),
1727 kIOHibernatePageStateFree);
1728 pageCount -= atop_32(segLen);
1729 }
1730
1731 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
1732 compressed = src + page_size;
1733 scratch = compressed + page_size;
1734
1735 pagesDone = 0;
1736 lastBlob = 0;
1737
1738 HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
1739 bitmap_size, header->previewSize,
1740 pageCount, vars->fileVars->position);
1741
1742 enum
1743 // pageType
1744 {
1745 kWired = 0x02,
1746 kEncrypt = 0x01,
1747 kWiredEncrypt = kWired | kEncrypt,
1748 kWiredClear = kWired,
1749 kUnwiredEncrypt = kEncrypt
1750 };
1751
1752 bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features()));
1753 #define _pmap_is_noencrypt(x) (cpuAES ? false : pmap_is_noencrypt((x)))
1754
1755 for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)
1756 {
1757 if (kUnwiredEncrypt == pageType)
1758 {
1759 // start unwired image
1760 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1761 {
1762 vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
1763 vars->fileVars->encryptEnd = UINT64_MAX;
1764 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1765 }
1766 bcopy(&cryptvars->aes_iv[0],
1767 &gIOHibernateCryptWakeContext.aes_iv[0],
1768 sizeof(cryptvars->aes_iv));
1769 cryptvars = &gIOHibernateCryptWakeContext;
1770 }
1771 for (iterDone = false, ppnum = 0; !iterDone; )
1772 {
1773 count = hibernate_page_list_iterate((kWired & pageType)
1774 ? vars->page_list_wired : vars->page_list,
1775 &ppnum);
1776 // kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
1777 iterDone = !count;
1778
1779 if (count && (kWired & pageType) && needEncrypt)
1780 {
1781 uint32_t checkIndex;
1782 for (checkIndex = 0;
1783 (checkIndex < count)
1784 && (((kEncrypt & pageType) == 0) == _pmap_is_noencrypt(ppnum + checkIndex));
1785 checkIndex++)
1786 {}
1787 if (!checkIndex)
1788 {
1789 ppnum++;
1790 continue;
1791 }
1792 count = checkIndex;
1793 }
1794
1795 switch (pageType)
1796 {
1797 case kWiredEncrypt: wiredPagesEncrypted += count; break;
1798 case kWiredClear: wiredPagesClear += count; break;
1799 case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
1800 }
1801
1802 if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */}
1803 else
1804 {
1805 pageAndCount[0] = ppnum;
1806 pageAndCount[1] = count;
1807 err = IOHibernatePolledFileWrite(vars->fileVars,
1808 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1809 cryptvars);
1810 if (kIOReturnSuccess != err)
1811 break;
1812 }
1813
1814 for (page = ppnum; page < (ppnum + count); page++)
1815 {
1816 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
1817 if (err)
1818 {
1819 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
1820 break;
1821 }
1822
1823 sum = hibernate_sum_page(src, page);
1824 if (kWired & pageType)
1825 sum1 += sum;
1826 else
1827 sum2 += sum;
1828
1829 clock_get_uptime(&startTime);
1830 wkresult = WKdm_compress_new((const WK_word*) src,
1831 (WK_word*) compressed,
1832 (WK_word*) scratch,
1833 page_size - 4);
1834
1835 clock_get_uptime(&endTime);
1836 ADD_ABSOLUTETIME(&compTime, &endTime);
1837 SUB_ABSOLUTETIME(&compTime, &startTime);
1838
1839 compBytes += page_size;
1840 pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
1841
1842 if (pageCompressedSize == 0)
1843 {
1844 pageCompressedSize = 4;
1845 data = src;
1846
1847 if (*(uint32_t *)src)
1848 svPageCount++;
1849 else
1850 zvPageCount++;
1851 }
1852 else
1853 {
1854 if (pageCompressedSize != page_size)
1855 data = compressed;
1856 else
1857 data = src;
1858 }
1859
1860 tag = pageCompressedSize | kIOHibernateTagSignature;
1861 err = IOHibernatePolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
1862 if (kIOReturnSuccess != err)
1863 break;
1864
1865 err = IOHibernatePolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
1866 if (kIOReturnSuccess != err)
1867 break;
1868
1869 compressedSize += pageCompressedSize;
1870 uncompressedSize += page_size;
1871 pagesDone++;
1872
1873 if (vars->consoleMapping && (0 == (1023 & pagesDone)))
1874 {
1875 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
1876 if (blob != lastBlob)
1877 {
1878 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
1879 lastBlob = blob;
1880 }
1881 }
1882 if (0 == (8191 & pagesDone))
1883 {
1884 clock_get_uptime(&endTime);
1885 SUB_ABSOLUTETIME(&endTime, &allTime);
1886 absolutetime_to_nanoseconds(endTime, &nsec);
1887 progressStamp = nsec / 750000000ULL;
1888 if (progressStamp != lastProgressStamp)
1889 {
1890 lastProgressStamp = progressStamp;
1891 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
1892 }
1893 }
1894 }
1895 if (kIOReturnSuccess != err)
1896 break;
1897 ppnum = page;
1898 }
1899
1900 if (kIOReturnSuccess != err)
1901 break;
1902
1903 if ((kEncrypt & pageType) && vars->fileVars->encryptStart)
1904 {
1905 vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
1906 HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
1907 }
1908
1909 if (kWiredEncrypt != pageType)
1910 {
1911 // end of image1/2 - fill to next block
1912 err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
1913 if (kIOReturnSuccess != err)
1914 break;
1915 }
1916 if (kWiredClear == pageType)
1917 {
1918 // enlarge wired image for test
1919 // err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
1920
1921 // end wired image
1922 header->encryptStart = vars->fileVars->encryptStart;
1923 header->encryptEnd = vars->fileVars->encryptEnd;
1924 image1Size = vars->fileVars->position;
1925 HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
1926 image1Size, header->encryptStart, header->encryptEnd);
1927 }
1928 }
1929 if (kIOReturnSuccess != err)
1930 {
1931 if (kIOReturnOverrun == err)
1932 {
1933 // update actual compression ratio on not enough space (for retry)
1934 gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
1935 }
1936
1937 // update partial amount written (for IOPolledFileClose cleanup/unmap)
1938 header->imageSize = vars->fileVars->position;
1939 break;
1940 }
1941
1942 // Header:
1943
1944 header->imageSize = vars->fileVars->position;
1945 header->image1Size = image1Size;
1946 header->bitmapSize = bitmap_size;
1947 header->pageCount = pageCount;
1948
1949 header->restore1Sum = restore1Sum;
1950 header->image1Sum = sum1;
1951 header->image2Sum = sum2;
1952 header->sleepTime = gIOLastSleepTime.tv_sec;
1953
1954 header->compression = (compressedSize << 8) / uncompressedSize;
1955 gIOHibernateCompression = header->compression;
1956
1957 count = vars->fileVars->fileExtents->getLength();
1958 if (count > sizeof(header->fileExtentMap))
1959 {
1960 header->fileExtentMapSize = count;
1961 count = sizeof(header->fileExtentMap);
1962 }
1963 else
1964 header->fileExtentMapSize = sizeof(header->fileExtentMap);
1965 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
1966
1967 header->deviceBase = vars->fileVars->block0;
1968 header->deviceBlockSize = vars->fileVars->blockSize;
1969
1970 IOPolledFileSeek(vars->fileVars, 0);
1971 err = IOHibernatePolledFileWrite(vars->fileVars,
1972 (uint8_t *) header, sizeof(IOHibernateImageHeader),
1973 cryptvars);
1974 if (kIOReturnSuccess != err)
1975 break;
1976 err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
1977 }
1978 while (false);
1979
1980 clock_get_uptime(&endTime);
1981
1982 IOService::getPMRootDomain()->pmStatsRecordEvent(
1983 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
1984
1985 SUB_ABSOLUTETIME(&endTime, &allTime);
1986 absolutetime_to_nanoseconds(endTime, &nsec);
1987 HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
1988
1989 absolutetime_to_nanoseconds(compTime, &nsec);
1990 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
1991 compBytes,
1992 nsec / 1000000ULL,
1993 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
1994
1995 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
1996 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
1997 vars->fileVars->cryptBytes,
1998 nsec / 1000000ULL,
1999 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2000
2001 HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
2002 header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
2003 uncompressedSize, atop_32(uncompressedSize), compressedSize,
2004 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
2005 sum1, sum2);
2006
2007 HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
2008 svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
2009
2010 if (pollerOpen)
2011 IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted );
2012
2013 if (vars->consoleMapping)
2014 ProgressUpdate(gIOHibernateGraphicsInfo,
2015 vars->consoleMapping, 0, kIOHibernateProgressCount);
2016
2017 HIBLOG("hibernate_write_image done(%x)\n", err);
2018
2019 // should we come back via regular wake, set the state in memory.
2020 gIOHibernateState = kIOHibernateStateInactive;
2021
2022 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END,
2023 wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted, 0, 0);
2024
2025 if (kIOReturnSuccess == err)
2026 {
2027 if (kIOHibernateModeSleep & gIOHibernateMode)
2028 {
2029 return (kIOHibernatePostWriteSleep);
2030 }
2031 else if(kIOHibernateModeRestart & gIOHibernateMode)
2032 {
2033 return (kIOHibernatePostWriteRestart);
2034 }
2035 else
2036 {
2037 /* by default, power down */
2038 return (kIOHibernatePostWriteHalt);
2039 }
2040 }
2041 else if (kIOReturnAborted == err)
2042 {
2043 return (kIOHibernatePostWriteWake);
2044 }
2045 else
2046 {
2047 /* on error, sleep */
2048 return (kIOHibernatePostWriteSleep);
2049 }
2050 }
2051
2052 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2053
2054 extern "C" void
2055 hibernate_machine_init(void)
2056 {
2057 IOReturn err;
2058 uint32_t sum;
2059 uint32_t pagesDone;
2060 uint32_t pagesRead = 0;
2061 AbsoluteTime startTime, compTime;
2062 AbsoluteTime allTime, endTime;
2063 AbsoluteTime startIOTime, endIOTime;
2064 uint64_t nsec, nsecIO;
2065 uint64_t compBytes;
2066 uint32_t lastProgressStamp = 0;
2067 uint32_t progressStamp;
2068 IOPolledFileCryptVars * cryptvars = 0;
2069
2070 IOHibernateVars * vars = &gIOHibernateVars;
2071 bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
2072
2073 if (!vars->fileVars || !vars->fileVars->pollers)
2074 return;
2075
2076 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2077 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2078
2079 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
2080 {
2081 HIBLOG("regular wake\n");
2082 return;
2083 }
2084
2085 HIBPRINT("diag %x %x %x %x\n",
2086 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
2087 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
2088
2089 #define t40ms(x) (tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)
2090 #define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
2091 tStat(booterStart, booterStart);
2092 gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart;
2093 tStat(booterDuration0, booterTime0);
2094 tStat(booterDuration1, booterTime1);
2095 tStat(booterDuration2, booterTime2);
2096 tStat(booterDuration, booterTime);
2097 tStat(booterConnectDisplayDuration, connectDisplayTime);
2098 tStat(booterSplashDuration, splashTime);
2099 tStat(trampolineDuration, trampolineTime);
2100
2101 gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size;
2102 gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize;
2103 gIOHibernateStats->image1Pages = pagesDone;
2104
2105 HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
2106 gIOHibernateStats->booterStart,
2107 gIOHibernateStats->smcStart,
2108 gIOHibernateStats->booterDuration0,
2109 gIOHibernateStats->booterDuration1,
2110 gIOHibernateStats->booterDuration2,
2111 gIOHibernateStats->booterDuration,
2112 gIOHibernateStats->booterConnectDisplayDuration,
2113 gIOHibernateStats->booterSplashDuration,
2114 gIOHibernateStats->trampolineDuration);
2115
2116 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
2117 gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
2118 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
2119
2120 if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
2121 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)))
2122 {
2123 hibernate_page_list_discard(vars->page_list);
2124 }
2125
2126 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
2127
2128 if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount)
2129 panic("handoff overflow");
2130
2131 IOHibernateHandoff * handoff;
2132 bool done = false;
2133 bool foundCryptData = false;
2134
2135 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
2136 !done;
2137 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
2138 {
2139 // HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
2140 uint8_t * data = &handoff->data[0];
2141 switch (handoff->type)
2142 {
2143 case kIOHibernateHandoffTypeEnd:
2144 done = true;
2145 break;
2146
2147 case kIOHibernateHandoffTypeGraphicsInfo:
2148 if (handoff->bytecount == sizeof(*gIOHibernateGraphicsInfo))
2149 {
2150 bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
2151 }
2152 break;
2153
2154 case kIOHibernateHandoffTypeCryptVars:
2155 if (cryptvars)
2156 {
2157 hibernate_cryptwakevars_t *
2158 wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
2159 bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
2160 }
2161 foundCryptData = true;
2162 bzero(data, handoff->bytecount);
2163 break;
2164
2165 case kIOHibernateHandoffTypeMemoryMap:
2166
2167 clock_get_uptime(&allTime);
2168
2169 hibernate_newruntime_map(data, handoff->bytecount,
2170 gIOHibernateCurrentHeader->systemTableOffset);
2171
2172 clock_get_uptime(&endTime);
2173
2174 SUB_ABSOLUTETIME(&endTime, &allTime);
2175 absolutetime_to_nanoseconds(endTime, &nsec);
2176
2177 HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
2178
2179 break;
2180
2181 case kIOHibernateHandoffTypeDeviceTree:
2182 {
2183 // DTEntry chosen = NULL;
2184 // HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen));
2185 }
2186 break;
2187
2188 default:
2189 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
2190 break;
2191 }
2192 }
2193 if (cryptvars && !foundCryptData)
2194 panic("hibernate handoff");
2195
2196 HIBPRINT("video 0x%llx %d %d %d status %x\n",
2197 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2198 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
2199
2200 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)
2201 {
2202 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2203 * gIOHibernateGraphicsInfo->rowBytes);
2204 if (vars->videoMapSize > vars->videoAllocSize) vars->videoMapSize = 0;
2205 else
2206 {
2207 IOMapPages(kernel_map,
2208 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2209 vars->videoMapSize, kIOMapInhibitCache );
2210 }
2211 }
2212
2213 if (vars->videoMapSize)
2214 ProgressUpdate(gIOHibernateGraphicsInfo,
2215 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2216
2217 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2218 uint8_t * compressed = src + page_size;
2219 uint8_t * scratch = compressed + page_size;
2220 uint32_t decoOffset;
2221
2222 clock_get_uptime(&allTime);
2223 AbsoluteTime_to_scalar(&compTime) = 0;
2224 compBytes = 0;
2225
2226 HIBLOG("IOPolledFilePollersOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2227 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledAfterSleepState, false);
2228 clock_get_uptime(&startIOTime);
2229 endTime = startIOTime;
2230 SUB_ABSOLUTETIME(&endTime, &allTime);
2231 absolutetime_to_nanoseconds(endTime, &nsec);
2232 HIBLOG("IOPolledFilePollersOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
2233
2234 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2235
2236 // kick off the read ahead
2237 vars->fileVars->bufferHalf = 0;
2238 vars->fileVars->bufferLimit = 0;
2239 vars->fileVars->lastRead = 0;
2240 vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
2241 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2242 vars->fileVars->cryptBytes = 0;
2243 AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
2244
2245 err = IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
2246 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2247 // --
2248
2249 HIBLOG("hibernate_machine_init reading\n");
2250
2251 uint32_t * header = (uint32_t *) src;
2252 sum = 0;
2253
2254 while (kIOReturnSuccess == err)
2255 {
2256 unsigned int count;
2257 unsigned int page;
2258 uint32_t tag;
2259 vm_offset_t ppnum, compressedSize;
2260
2261 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2262 if (kIOReturnSuccess != err)
2263 break;
2264
2265 ppnum = header[0];
2266 count = header[1];
2267
2268 // HIBPRINT("(%x, %x)\n", ppnum, count);
2269
2270 if (!count)
2271 break;
2272
2273 for (page = 0; page < count; page++)
2274 {
2275 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2276 if (kIOReturnSuccess != err)
2277 break;
2278
2279 compressedSize = kIOHibernateTagLength & tag;
2280 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength))
2281 {
2282 err = kIOReturnIPCError;
2283 break;
2284 }
2285
2286 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2287 if (kIOReturnSuccess != err) break;
2288
2289 if (compressedSize < page_size)
2290 {
2291 decoOffset = page_size;
2292 clock_get_uptime(&startTime);
2293
2294 if (compressedSize == 4) {
2295 int i;
2296 uint32_t *s, *d;
2297
2298 s = (uint32_t *)src;
2299 d = (uint32_t *)(uintptr_t)compressed;
2300
2301 for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++)
2302 *d++ = *s;
2303 }
2304 else
2305 WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, compressedSize);
2306 clock_get_uptime(&endTime);
2307 ADD_ABSOLUTETIME(&compTime, &endTime);
2308 SUB_ABSOLUTETIME(&compTime, &startTime);
2309 compBytes += page_size;
2310 }
2311 else decoOffset = 0;
2312
2313 sum += hibernate_sum_page((src + decoOffset), ppnum);
2314 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2315 if (err)
2316 {
2317 HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
2318 break;
2319 }
2320
2321 ppnum++;
2322 pagesDone++;
2323 pagesRead++;
2324
2325 if (0 == (8191 & pagesDone))
2326 {
2327 clock_get_uptime(&endTime);
2328 SUB_ABSOLUTETIME(&endTime, &allTime);
2329 absolutetime_to_nanoseconds(endTime, &nsec);
2330 progressStamp = nsec / 750000000ULL;
2331 if (progressStamp != lastProgressStamp)
2332 {
2333 lastProgressStamp = progressStamp;
2334 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2335 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2336 }
2337 }
2338 }
2339 }
2340 if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages))
2341 err = kIOReturnLockedRead;
2342
2343 if (kIOReturnSuccess != err)
2344 panic("Hibernate restore error %x", err);
2345
2346 gIOHibernateCurrentHeader->actualImage2Sum = sum;
2347 gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
2348
2349 clock_get_uptime(&endIOTime);
2350
2351 err = IOPolledFilePollersClose(vars->fileVars, kIOPolledAfterSleepState);
2352
2353 clock_get_uptime(&endTime);
2354
2355 IOService::getPMRootDomain()->pmStatsRecordEvent(
2356 kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
2357 IOService::getPMRootDomain()->pmStatsRecordEvent(
2358 kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
2359
2360 SUB_ABSOLUTETIME(&endTime, &allTime);
2361 absolutetime_to_nanoseconds(endTime, &nsec);
2362
2363 SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
2364 absolutetime_to_nanoseconds(endIOTime, &nsecIO);
2365
2366 gIOHibernateStats->kernelImageReadDuration = nsec / 1000000ULL;
2367 gIOHibernateStats->imagePages = pagesDone;
2368
2369 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
2370 pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
2371 nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
2372
2373 absolutetime_to_nanoseconds(compTime, &nsec);
2374 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2375 compBytes,
2376 nsec / 1000000ULL,
2377 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2378
2379 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2380 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
2381 vars->fileVars->cryptBytes,
2382 nsec / 1000000ULL,
2383 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2384
2385 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_NONE, pagesRead, pagesDone, 0, 0, 0);
2386 }
2387
2388 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2389
2390 void IOHibernateSetWakeCapabilities(uint32_t capability)
2391 {
2392 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
2393 {
2394 gIOHibernateStats->wakeCapability = capability;
2395
2396 if (kIOPMSystemCapabilityGraphics & capability)
2397 {
2398 vm_compressor_do_warmup();
2399 }
2400 }
2401 }
2402
2403 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2404
2405 void IOHibernateSystemRestart(void)
2406 {
2407 static uint8_t noteStore[32] __attribute__((aligned(32)));
2408 IORegistryEntry * regEntry;
2409 const OSSymbol * sym;
2410 OSData * noteProp;
2411 OSData * data;
2412 uintptr_t * smcVars;
2413 uint8_t * smcBytes;
2414 size_t len;
2415 addr64_t element;
2416
2417 data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
2418 if (!data) return;
2419
2420 smcVars = (typeof(smcVars)) data->getBytesNoCopy();
2421 smcBytes = (typeof(smcBytes)) smcVars[1];
2422 len = smcVars[0];
2423 if (len > sizeof(noteStore)) len = sizeof(noteStore);
2424 noteProp = OSData::withCapacity(3 * sizeof(element));
2425 if (!noteProp) return;
2426 element = len;
2427 noteProp->appendBytes(&element, sizeof(element));
2428 element = crc32(0, smcBytes, len);
2429 noteProp->appendBytes(&element, sizeof(element));
2430
2431 bcopy(smcBytes, noteStore, len);
2432 element = (addr64_t) &noteStore[0];
2433 element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
2434 noteProp->appendBytes(&element, sizeof(element));
2435
2436 if (!gIOOptionsEntry)
2437 {
2438 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
2439 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
2440 if (regEntry && !gIOOptionsEntry)
2441 regEntry->release();
2442 }
2443
2444 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
2445 if (gIOOptionsEntry && sym) gIOOptionsEntry->setProperty(sym, noteProp);
2446 if (noteProp) noteProp->release();
2447 if (sym) sym->release();
2448 }
2449
2450
2451
mirror of XNU