]> git.saurik.com Git - apple/xnu.git/blob - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blob
? search:


953c6d44530a4a6fed9776f99cae9509ae474b50
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
1 /*
2 * Copyright (c) 2004-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30
31 Sleep:
32
33 - PMRootDomain calls IOHibernateSystemSleep() before system sleep
34 (devices awake, normal execution context)
35 - IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
36 grabs its extents and searches for a polling driver willing to work with that IOMedia.
37 The BSD code makes an ioctl to the storage driver to get the partition base offset to
38 the disk, and other ioctls to get the transfer constraints
39 If successful, the file is written to make sure its initially not bootable (in case of
40 later failure) and nvram set to point to the first block of the file. (Has to be done
41 here so blocking is possible in nvram support).
42 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
43 page out any pages it wants to (currently zero, but probably some percentage of memory).
44 Its assumed just allocating pages will cause the VM system to naturally select the best
45 pages for eviction. It also copies processor flags needed for the restore path and sets
46 a flag in the boot processor proc info.
47 gIOHibernateState = kIOHibernateStateHibernating.
48 - Regular sleep progresses - some drivers may inspect the root domain property
49 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
50 as usual but leaves motherboard I/O on.
51 - Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
52 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
53 all ppc RC bits out of the hash table and caches into the mapping structures.
54 - hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
55 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
56 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
57 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
58 The image header and block list are written. The header includes the second file extent so
59 only the header block is needed to read the file, regardless of filesystem.
60 The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
61 (only) is used to decompress the image during wake/boot.
62 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
63 The bitmaps are written to the image.
64 More areas are removed from the bitmaps (after they have been written to the image) - the
65 segment "__HIB" pages and interrupt stack.
66 Each wired page is compressed and written and then each non-wired page. Compression and
67 disk writes are in parallel.
68 The image header is written to the start of the file and the polling driver closed.
69 The machine powers down (or sleeps).
70
71 Boot/Wake:
72
73 - BootX sees the boot-image nvram variable containing the device and block number of the image,
74 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
75 - BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
76 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
77 that is in the kernel's __HIB section.
78 - BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
79 hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
80 only code & data in that section is safe to call since all the other wired pages are still
81 compressed in the image.
82 - hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
83 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
84 location directly, and copies those that can't to interim free pages. When the image has been
85 completed, the copies are uncompressed, overwriting the wired image pages.
86 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
87 is used to get pages into place for 64bit.
88 - the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
89 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
90 are removed from the software strutures, and the hash table is reinitialized.
91 - After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
92 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
93 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
94 for the remaining non wired pages.
95 - The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
96 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
97 is closed via bsd.
98
99 Polled Mode I/O:
100
101 IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
102 registry, specifying an object of calls IOPolledInterface.
103
104 Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
105 partition) that the image is going to live, looking for polled interface properties. If it finds
106 one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
107 interfaces found are kept in an ordered list.
108
109 There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
110 few different contexts things happen in:
111
112 - there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
113 up and running) and after wake - this is safe to allocate memory and do anything. The device
114 ignores sleep requests from that point since its a waste of time if it goes to sleep and
115 immediately wakes back up for the image write.
116
117 - there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
118 immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
119 from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
120 used to flush and set the disk to sleep.
121
122 - there is an Open/Close (AfterSleep) pair made around the image read operations that happen
123 immediately after sleep. These can't block or allocate memory. This is happening after the platform
124 expert has woken the low level bits of the system, but most of the I/O system has not. There is only
125 one thread running.
126
127 For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
128 (prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
129 the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
130 Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
131 that is called for the hardware to check for events, and complete the I/O via the callback.
132 The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
133 to restrict I/O ops.
134 */
135
136 #include <sys/systm.h>
137
138 #include <IOKit/IOWorkLoop.h>
139 #include <IOKit/IOCommandGate.h>
140 #include <IOKit/IOTimerEventSource.h>
141 #include <IOKit/IOPlatformExpert.h>
142 #include <IOKit/IOKitDebug.h>
143 #include <IOKit/IOTimeStamp.h>
144 #include <IOKit/pwr_mgt/RootDomain.h>
145 #include <IOKit/pwr_mgt/IOPMPrivate.h>
146 #include <IOKit/IOMessage.h>
147 #include <IOKit/IODeviceTreeSupport.h>
148 #include <IOKit/IOBSD.h>
149 #include "RootDomainUserClient.h"
150 #include <IOKit/pwr_mgt/IOPowerConnection.h>
151 #include "IOPMPowerStateQueue.h"
152 #include <IOKit/IOBufferMemoryDescriptor.h>
153 #include <IOKit/AppleKeyStoreInterface.h>
154 #include <libkern/crypto/aes.h>
155
156 #include <sys/uio.h>
157 #include <sys/conf.h>
158 #include <sys/stat.h>
159 #include <sys/fcntl.h> // (FWRITE, ...)
160 #include <sys/sysctl.h>
161 #include <sys/kdebug.h>
162 #include <stdint.h>
163
164 #include <IOKit/IOHibernatePrivate.h>
165 #include <IOKit/IOPolledInterface.h>
166 #include <IOKit/IONVRAM.h>
167 #include "IOHibernateInternal.h"
168 #include <vm/WKdm_new.h>
169 #include <vm/vm_protos.h>
170 #include "IOKitKernelInternal.h"
171 #include <pexpert/device_tree.h>
172
173 #include <machine/pal_routines.h>
174 #include <machine/pal_hibernate.h>
175 #include <i386/tsc.h>
176 #include <i386/cpuid.h>
177
178 extern "C" addr64_t kvtophys(vm_offset_t va);
179 extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
180
181 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
182
183 #define DISABLE_TRIM 0
184 #define TRIM_DELAY 25000
185
186 extern unsigned int save_kdebug_enable;
187 extern uint32_t gIOHibernateState;
188 uint32_t gIOHibernateMode;
189 static char gIOHibernateBootSignature[256+1];
190 static char gIOHibernateFilename[MAXPATHLEN+1];
191 static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
192 uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
193 static uint64_t gIOHibernateCompression = 0x80; // default compression 50%
194 boolean_t gIOHibernateStandbyDisabled;
195
196 static IODTNVRAM * gIOOptionsEntry;
197 static IORegistryEntry * gIOChosenEntry;
198
199 static const OSSymbol * gIOHibernateBootImageKey;
200
201 #if defined(__i386__) || defined(__x86_64__)
202
203 static const OSSymbol * gIOHibernateRTCVariablesKey;
204 static const OSSymbol * gIOHibernateBoot0082Key;
205 static const OSSymbol * gIOHibernateBootNextKey;
206 static OSData * gIOHibernateBoot0082Data;
207 static OSData * gIOHibernateBootNextData;
208 static OSObject * gIOHibernateBootNextSave;
209
210 static IOPolledFileIOVars * gDebugImageFileVars;
211 static IOLock * gDebugImageLock;
212
213 #endif /* defined(__i386__) || defined(__x86_64__) */
214
215 static IOLock * gFSLock;
216 static uint32_t gFSState;
217 static thread_call_t gIOHibernateTrimCalloutEntry;
218 static IOPolledFileIOVars gFileVars;
219 static IOHibernateVars gIOHibernateVars;
220 static IOPolledFileCryptVars gIOHibernateCryptWakeContext;
221 static hibernate_graphics_t _hibernateGraphics;
222 static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
223 static hibernate_statistics_t _hibernateStats;
224 static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats;
225
226 enum
227 {
228 kFSIdle = 0,
229 kFSOpening = 2,
230 kFSOpened = 3,
231 kFSTimedOut = 4,
232 kFSTrimDelay = 5
233 };
234
235 static IOReturn IOHibernateDone(IOHibernateVars * vars);
236 static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature);
237 static void IOSetBootImageNVRAM(OSData * data);
238
239 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
240
241 enum { kDefaultIOSize = 128 * 1024 };
242 enum { kVideoMapSize = 80 * 1024 * 1024 };
243
244 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
245
246 // copy from phys addr to MD
247
248 static IOReturn
249 IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
250 IOByteCount offset, addr64_t bytes, IOByteCount length)
251 {
252 addr64_t srcAddr = bytes;
253 IOByteCount remaining;
254
255 remaining = length = min(length, md->getLength() - offset);
256 while (remaining) { // (process another target segment?)
257 addr64_t dstAddr64;
258 IOByteCount dstLen;
259
260 dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
261 if (!dstAddr64)
262 break;
263
264 // Clip segment length to remaining
265 if (dstLen > remaining)
266 dstLen = remaining;
267
268 #if 1
269 bcopy_phys(srcAddr, dstAddr64, dstLen);
270 #else
271 copypv(srcAddr, dstAddr64, dstLen,
272 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
273 #endif
274 srcAddr += dstLen;
275 offset += dstLen;
276 remaining -= dstLen;
277 }
278
279 assert(!remaining);
280
281 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
282 }
283
284 // copy from MD to phys addr
285
286 static IOReturn
287 IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
288 IOByteCount offset, addr64_t bytes, IOByteCount length)
289 {
290 addr64_t dstAddr = bytes;
291 IOByteCount remaining;
292
293 remaining = length = min(length, md->getLength() - offset);
294 while (remaining) { // (process another target segment?)
295 addr64_t srcAddr64;
296 IOByteCount dstLen;
297
298 srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
299 if (!srcAddr64)
300 break;
301
302 // Clip segment length to remaining
303 if (dstLen > remaining)
304 dstLen = remaining;
305
306 #if 1
307 bcopy_phys(srcAddr64, dstAddr, dstLen);
308 #else
309 copypv(srcAddr, dstAddr64, dstLen,
310 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
311 #endif
312 dstAddr += dstLen;
313 offset += dstLen;
314 remaining -= dstLen;
315 }
316
317 assert(!remaining);
318
319 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
320 }
321
322 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
323
324 void
325 hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
326 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
327 {
328 count += ppnum;
329 switch (kind)
330 {
331 case kIOHibernatePageStateUnwiredSave:
332 // unwired save
333 for (; ppnum < count; ppnum++)
334 {
335 hibernate_page_bitset(page_list, FALSE, ppnum);
336 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
337 }
338 break;
339 case kIOHibernatePageStateWiredSave:
340 // wired save
341 for (; ppnum < count; ppnum++)
342 {
343 hibernate_page_bitset(page_list, FALSE, ppnum);
344 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
345 }
346 break;
347 case kIOHibernatePageStateFree:
348 // free page
349 for (; ppnum < count; ppnum++)
350 {
351 hibernate_page_bitset(page_list, TRUE, ppnum);
352 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
353 }
354 break;
355 default:
356 panic("hibernate_set_page_state");
357 }
358 }
359
360 static vm_offset_t
361 hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
362 {
363 uint32_t page = *pPage;
364 uint32_t count;
365 hibernate_bitmap_t * bitmap;
366
367 while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
368 {
369 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
370 if (!count)
371 break;
372 page += count;
373 if (page <= bitmap->last_page)
374 break;
375 }
376
377 *pPage = page;
378 if (bitmap)
379 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
380 else
381 count = 0;
382
383 return (count);
384 }
385
386 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
387
388 IOReturn
389 IOHibernateSystemSleep(void)
390 {
391 IOReturn err;
392 OSData * nvramData;
393 OSObject * obj;
394 OSString * str;
395 OSNumber * num;
396 bool dsSSD, vmflush, swapPinned;
397 IOHibernateVars * vars;
398 uint64_t setFileSize = 0;
399
400 gIOHibernateState = kIOHibernateStateInactive;
401
402 gIOHibernateDebugFlags = 0;
403 if (kIOLogHibernate & gIOKitDebug)
404 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
405
406 if (IOService::getPMRootDomain()->getHibernateSettings(
407 &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime))
408 {
409 if (kIOHibernateModeSleep & gIOHibernateMode)
410 // default to discard clean for safe sleep
411 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
412 | kIOHibernateModeDiscardCleanActive);
413 }
414
415 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
416 {
417 if ((str = OSDynamicCast(OSString, obj)))
418 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
419 sizeof(gIOHibernateFilename));
420 obj->release();
421 }
422
423 if (!gIOHibernateMode || !gIOHibernateFilename[0])
424 return (kIOReturnUnsupported);
425
426 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
427
428 vars = IONew(IOHibernateVars, 1);
429 if (!vars) return (kIOReturnNoMemory);
430 bzero(vars, sizeof(*vars));
431
432 IOLockLock(gFSLock);
433 if (!gIOHibernateTrimCalloutEntry)
434 {
435 gIOHibernateTrimCalloutEntry = thread_call_allocate(&IOHibernateSystemPostWakeTrim, &gFSLock);
436 }
437 IOHibernateSystemPostWakeTrim(NULL, NULL);
438 if (kFSIdle != gFSState)
439 {
440 HIBLOG("hibernate file busy\n");
441 IOLockUnlock(gFSLock);
442 IODelete(vars, IOHibernateVars, 1);
443 return (kIOReturnBusy);
444 }
445 gFSState = kFSOpening;
446 IOLockUnlock(gFSLock);
447
448 swapPinned = false;
449 do
450 {
451 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
452 2 * page_size + WKdm_SCRATCH_BUF_SIZE_INTERNAL, page_size);
453
454 vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
455 ptoa_64(gIOHibernateHandoffPageCount), page_size);
456
457 if (!vars->srcBuffer || !vars->handoffBuffer)
458 {
459 err = kIOReturnNoMemory;
460 break;
461 }
462
463 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey)))
464 {
465 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMinSize = num->unsigned64BitValue();
466 obj->release();
467 }
468 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey)))
469 {
470 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMaxSize = num->unsigned64BitValue();
471 obj->release();
472 }
473
474 boolean_t encryptedswap = true;
475 uint32_t pageCount;
476 AbsoluteTime startTime, endTime;
477 uint64_t nsec;
478
479 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
480 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
481 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
482
483 vmflush = ((kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
484 err = hibernate_alloc_page_lists(&vars->page_list,
485 &vars->page_list_wired,
486 &vars->page_list_pal);
487 if (KERN_SUCCESS != err) break;
488
489 err = hibernate_pin_swap(TRUE);
490 if (KERN_SUCCESS != err) break;
491 swapPinned = true;
492
493 if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode))
494 {
495 hibernate_page_list_setall(vars->page_list,
496 vars->page_list_wired,
497 vars->page_list_pal,
498 true /* preflight */,
499 vmflush /* discard */,
500 &pageCount);
501 PE_Video consoleInfo;
502 bzero(&consoleInfo, sizeof(consoleInfo));
503 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
504
505 // estimate: 6% increase in pages compressed
506 // screen preview 2 images compressed 0%
507 setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
508 + vars->page_list->list_size
509 + (consoleInfo.v_width * consoleInfo.v_height * 8);
510 enum { setFileRound = 1024*1024ULL };
511 setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
512
513 HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
514 pageCount, (100ULL * gIOHibernateCompression) >> 8,
515 setFileSize, vars->fileMinSize);
516
517 if (!(kIOHibernateModeFileResize & gIOHibernateMode)
518 && (setFileSize < vars->fileMinSize))
519 {
520 setFileSize = vars->fileMinSize;
521 }
522 }
523
524 // Invalidate the image file
525 if (gDebugImageLock) {
526 IOLockLock(gDebugImageLock);
527 if (gDebugImageFileVars != 0) {
528 IOSetBootImageNVRAM(0);
529 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
530 }
531 IOLockUnlock(gDebugImageLock);
532 }
533
534 err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, 0,
535 gIOHibernateCurrentHeader, sizeof(gIOHibernateCurrentHeader),
536 &vars->fileVars, &nvramData,
537 &vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
538
539 if (KERN_SUCCESS != err)
540 {
541 IOLockLock(gFSLock);
542 if (kFSOpening != gFSState) err = kIOReturnTimeout;
543 IOLockUnlock(gFSLock);
544 }
545
546 if (KERN_SUCCESS != err)
547 {
548 HIBLOG("IOPolledFileOpen(%x)\n", err);
549 break;
550 }
551
552 // write extents for debug data usage in EFI
553 IOWriteExtentsToFile(vars->fileVars, kIOHibernateHeaderOpenSignature);
554
555 err = IOPolledFilePollersSetup(vars->fileVars, kIOPolledPreflightState);
556 if (KERN_SUCCESS != err) break;
557
558 clock_get_uptime(&startTime);
559 err = hibernate_setup(gIOHibernateCurrentHeader,
560 vmflush,
561 vars->page_list, vars->page_list_wired, vars->page_list_pal);
562 clock_get_uptime(&endTime);
563 SUB_ABSOLUTETIME(&endTime, &startTime);
564 absolutetime_to_nanoseconds(endTime, &nsec);
565
566 boolean_t haveSwapPin, hibFileSSD;
567 haveSwapPin = vm_swap_files_pinned();
568
569 hibFileSSD = (kIOPolledFileSSD & vars->fileVars->flags);
570
571 HIBLOG("hibernate_setup(%d) took %qd ms, swapPin(%d) ssd(%d)\n",
572 err, nsec / 1000000ULL,
573 haveSwapPin, hibFileSSD);
574 if (KERN_SUCCESS != err) break;
575
576 gIOHibernateStandbyDisabled = ((!haveSwapPin || !hibFileSSD));
577
578 dsSSD = ((0 != (kIOPolledFileSSD & vars->fileVars->flags))
579 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
580
581 if (dsSSD) gIOHibernateCurrentHeader->options |= kIOHibernateOptionSSD | kIOHibernateOptionColor;
582 else gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
583
584
585 #if defined(__i386__) || defined(__x86_64__)
586 if (!uuid_is_null(vars->volumeCryptKey) &&
587 (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey)))
588 {
589 uintptr_t smcVars[2];
590 smcVars[0] = sizeof(vars->volumeCryptKey);
591 smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
592
593 IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
594 bzero(smcVars, sizeof(smcVars));
595 }
596 #endif
597
598
599 if (encryptedswap || !uuid_is_null(vars->volumeCryptKey))
600 gIOHibernateMode ^= kIOHibernateModeEncrypt;
601
602 if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
603 {
604 vars->videoAllocSize = kVideoMapSize;
605 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize, VM_KERN_MEMORY_IOKIT))
606 vars->videoMapping = 0;
607 }
608
609 // generate crypt keys
610 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
611 vars->wiredCryptKey[i] = random();
612 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
613 vars->cryptKey[i] = random();
614
615 // set nvram
616
617 IOSetBootImageNVRAM(nvramData);
618 nvramData->release();
619
620 #if defined(__i386__) || defined(__x86_64__)
621 {
622 struct AppleRTCHibernateVars
623 {
624 uint8_t signature[4];
625 uint32_t revision;
626 uint8_t booterSignature[20];
627 uint8_t wiredCryptKey[16];
628 };
629 AppleRTCHibernateVars rtcVars;
630 OSData * data;
631
632 rtcVars.signature[0] = 'A';
633 rtcVars.signature[1] = 'A';
634 rtcVars.signature[2] = 'P';
635 rtcVars.signature[3] = 'L';
636 rtcVars.revision = 1;
637 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
638 if (gIOHibernateBootSignature[0])
639 {
640 char c;
641 uint8_t value = 0;
642 for (uint32_t i = 0;
643 (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
644 i++)
645 {
646 if (c >= 'a') c -= 'a' - 10;
647 else if (c >= 'A') c -= 'A' - 10;
648 else if (c >= '0') c -= '0';
649 else continue;
650 value = (value << 4) | c;
651 if (i & 1) rtcVars.booterSignature[i >> 1] = value;
652 }
653 }
654 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
655 if (data)
656 {
657 if (gIOHibernateRTCVariablesKey)
658 IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
659 data->release();
660 }
661 if (gIOChosenEntry)
662 {
663 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
664 if (data) gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
665 // set BootNext
666 if (!gIOHibernateBoot0082Data)
667 {
668 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
669 if (data)
670 {
671 // AppleNVRAM_EFI_LOAD_OPTION
672 struct {
673 uint32_t Attributes;
674 uint16_t FilePathLength;
675 uint16_t Desc;
676 } loadOptionHeader;
677 loadOptionHeader.Attributes = 1;
678 loadOptionHeader.FilePathLength = data->getLength();
679 loadOptionHeader.Desc = 0;
680 gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
681 if (gIOHibernateBoot0082Data)
682 {
683 gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
684 gIOHibernateBoot0082Data->appendBytes(data);
685 }
686 }
687 }
688 if (!gIOHibernateBootNextData)
689 {
690 uint16_t bits = 0x0082;
691 gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
692 }
693 if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData)
694 {
695 gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
696 gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
697 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
698 }
699 // BootNext
700 }
701 }
702 #endif /* !i386 && !x86_64 */
703 }
704 while (false);
705
706 if (swapPinned) hibernate_pin_swap(FALSE);
707
708 IOLockLock(gFSLock);
709 if ((kIOReturnSuccess == err) && (kFSOpening != gFSState))
710 {
711 HIBLOG("hibernate file close due timeout\n");
712 err = kIOReturnTimeout;
713 }
714 if (kIOReturnSuccess == err)
715 {
716 gFSState = kFSOpened;
717 gIOHibernateVars = *vars;
718 gFileVars = *vars->fileVars;
719 gFileVars.allocated = false;
720 gIOHibernateVars.fileVars = &gFileVars;
721 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
722 gIOHibernateState = kIOHibernateStateHibernating;
723 }
724 else
725 {
726 IOPolledFileIOVars * fileVars = vars->fileVars;
727 IOHibernateDone(vars);
728 IOPolledFileClose(&fileVars,
729 #if DISABLE_TRIM
730 0, NULL, 0, 0, 0);
731 #else
732 0, NULL, 0, sizeof(IOHibernateImageHeader), setFileSize);
733 #endif
734 gFSState = kFSIdle;
735 }
736 IOLockUnlock(gFSLock);
737
738 if (vars->fileVars) IODelete(vars->fileVars, IOPolledFileIOVars, 1);
739 IODelete(vars, IOHibernateVars, 1);
740
741 return (err);
742 }
743
744 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
745
746 static void
747 IOSetBootImageNVRAM(OSData * data)
748 {
749 IORegistryEntry * regEntry;
750
751 if (!gIOOptionsEntry)
752 {
753 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
754 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
755 if (regEntry && !gIOOptionsEntry)
756 regEntry->release();
757 }
758 if (gIOOptionsEntry && gIOHibernateBootImageKey)
759 {
760 if (data) gIOOptionsEntry->setProperty(gIOHibernateBootImageKey, data);
761 else
762 {
763 gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
764 gIOOptionsEntry->sync();
765 }
766 }
767 }
768
769 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
770 /*
771 * Writes header to disk with signature, block size and file extents data.
772 * If there are more than 2 extents, then they are written on second block.
773 */
774 static IOReturn
775 IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature)
776 {
777 IOHibernateImageHeader hdr;
778 IOItemCount count;
779 IOReturn err = kIOReturnSuccess;
780 int rc;
781 IOPolledFileExtent * fileExtents;
782
783 fileExtents = (typeof(fileExtents)) vars->fileExtents->getBytesNoCopy();
784
785 memset(&hdr, 0, sizeof(IOHibernateImageHeader));
786 count = vars->fileExtents->getLength();
787 if (count > sizeof(hdr.fileExtentMap))
788 {
789 hdr.fileExtentMapSize = count;
790 count = sizeof(hdr.fileExtentMap);
791 }
792 else
793 hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap);
794
795 bcopy(fileExtents, &hdr.fileExtentMap[0], count);
796
797 // copy file block extent list if larger than header
798 if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap))
799 {
800 count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap);
801 rc = kern_write_file(vars->fileRef, vars->blockSize,
802 (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)),
803 count, IO_SKIP_ENCRYPTION);
804 if (rc != 0) {
805 HIBLOG("kern_write_file returned %d\n", rc);
806 err = kIOReturnIOError;
807 goto exit;
808 }
809 }
810 hdr.signature = signature;
811 hdr.deviceBlockSize = vars->blockSize;
812
813 rc = kern_write_file(vars->fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION);
814 if (rc != 0) {
815 HIBLOG("kern_write_file returned %d\n", rc);
816 err = kIOReturnIOError;
817 goto exit;
818 }
819
820 exit:
821 return err;
822 }
823
824 extern "C" boolean_t root_is_CF_drive;
825
826 void
827 IOOpenDebugDataFile(const char *fname, uint64_t size)
828 {
829 IOReturn err;
830 OSData * imagePath = NULL;
831 uint64_t padding;
832
833 if (!gDebugImageLock) {
834 gDebugImageLock = IOLockAlloc();
835 }
836
837 if (root_is_CF_drive) return;
838
839 // Try to get a lock, but don't block for getting lock
840 if (!IOLockTryLock(gDebugImageLock)) {
841 HIBLOG("IOOpenDebugDataFile: Failed to get lock\n");
842 return;
843 }
844
845 if (gDebugImageFileVars || !fname || !size) {
846 HIBLOG("IOOpenDebugDataFile: conditions failed\n");
847 goto exit;
848 }
849
850 padding = (PAGE_SIZE*2); // allocate couple more pages for header and fileextents
851 err = IOPolledFileOpen(fname, size+padding, 32ULL*1024*1024*1024,
852 NULL, 0,
853 &gDebugImageFileVars, &imagePath, NULL, 0);
854
855 if ((kIOReturnSuccess == err) && imagePath)
856 {
857 if ((gDebugImageFileVars->fileSize < (size+padding)) ||
858 (gDebugImageFileVars->fileExtents->getLength() > PAGE_SIZE)) {
859 // Can't use the file
860 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
861 HIBLOG("IOOpenDebugDataFile: too many file extents\n");
862 goto exit;
863 }
864
865 // write extents for debug data usage in EFI
866 IOWriteExtentsToFile(gDebugImageFileVars, kIOHibernateHeaderOpenSignature);
867 IOSetBootImageNVRAM(imagePath);
868 }
869
870 exit:
871 IOLockUnlock(gDebugImageLock);
872
873 if (imagePath) imagePath->release();
874 return;
875 }
876
877 void
878 IOCloseDebugDataFile()
879 {
880 IOSetBootImageNVRAM(0);
881
882 if (gDebugImageLock) {
883 IOLockLock(gDebugImageLock);
884 if (gDebugImageFileVars != 0) {
885 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
886 }
887 IOLockUnlock(gDebugImageLock);
888 }
889
890
891 }
892
893 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
894
895 DECLARE_IOHIBERNATEPROGRESSALPHA
896
897 static void
898 ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
899 {
900 uint32_t rowBytes, pixelShift;
901 uint32_t x, y;
902 int32_t blob;
903 uint32_t alpha, in, color, result;
904 uint8_t * out;
905 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
906
907 rowBytes = display->rowBytes;
908 pixelShift = display->depth >> 4;
909 if (pixelShift < 1) return;
910
911 screen += ((display->width
912 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
913 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
914
915 for (y = 0; y < kIOHibernateProgressHeight; y++)
916 {
917 out = screen + y * rowBytes;
918 for (blob = 0; blob < kIOHibernateProgressCount; blob++)
919 {
920 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
921 for (x = 0; x < kIOHibernateProgressWidth; x++)
922 {
923 alpha = gIOHibernateProgressAlpha[y][x];
924 result = color;
925 if (alpha)
926 {
927 if (0xff != alpha)
928 {
929 if (1 == pixelShift)
930 {
931 in = *((uint16_t *)out) & 0x1f; // 16
932 in = (in << 3) | (in >> 2);
933 }
934 else
935 in = *((uint32_t *)out) & 0xff; // 32
936 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
937 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
938 }
939 if (1 == pixelShift)
940 {
941 result >>= 3;
942 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
943 }
944 else
945 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
946 }
947 out += (1 << pixelShift);
948 }
949 out += (kIOHibernateProgressSpacing << pixelShift);
950 }
951 }
952 }
953
954
955 static void
956 ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
957 {
958 uint32_t rowBytes, pixelShift;
959 uint32_t x, y;
960 int32_t blob, lastBlob;
961 uint32_t alpha, in, color, result;
962 uint8_t * out;
963 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
964
965 pixelShift = display->depth >> 4;
966 if (pixelShift < 1)
967 return;
968
969 rowBytes = display->rowBytes;
970
971 screen += ((display->width
972 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
973 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
974
975 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
976
977 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
978
979 for (y = 0; y < kIOHibernateProgressHeight; y++)
980 {
981 out = screen + y * rowBytes;
982 for (blob = firstBlob; blob <= lastBlob; blob++)
983 {
984 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
985 for (x = 0; x < kIOHibernateProgressWidth; x++)
986 {
987 alpha = gIOHibernateProgressAlpha[y][x];
988 result = color;
989 if (alpha)
990 {
991 if (0xff != alpha)
992 {
993 in = display->progressSaveUnder[blob][saveindex[blob]++];
994 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
995 }
996 if (1 == pixelShift)
997 {
998 result >>= 3;
999 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1000 }
1001 else
1002 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1003 }
1004 out += (1 << pixelShift);
1005 }
1006 out += (kIOHibernateProgressSpacing << pixelShift);
1007 }
1008 }
1009 }
1010
1011 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1012
1013 IOReturn
1014 IOHibernateIOKitSleep(void)
1015 {
1016 IOReturn ret = kIOReturnSuccess;
1017 IOLockLock(gFSLock);
1018 if (kFSOpening == gFSState)
1019 {
1020 gFSState = kFSTimedOut;
1021 HIBLOG("hibernate file open timed out\n");
1022 ret = kIOReturnTimeout;
1023 }
1024 IOLockUnlock(gFSLock);
1025 return (ret);
1026 }
1027
1028 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1029
1030 IOReturn
1031 IOHibernateSystemHasSlept(void)
1032 {
1033 IOReturn ret = kIOReturnSuccess;
1034 IOHibernateVars * vars = &gIOHibernateVars;
1035 OSObject * obj = 0;
1036 OSData * data;
1037
1038 IOLockLock(gFSLock);
1039 if ((kFSOpened != gFSState) && gIOHibernateMode)
1040 {
1041 ret = kIOReturnTimeout;
1042 }
1043 IOLockUnlock(gFSLock);
1044 if (kIOReturnSuccess != ret) return (ret);
1045
1046 if (gIOHibernateMode) obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
1047 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1048 if (obj && !vars->previewBuffer)
1049 obj->release();
1050
1051 vars->consoleMapping = NULL;
1052 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
1053 {
1054 vars->previewBuffer->release();
1055 vars->previewBuffer = 0;
1056 }
1057
1058 if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1059 && vars->previewBuffer
1060 && (data = OSDynamicCast(OSData,
1061 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
1062 {
1063 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
1064 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
1065
1066 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1067
1068 if (kIOHibernatePreviewUpdates & flags)
1069 {
1070 PE_Video consoleInfo;
1071 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1072
1073 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1074
1075 graphicsInfo->width = consoleInfo.v_width;
1076 graphicsInfo->height = consoleInfo.v_height;
1077 graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
1078 graphicsInfo->depth = consoleInfo.v_depth;
1079 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1080
1081 HIBPRINT("video %p %d %d %d\n",
1082 vars->consoleMapping, graphicsInfo->depth,
1083 graphicsInfo->width, graphicsInfo->height);
1084 if (vars->consoleMapping)
1085 ProgressInit(graphicsInfo, vars->consoleMapping,
1086 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1087 }
1088 }
1089
1090 if (gIOOptionsEntry)
1091 gIOOptionsEntry->sync();
1092
1093 return (ret);
1094 }
1095
1096 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1097
1098 static DeviceTreeNode *
1099 MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry)
1100 {
1101 DeviceTreeNodeProperty * prop;
1102 DeviceTreeNode * child;
1103 IORegistryEntry * childRegEntry;
1104 const char * nameProp;
1105 unsigned int propLen, idx;
1106
1107 prop = (DeviceTreeNodeProperty *) (entry + 1);
1108 for (idx = 0; idx < entry->nProperties; idx++)
1109 {
1110 if (regEntry && (0 != strcmp("name", prop->name)))
1111 {
1112 regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
1113 // HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
1114 }
1115 prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
1116 }
1117
1118 child = (DeviceTreeNode *) prop;
1119 for (idx = 0; idx < entry->nChildren; idx++)
1120 {
1121 if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen))
1122 panic("no name");
1123 childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
1124 // HIBPRINT("%s == %p\n", nameProp, childRegEntry);
1125 child = MergeDeviceTree(child, childRegEntry);
1126 }
1127 return (child);
1128 }
1129
1130 IOReturn
1131 IOHibernateSystemWake(void)
1132 {
1133 if (kFSOpened == gFSState)
1134 {
1135 IOPolledFilePollersClose(gIOHibernateVars.fileVars, kIOPolledPostflightState);
1136 IOHibernateDone(&gIOHibernateVars);
1137 }
1138 else
1139 {
1140 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1141 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1142 }
1143 return (kIOReturnSuccess);
1144 }
1145
1146 static IOReturn
1147 IOHibernateDone(IOHibernateVars * vars)
1148 {
1149 hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
1150
1151 if (vars->videoMapping)
1152 {
1153 if (vars->videoMapSize)
1154 // remove mappings
1155 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1156 if (vars->videoAllocSize)
1157 // dealloc range
1158 kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
1159 }
1160
1161 if (vars->previewBuffer)
1162 {
1163 vars->previewBuffer->release();
1164 vars->previewBuffer = 0;
1165 }
1166
1167 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1168 {
1169 IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
1170 gIOHibernateCurrentHeader->options, 32);
1171 }
1172 else
1173 {
1174 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1175 }
1176
1177 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1178 && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus))
1179 {
1180 IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
1181 &gIOHibernateGraphicsInfo->gfxStatus,
1182 sizeof(gIOHibernateGraphicsInfo->gfxStatus));
1183 }
1184 else
1185 {
1186 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1187 }
1188
1189 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1190
1191 #if defined(__i386__) || defined(__x86_64__)
1192 IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
1193 IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
1194
1195 /*
1196 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1197 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1198 */
1199 if (gIOOptionsEntry) {
1200
1201 if (gIOHibernateRTCVariablesKey) {
1202 if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
1203 gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
1204 }
1205 }
1206
1207 if (gIOHibernateBootNextKey)
1208 {
1209 if (gIOHibernateBootNextSave)
1210 {
1211 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
1212 gIOHibernateBootNextSave->release();
1213 gIOHibernateBootNextSave = NULL;
1214 }
1215 else
1216 gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
1217 }
1218 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) gIOOptionsEntry->sync();
1219 }
1220 #endif
1221
1222 if (vars->srcBuffer) vars->srcBuffer->release();
1223 bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
1224 if (vars->handoffBuffer)
1225 {
1226 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1227 {
1228 IOHibernateHandoff * handoff;
1229 bool done = false;
1230 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
1231 !done;
1232 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
1233 {
1234 HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
1235 uint8_t * data = &handoff->data[0];
1236 switch (handoff->type)
1237 {
1238 case kIOHibernateHandoffTypeEnd:
1239 done = true;
1240 break;
1241
1242 case kIOHibernateHandoffTypeDeviceTree:
1243 MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
1244 break;
1245
1246 case kIOHibernateHandoffTypeKeyStore:
1247 #if defined(__i386__) || defined(__x86_64__)
1248 {
1249 IOBufferMemoryDescriptor *
1250 md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
1251 if (md)
1252 {
1253 IOSetKeyStoreData(md);
1254 }
1255 }
1256 #endif
1257 break;
1258
1259 default:
1260 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
1261 break;
1262 }
1263 }
1264 }
1265 vars->handoffBuffer->release();
1266 }
1267
1268 bzero(vars, sizeof(*vars));
1269
1270 // gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1271
1272 return (kIOReturnSuccess);
1273 }
1274
1275 void
1276 IOHibernateSystemPostWakeTrim(void * p1, void * p2)
1277 {
1278 // invalidate & close the image file
1279 if (p1) IOLockLock(gFSLock);
1280 if (kFSTrimDelay == gFSState)
1281 {
1282 IOPolledFileIOVars * vars = &gFileVars;
1283 IOPolledFileClose(&vars,
1284 #if DISABLE_TRIM
1285 0, NULL, 0, 0, 0);
1286 #else
1287 0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader),
1288 sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize);
1289 #endif
1290 gFSState = kFSIdle;
1291 }
1292 if (p1) IOLockUnlock(gFSLock);
1293 }
1294
1295 IOReturn
1296 IOHibernateSystemPostWake(void)
1297 {
1298 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1299 IOLockLock(gFSLock);
1300 if (kFSTrimDelay == gFSState) IOHibernateSystemPostWakeTrim(NULL, NULL);
1301 else if (kFSOpened != gFSState) gFSState = kFSIdle;
1302 else
1303 {
1304 AbsoluteTime deadline;
1305
1306 gFSState = kFSTrimDelay;
1307 clock_interval_to_deadline(TRIM_DELAY, kMillisecondScale, &deadline );
1308 thread_call_enter1_delayed(gIOHibernateTrimCalloutEntry, NULL, deadline);
1309 }
1310 IOLockUnlock(gFSLock);
1311
1312 // IOCloseDebugDataFile() calls IOSetBootImageNVRAM() unconditionally
1313 IOCloseDebugDataFile( );
1314 return (kIOReturnSuccess);
1315 }
1316
1317 uint32_t IOHibernateWasScreenLocked(void)
1318 {
1319 uint32_t ret = 0;
1320 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry)
1321 {
1322 OSData *
1323 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
1324 if (data)
1325 {
1326 ret = ((uint32_t *)data->getBytesNoCopy())[0];
1327 gIOChosenEntry->setProperty(kIOBooterScreenLockStateKey, data);
1328 }
1329 }
1330 else gIOChosenEntry->removeProperty(kIOBooterScreenLockStateKey);
1331
1332 return (ret);
1333 }
1334
1335 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1336
1337 SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
1338 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1339 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
1340 SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
1341 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1342 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
1343 SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
1344 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1345 &gIOHibernateMode, 0, "");
1346 SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
1347 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1348 &_hibernateStats, hibernate_statistics_t, "");
1349
1350 SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
1351 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1352 &_hibernateStats.graphicsReadyTime, 0, "");
1353 SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
1354 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1355 &_hibernateStats.wakeNotificationTime, 0, "");
1356 SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
1357 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1358 &_hibernateStats.lockScreenReadyTime, 0, "");
1359 SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
1360 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1361 &_hibernateStats.hidReadyTime, 0, "");
1362
1363
1364 void
1365 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1366 {
1367 gIOHibernateBootImageKey = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1368
1369 #if defined(__i386__) || defined(__x86_64__)
1370 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1371 gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
1372 gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
1373 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1374 #endif /* defined(__i386__) || defined(__x86_64__) */
1375
1376 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
1377 if (data)
1378 {
1379 rootDomain->setProperty(kIOHibernateStateKey, data);
1380 data->release();
1381 }
1382
1383 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))
1384 gIOHibernateMode = kIOHibernateModeOn;
1385 else
1386 gIOHibernateFilename[0] = 0;
1387
1388 sysctl_register_oid(&sysctl__kern_hibernatefile);
1389 sysctl_register_oid(&sysctl__kern_bootsignature);
1390 sysctl_register_oid(&sysctl__kern_hibernatemode);
1391 sysctl_register_oid(&sysctl__kern_hibernatestatistics);
1392 sysctl_register_oid(&sysctl__kern_hibernategraphicsready);
1393 sysctl_register_oid(&sysctl__kern_hibernatewakenotification);
1394 sysctl_register_oid(&sysctl__kern_hibernatelockscreenready);
1395 sysctl_register_oid(&sysctl__kern_hibernatehidready);
1396
1397 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1398
1399 gFSLock = IOLockAlloc();
1400 }
1401
1402 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1403
1404 static IOReturn
1405 IOHibernatePolledFileWrite(IOPolledFileIOVars * vars,
1406 const uint8_t * bytes, IOByteCount size,
1407 IOPolledFileCryptVars * cryptvars)
1408 {
1409 IOReturn err;
1410
1411 err = IOPolledFileWrite(vars, bytes, size, cryptvars);
1412 if ((kIOReturnSuccess == err) && hibernate_should_abort()) err = kIOReturnAborted;
1413
1414 return (err);
1415 }
1416
1417 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1418
1419 extern "C" uint32_t
1420 hibernate_write_image(void)
1421 {
1422 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
1423 IOHibernateVars * vars = &gIOHibernateVars;
1424 IOPolledFileExtent * fileExtents;
1425
1426 _static_assert_1_arg(sizeof(IOHibernateImageHeader) == 512);
1427
1428 uint32_t pageCount, pagesDone;
1429 IOReturn err;
1430 vm_offset_t ppnum, page;
1431 IOItemCount count;
1432 uint8_t * src;
1433 uint8_t * data;
1434 uint8_t * compressed;
1435 uint8_t * scratch;
1436 IOByteCount pageCompressedSize;
1437 uint64_t compressedSize, uncompressedSize;
1438 uint64_t image1Size = 0;
1439 uint32_t bitmap_size;
1440 bool iterDone, pollerOpen, needEncrypt;
1441 uint32_t restore1Sum, sum, sum1, sum2;
1442 int wkresult;
1443 uint32_t tag;
1444 uint32_t pageType;
1445 uint32_t pageAndCount[2];
1446 addr64_t phys64;
1447 IOByteCount segLen;
1448 uintptr_t hibernateBase;
1449 uintptr_t hibernateEnd;
1450
1451 AbsoluteTime startTime, endTime;
1452 AbsoluteTime allTime, compTime;
1453 uint64_t compBytes;
1454 uint64_t nsec;
1455 uint32_t lastProgressStamp = 0;
1456 uint32_t progressStamp;
1457 uint32_t blob, lastBlob = (uint32_t) -1L;
1458
1459 uint32_t wiredPagesEncrypted;
1460 uint32_t dirtyPagesEncrypted;
1461 uint32_t wiredPagesClear;
1462 uint32_t svPageCount;
1463 uint32_t zvPageCount;
1464
1465 IOPolledFileCryptVars _cryptvars;
1466 IOPolledFileCryptVars * cryptvars = 0;
1467
1468 wiredPagesEncrypted = 0;
1469 dirtyPagesEncrypted = 0;
1470 wiredPagesClear = 0;
1471 svPageCount = 0;
1472 zvPageCount = 0;
1473
1474 if (!vars->fileVars
1475 || !vars->fileVars->pollers
1476 || !(kIOHibernateModeOn & gIOHibernateMode)) return (kIOHibernatePostWriteSleep);
1477
1478 if (kIOHibernateModeSleep & gIOHibernateMode)
1479 kdebug_enable = save_kdebug_enable;
1480
1481 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START, 0, 0, 0, 0, 0);
1482 IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
1483
1484 restore1Sum = sum1 = sum2 = 0;
1485
1486 #if CRYPTO
1487 // encryption data. "iv" is the "initial vector".
1488 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1489 {
1490 static const unsigned char first_iv[AES_BLOCK_SIZE]
1491 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
1492 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
1493
1494 cryptvars = &gIOHibernateCryptWakeContext;
1495 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1496 aes_encrypt_key(vars->cryptKey,
1497 kIOHibernateAESKeySize,
1498 &cryptvars->ctx.encrypt);
1499 aes_decrypt_key(vars->cryptKey,
1500 kIOHibernateAESKeySize,
1501 &cryptvars->ctx.decrypt);
1502
1503 cryptvars = &_cryptvars;
1504 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1505 for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++)
1506 vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
1507 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
1508 aes_encrypt_key(vars->wiredCryptKey,
1509 kIOHibernateAESKeySize,
1510 &cryptvars->ctx.encrypt);
1511
1512 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1513 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1514 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
1515 }
1516 #endif /* CRYPTO */
1517
1518 hibernate_page_list_setall(vars->page_list,
1519 vars->page_list_wired,
1520 vars->page_list_pal,
1521 false /* !preflight */,
1522 /* discard_all */
1523 ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
1524 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
1525 &pageCount);
1526
1527 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
1528
1529 fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy();
1530
1531 #if 0
1532 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
1533 for (page = 0; page < count; page++)
1534 {
1535 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
1536 fileExtents[page].start, fileExtents[page].length,
1537 fileExtents[page].start + fileExtents[page].length);
1538 }
1539 #endif
1540
1541 needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
1542 AbsoluteTime_to_scalar(&compTime) = 0;
1543 compBytes = 0;
1544
1545 clock_get_uptime(&allTime);
1546 IOService::getPMRootDomain()->pmStatsRecordEvent(
1547 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
1548 do
1549 {
1550 compressedSize = 0;
1551 uncompressedSize = 0;
1552 svPageCount = 0;
1553 zvPageCount = 0;
1554
1555 IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
1556
1557 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
1558 ml_get_interrupts_enabled());
1559 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState,
1560 // abortable if not low battery
1561 !IOService::getPMRootDomain()->mustHibernate());
1562 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
1563 pollerOpen = (kIOReturnSuccess == err);
1564 if (!pollerOpen)
1565 break;
1566
1567 // copy file block extent list if larger than header
1568
1569 count = vars->fileVars->fileExtents->getLength();
1570 if (count > sizeof(header->fileExtentMap))
1571 {
1572 count -= sizeof(header->fileExtentMap);
1573 err = IOHibernatePolledFileWrite(vars->fileVars,
1574 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
1575 if (kIOReturnSuccess != err)
1576 break;
1577 }
1578
1579 hibernateBase = HIB_BASE; /* Defined in PAL headers */
1580 hibernateEnd = (segHIBB + segSizeHIB);
1581
1582 // copy out restore1 code
1583
1584 for (count = 0;
1585 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1586 count += segLen)
1587 {
1588 for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++)
1589 {
1590 gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
1591 }
1592 }
1593
1594 page = atop_32(kvtophys(hibernateBase));
1595 count = atop_32(round_page(hibernateEnd) - hibernateBase);
1596 header->restore1CodePhysPage = page;
1597 header->restore1CodeVirt = hibernateBase;
1598 header->restore1PageCount = count;
1599 header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
1600 header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
1601
1602 // sum __HIB seg, with zeros for the stack
1603 src = (uint8_t *) trunc_page(hibernateBase);
1604 for (page = 0; page < count; page++)
1605 {
1606 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
1607 restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
1608 else
1609 restore1Sum += 0x00000000;
1610 src += page_size;
1611 }
1612 sum1 = restore1Sum;
1613
1614 // write the __HIB seg, with zeros for the stack
1615
1616 src = (uint8_t *) trunc_page(hibernateBase);
1617 count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
1618 if (count)
1619 {
1620 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
1621 if (kIOReturnSuccess != err)
1622 break;
1623 }
1624 err = IOHibernatePolledFileWrite(vars->fileVars,
1625 (uint8_t *) 0,
1626 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
1627 cryptvars);
1628 if (kIOReturnSuccess != err)
1629 break;
1630 src = &gIOHibernateRestoreStackEnd[0];
1631 count = round_page(hibernateEnd) - ((uintptr_t) src);
1632 if (count)
1633 {
1634 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
1635 if (kIOReturnSuccess != err)
1636 break;
1637 }
1638
1639 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1640 {
1641 vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
1642 vars->fileVars->encryptEnd = UINT64_MAX;
1643 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1644 }
1645
1646 // write the preview buffer
1647
1648 if (vars->previewBuffer)
1649 {
1650 ppnum = 0;
1651 count = 0;
1652 do
1653 {
1654 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
1655 pageAndCount[0] = atop_64(phys64);
1656 pageAndCount[1] = atop_32(segLen);
1657 err = IOHibernatePolledFileWrite(vars->fileVars,
1658 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1659 cryptvars);
1660 if (kIOReturnSuccess != err)
1661 break;
1662 count += segLen;
1663 ppnum += sizeof(pageAndCount);
1664 }
1665 while (phys64);
1666 if (kIOReturnSuccess != err)
1667 break;
1668
1669 src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
1670
1671 ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
1672
1673 count = vars->previewBuffer->getLength();
1674
1675 header->previewPageListSize = ppnum;
1676 header->previewSize = count + ppnum;
1677
1678 for (page = 0; page < count; page += page_size)
1679 {
1680 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
1681 sum1 += hibernate_sum_page(src + page, atop_64(phys64));
1682 }
1683 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
1684 if (kIOReturnSuccess != err)
1685 break;
1686 }
1687
1688 // mark areas for no save
1689 IOMemoryDescriptor * ioBuffer;
1690 ioBuffer = IOPolledFileGetIOBuffer(vars->fileVars);
1691 for (count = 0;
1692 (phys64 = ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1693 count += segLen)
1694 {
1695 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1696 atop_64(phys64), atop_32(segLen),
1697 kIOHibernatePageStateFree);
1698 pageCount -= atop_32(segLen);
1699 }
1700
1701 for (count = 0;
1702 (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1703 count += segLen)
1704 {
1705 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1706 atop_64(phys64), atop_32(segLen),
1707 kIOHibernatePageStateFree);
1708 pageCount -= atop_32(segLen);
1709 }
1710
1711 // copy out bitmap of pages available for trashing during restore
1712
1713 bitmap_size = vars->page_list_wired->list_size;
1714 src = (uint8_t *) vars->page_list_wired;
1715 err = IOHibernatePolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
1716 if (kIOReturnSuccess != err)
1717 break;
1718
1719 // mark more areas for no save, but these are not available
1720 // for trashing during restore
1721
1722 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
1723
1724
1725 page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
1726 count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
1727 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1728 page, count,
1729 kIOHibernatePageStateFree);
1730 pageCount -= count;
1731
1732 if (vars->previewBuffer) for (count = 0;
1733 (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1734 count += segLen)
1735 {
1736 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1737 atop_64(phys64), atop_32(segLen),
1738 kIOHibernatePageStateFree);
1739 pageCount -= atop_32(segLen);
1740 }
1741
1742 for (count = 0;
1743 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1744 count += segLen)
1745 {
1746 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1747 atop_64(phys64), atop_32(segLen),
1748 kIOHibernatePageStateFree);
1749 pageCount -= atop_32(segLen);
1750 }
1751
1752 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
1753 compressed = src + page_size;
1754 scratch = compressed + page_size;
1755
1756 pagesDone = 0;
1757 lastBlob = 0;
1758
1759 HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
1760 bitmap_size, header->previewSize,
1761 pageCount, vars->fileVars->position);
1762
1763 enum
1764 // pageType
1765 {
1766 kWired = 0x02,
1767 kEncrypt = 0x01,
1768 kWiredEncrypt = kWired | kEncrypt,
1769 kWiredClear = kWired,
1770 kUnwiredEncrypt = kEncrypt
1771 };
1772
1773 bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features()));
1774 #define _pmap_is_noencrypt(x) (cpuAES ? false : pmap_is_noencrypt((x)))
1775
1776 for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)
1777 {
1778 if (kUnwiredEncrypt == pageType)
1779 {
1780 // start unwired image
1781 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1782 {
1783 vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
1784 vars->fileVars->encryptEnd = UINT64_MAX;
1785 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1786 }
1787 bcopy(&cryptvars->aes_iv[0],
1788 &gIOHibernateCryptWakeContext.aes_iv[0],
1789 sizeof(cryptvars->aes_iv));
1790 cryptvars = &gIOHibernateCryptWakeContext;
1791 }
1792 for (iterDone = false, ppnum = 0; !iterDone; )
1793 {
1794 count = hibernate_page_list_iterate((kWired & pageType)
1795 ? vars->page_list_wired : vars->page_list,
1796 &ppnum);
1797 // kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
1798 iterDone = !count;
1799
1800 if (count && (kWired & pageType) && needEncrypt)
1801 {
1802 uint32_t checkIndex;
1803 for (checkIndex = 0;
1804 (checkIndex < count)
1805 && (((kEncrypt & pageType) == 0) == _pmap_is_noencrypt(ppnum + checkIndex));
1806 checkIndex++)
1807 {}
1808 if (!checkIndex)
1809 {
1810 ppnum++;
1811 continue;
1812 }
1813 count = checkIndex;
1814 }
1815
1816 switch (pageType)
1817 {
1818 case kWiredEncrypt: wiredPagesEncrypted += count; break;
1819 case kWiredClear: wiredPagesClear += count; break;
1820 case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
1821 }
1822
1823 if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */}
1824 else
1825 {
1826 pageAndCount[0] = ppnum;
1827 pageAndCount[1] = count;
1828 err = IOHibernatePolledFileWrite(vars->fileVars,
1829 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1830 cryptvars);
1831 if (kIOReturnSuccess != err)
1832 break;
1833 }
1834
1835 for (page = ppnum; page < (ppnum + count); page++)
1836 {
1837 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
1838 if (err)
1839 {
1840 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
1841 break;
1842 }
1843
1844 sum = hibernate_sum_page(src, page);
1845 if (kWired & pageType)
1846 sum1 += sum;
1847 else
1848 sum2 += sum;
1849
1850 clock_get_uptime(&startTime);
1851 wkresult = WKdm_compress_new((const WK_word*) src,
1852 (WK_word*) compressed,
1853 (WK_word*) scratch,
1854 page_size - 4);
1855
1856 clock_get_uptime(&endTime);
1857 ADD_ABSOLUTETIME(&compTime, &endTime);
1858 SUB_ABSOLUTETIME(&compTime, &startTime);
1859
1860 compBytes += page_size;
1861 pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
1862
1863 if (pageCompressedSize == 0)
1864 {
1865 pageCompressedSize = 4;
1866 data = src;
1867
1868 if (*(uint32_t *)src)
1869 svPageCount++;
1870 else
1871 zvPageCount++;
1872 }
1873 else
1874 {
1875 if (pageCompressedSize != page_size)
1876 data = compressed;
1877 else
1878 data = src;
1879 }
1880
1881 tag = pageCompressedSize | kIOHibernateTagSignature;
1882 err = IOHibernatePolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
1883 if (kIOReturnSuccess != err)
1884 break;
1885
1886 err = IOHibernatePolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
1887 if (kIOReturnSuccess != err)
1888 break;
1889
1890 compressedSize += pageCompressedSize;
1891 uncompressedSize += page_size;
1892 pagesDone++;
1893
1894 if (vars->consoleMapping && (0 == (1023 & pagesDone)))
1895 {
1896 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
1897 if (blob != lastBlob)
1898 {
1899 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
1900 lastBlob = blob;
1901 }
1902 }
1903 if (0 == (8191 & pagesDone))
1904 {
1905 clock_get_uptime(&endTime);
1906 SUB_ABSOLUTETIME(&endTime, &allTime);
1907 absolutetime_to_nanoseconds(endTime, &nsec);
1908 progressStamp = nsec / 750000000ULL;
1909 if (progressStamp != lastProgressStamp)
1910 {
1911 lastProgressStamp = progressStamp;
1912 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
1913 }
1914 }
1915 }
1916 if (kIOReturnSuccess != err)
1917 break;
1918 ppnum = page;
1919 }
1920
1921 if (kIOReturnSuccess != err)
1922 break;
1923
1924 if ((kEncrypt & pageType) && vars->fileVars->encryptStart)
1925 {
1926 vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
1927 HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
1928 }
1929
1930 if (kWiredEncrypt != pageType)
1931 {
1932 // end of image1/2 - fill to next block
1933 err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
1934 if (kIOReturnSuccess != err)
1935 break;
1936 }
1937 if (kWiredClear == pageType)
1938 {
1939 // enlarge wired image for test
1940 // err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
1941
1942 // end wired image
1943 header->encryptStart = vars->fileVars->encryptStart;
1944 header->encryptEnd = vars->fileVars->encryptEnd;
1945 image1Size = vars->fileVars->position;
1946 HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
1947 image1Size, header->encryptStart, header->encryptEnd);
1948 }
1949 }
1950 if (kIOReturnSuccess != err)
1951 {
1952 if (kIOReturnOverrun == err)
1953 {
1954 // update actual compression ratio on not enough space (for retry)
1955 gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
1956 }
1957
1958 // update partial amount written (for IOPolledFileClose cleanup/unmap)
1959 header->imageSize = vars->fileVars->position;
1960 break;
1961 }
1962
1963 // Header:
1964
1965 header->imageSize = vars->fileVars->position;
1966 header->image1Size = image1Size;
1967 header->bitmapSize = bitmap_size;
1968 header->pageCount = pageCount;
1969
1970 header->restore1Sum = restore1Sum;
1971 header->image1Sum = sum1;
1972 header->image2Sum = sum2;
1973 header->sleepTime = gIOLastSleepTime.tv_sec;
1974
1975 header->compression = (compressedSize << 8) / uncompressedSize;
1976 gIOHibernateCompression = header->compression;
1977
1978 count = vars->fileVars->fileExtents->getLength();
1979 if (count > sizeof(header->fileExtentMap))
1980 {
1981 header->fileExtentMapSize = count;
1982 count = sizeof(header->fileExtentMap);
1983 }
1984 else
1985 header->fileExtentMapSize = sizeof(header->fileExtentMap);
1986 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
1987
1988 header->deviceBase = vars->fileVars->block0;
1989 header->deviceBlockSize = vars->fileVars->blockSize;
1990
1991 IOPolledFileSeek(vars->fileVars, 0);
1992 err = IOHibernatePolledFileWrite(vars->fileVars,
1993 (uint8_t *) header, sizeof(IOHibernateImageHeader),
1994 cryptvars);
1995 if (kIOReturnSuccess != err)
1996 break;
1997 err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
1998 }
1999 while (false);
2000
2001 clock_get_uptime(&endTime);
2002
2003 IOService::getPMRootDomain()->pmStatsRecordEvent(
2004 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
2005
2006 SUB_ABSOLUTETIME(&endTime, &allTime);
2007 absolutetime_to_nanoseconds(endTime, &nsec);
2008 HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
2009
2010 absolutetime_to_nanoseconds(compTime, &nsec);
2011 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2012 compBytes,
2013 nsec / 1000000ULL,
2014 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2015
2016 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2017 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
2018 vars->fileVars->cryptBytes,
2019 nsec / 1000000ULL,
2020 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2021
2022 HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
2023 header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
2024 uncompressedSize, atop_32(uncompressedSize), compressedSize,
2025 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
2026 sum1, sum2);
2027
2028 HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
2029 svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
2030
2031 if (pollerOpen)
2032 IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted );
2033
2034 if (vars->consoleMapping)
2035 ProgressUpdate(gIOHibernateGraphicsInfo,
2036 vars->consoleMapping, 0, kIOHibernateProgressCount);
2037
2038 HIBLOG("hibernate_write_image done(%x)\n", err);
2039
2040 // should we come back via regular wake, set the state in memory.
2041 gIOHibernateState = kIOHibernateStateInactive;
2042
2043 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END,
2044 wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted, 0, 0);
2045
2046 if (kIOReturnSuccess == err)
2047 {
2048 if (kIOHibernateModeSleep & gIOHibernateMode)
2049 {
2050 return (kIOHibernatePostWriteSleep);
2051 }
2052 else if(kIOHibernateModeRestart & gIOHibernateMode)
2053 {
2054 return (kIOHibernatePostWriteRestart);
2055 }
2056 else
2057 {
2058 /* by default, power down */
2059 return (kIOHibernatePostWriteHalt);
2060 }
2061 }
2062 else if (kIOReturnAborted == err)
2063 {
2064 return (kIOHibernatePostWriteWake);
2065 }
2066 else
2067 {
2068 /* on error, sleep */
2069 return (kIOHibernatePostWriteSleep);
2070 }
2071 }
2072
2073 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2074
2075 extern "C" void
2076 hibernate_machine_init(void)
2077 {
2078 IOReturn err;
2079 uint32_t sum;
2080 uint32_t pagesDone;
2081 uint32_t pagesRead = 0;
2082 AbsoluteTime startTime, compTime;
2083 AbsoluteTime allTime, endTime;
2084 AbsoluteTime startIOTime, endIOTime;
2085 uint64_t nsec, nsecIO;
2086 uint64_t compBytes;
2087 uint32_t lastProgressStamp = 0;
2088 uint32_t progressStamp;
2089 IOPolledFileCryptVars * cryptvars = 0;
2090
2091 IOHibernateVars * vars = &gIOHibernateVars;
2092 bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
2093
2094 if (!vars->fileVars || !vars->fileVars->pollers)
2095 return;
2096
2097 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2098 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2099
2100 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
2101 {
2102 HIBLOG("regular wake\n");
2103 return;
2104 }
2105
2106 HIBPRINT("diag %x %x %x %x\n",
2107 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
2108 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
2109
2110 #define t40ms(x) (tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)
2111 #define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
2112 tStat(booterStart, booterStart);
2113 gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart;
2114 tStat(booterDuration0, booterTime0);
2115 tStat(booterDuration1, booterTime1);
2116 tStat(booterDuration2, booterTime2);
2117 tStat(booterDuration, booterTime);
2118 tStat(booterConnectDisplayDuration, connectDisplayTime);
2119 tStat(booterSplashDuration, splashTime);
2120 tStat(trampolineDuration, trampolineTime);
2121
2122 gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size;
2123 gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize;
2124 gIOHibernateStats->image1Pages = pagesDone;
2125
2126 HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
2127 gIOHibernateStats->booterStart,
2128 gIOHibernateStats->smcStart,
2129 gIOHibernateStats->booterDuration0,
2130 gIOHibernateStats->booterDuration1,
2131 gIOHibernateStats->booterDuration2,
2132 gIOHibernateStats->booterDuration,
2133 gIOHibernateStats->booterConnectDisplayDuration,
2134 gIOHibernateStats->booterSplashDuration,
2135 gIOHibernateStats->trampolineDuration);
2136
2137 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
2138 gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
2139 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
2140
2141 if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
2142 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)))
2143 {
2144 hibernate_page_list_discard(vars->page_list);
2145 }
2146
2147 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
2148
2149 if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount)
2150 panic("handoff overflow");
2151
2152 IOHibernateHandoff * handoff;
2153 bool done = false;
2154 bool foundCryptData = false;
2155
2156 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
2157 !done;
2158 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
2159 {
2160 // HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
2161 uint8_t * data = &handoff->data[0];
2162 switch (handoff->type)
2163 {
2164 case kIOHibernateHandoffTypeEnd:
2165 done = true;
2166 break;
2167
2168 case kIOHibernateHandoffTypeGraphicsInfo:
2169 if (handoff->bytecount == sizeof(*gIOHibernateGraphicsInfo))
2170 {
2171 bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
2172 }
2173 break;
2174
2175 case kIOHibernateHandoffTypeCryptVars:
2176 if (cryptvars)
2177 {
2178 hibernate_cryptwakevars_t *
2179 wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
2180 bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
2181 }
2182 foundCryptData = true;
2183 bzero(data, handoff->bytecount);
2184 break;
2185
2186 case kIOHibernateHandoffTypeMemoryMap:
2187
2188 clock_get_uptime(&allTime);
2189
2190 hibernate_newruntime_map(data, handoff->bytecount,
2191 gIOHibernateCurrentHeader->systemTableOffset);
2192
2193 clock_get_uptime(&endTime);
2194
2195 SUB_ABSOLUTETIME(&endTime, &allTime);
2196 absolutetime_to_nanoseconds(endTime, &nsec);
2197
2198 HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
2199
2200 break;
2201
2202 case kIOHibernateHandoffTypeDeviceTree:
2203 {
2204 // DTEntry chosen = NULL;
2205 // HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen));
2206 }
2207 break;
2208
2209 default:
2210 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
2211 break;
2212 }
2213 }
2214 if (cryptvars && !foundCryptData)
2215 panic("hibernate handoff");
2216
2217 HIBPRINT("video 0x%llx %d %d %d status %x\n",
2218 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2219 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
2220
2221 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)
2222 {
2223 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2224 * gIOHibernateGraphicsInfo->rowBytes);
2225 if (vars->videoMapSize > vars->videoAllocSize) vars->videoMapSize = 0;
2226 else
2227 {
2228 IOMapPages(kernel_map,
2229 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2230 vars->videoMapSize, kIOMapInhibitCache );
2231 }
2232 }
2233
2234 if (vars->videoMapSize)
2235 ProgressUpdate(gIOHibernateGraphicsInfo,
2236 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2237
2238 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2239 uint8_t * compressed = src + page_size;
2240 uint8_t * scratch = compressed + page_size;
2241 uint32_t decoOffset;
2242
2243 clock_get_uptime(&allTime);
2244 AbsoluteTime_to_scalar(&compTime) = 0;
2245 compBytes = 0;
2246
2247 HIBLOG("IOPolledFilePollersOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2248 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledAfterSleepState, false);
2249 clock_get_uptime(&startIOTime);
2250 endTime = startIOTime;
2251 SUB_ABSOLUTETIME(&endTime, &allTime);
2252 absolutetime_to_nanoseconds(endTime, &nsec);
2253 HIBLOG("IOPolledFilePollersOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
2254
2255 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2256
2257 // kick off the read ahead
2258 vars->fileVars->bufferHalf = 0;
2259 vars->fileVars->bufferLimit = 0;
2260 vars->fileVars->lastRead = 0;
2261 vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
2262 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2263 vars->fileVars->cryptBytes = 0;
2264 AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
2265
2266 err = IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
2267 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2268 // --
2269
2270 HIBLOG("hibernate_machine_init reading\n");
2271
2272 uint32_t * header = (uint32_t *) src;
2273 sum = 0;
2274
2275 while (kIOReturnSuccess == err)
2276 {
2277 unsigned int count;
2278 unsigned int page;
2279 uint32_t tag;
2280 vm_offset_t ppnum, compressedSize;
2281
2282 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2283 if (kIOReturnSuccess != err)
2284 break;
2285
2286 ppnum = header[0];
2287 count = header[1];
2288
2289 // HIBPRINT("(%x, %x)\n", ppnum, count);
2290
2291 if (!count)
2292 break;
2293
2294 for (page = 0; page < count; page++)
2295 {
2296 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2297 if (kIOReturnSuccess != err)
2298 break;
2299
2300 compressedSize = kIOHibernateTagLength & tag;
2301 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength))
2302 {
2303 err = kIOReturnIPCError;
2304 break;
2305 }
2306
2307 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2308 if (kIOReturnSuccess != err) break;
2309
2310 if (compressedSize < page_size)
2311 {
2312 decoOffset = page_size;
2313 clock_get_uptime(&startTime);
2314
2315 if (compressedSize == 4) {
2316 int i;
2317 uint32_t *s, *d;
2318
2319 s = (uint32_t *)src;
2320 d = (uint32_t *)(uintptr_t)compressed;
2321
2322 for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++)
2323 *d++ = *s;
2324 }
2325 else
2326 WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, compressedSize);
2327 clock_get_uptime(&endTime);
2328 ADD_ABSOLUTETIME(&compTime, &endTime);
2329 SUB_ABSOLUTETIME(&compTime, &startTime);
2330 compBytes += page_size;
2331 }
2332 else decoOffset = 0;
2333
2334 sum += hibernate_sum_page((src + decoOffset), ppnum);
2335 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2336 if (err)
2337 {
2338 HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
2339 break;
2340 }
2341
2342 ppnum++;
2343 pagesDone++;
2344 pagesRead++;
2345
2346 if (0 == (8191 & pagesDone))
2347 {
2348 clock_get_uptime(&endTime);
2349 SUB_ABSOLUTETIME(&endTime, &allTime);
2350 absolutetime_to_nanoseconds(endTime, &nsec);
2351 progressStamp = nsec / 750000000ULL;
2352 if (progressStamp != lastProgressStamp)
2353 {
2354 lastProgressStamp = progressStamp;
2355 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2356 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2357 }
2358 }
2359 }
2360 }
2361 if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages))
2362 err = kIOReturnLockedRead;
2363
2364 if (kIOReturnSuccess != err)
2365 panic("Hibernate restore error %x", err);
2366
2367 gIOHibernateCurrentHeader->actualImage2Sum = sum;
2368 gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
2369
2370 clock_get_uptime(&endIOTime);
2371
2372 err = IOPolledFilePollersClose(vars->fileVars, kIOPolledAfterSleepState);
2373
2374 clock_get_uptime(&endTime);
2375
2376 IOService::getPMRootDomain()->pmStatsRecordEvent(
2377 kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
2378 IOService::getPMRootDomain()->pmStatsRecordEvent(
2379 kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
2380
2381 SUB_ABSOLUTETIME(&endTime, &allTime);
2382 absolutetime_to_nanoseconds(endTime, &nsec);
2383
2384 SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
2385 absolutetime_to_nanoseconds(endIOTime, &nsecIO);
2386
2387 gIOHibernateStats->kernelImageReadDuration = nsec / 1000000ULL;
2388 gIOHibernateStats->imagePages = pagesDone;
2389
2390 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
2391 pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
2392 nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
2393
2394 absolutetime_to_nanoseconds(compTime, &nsec);
2395 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2396 compBytes,
2397 nsec / 1000000ULL,
2398 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2399
2400 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2401 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
2402 vars->fileVars->cryptBytes,
2403 nsec / 1000000ULL,
2404 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2405
2406 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_NONE, pagesRead, pagesDone, 0, 0, 0);
2407 }
2408
2409 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2410
2411 void IOHibernateSetWakeCapabilities(uint32_t capability)
2412 {
2413 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
2414 {
2415 gIOHibernateStats->wakeCapability = capability;
2416
2417 if (kIOPMSystemCapabilityGraphics & capability)
2418 {
2419 vm_compressor_do_warmup();
2420 }
2421 }
2422 }
2423
2424 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2425
2426 void IOHibernateSystemRestart(void)
2427 {
2428 static uint8_t noteStore[32] __attribute__((aligned(32)));
2429 IORegistryEntry * regEntry;
2430 const OSSymbol * sym;
2431 OSData * noteProp;
2432 OSData * data;
2433 uintptr_t * smcVars;
2434 uint8_t * smcBytes;
2435 size_t len;
2436 addr64_t element;
2437
2438 data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
2439 if (!data) return;
2440
2441 smcVars = (typeof(smcVars)) data->getBytesNoCopy();
2442 smcBytes = (typeof(smcBytes)) smcVars[1];
2443 len = smcVars[0];
2444 if (len > sizeof(noteStore)) len = sizeof(noteStore);
2445 noteProp = OSData::withCapacity(3 * sizeof(element));
2446 if (!noteProp) return;
2447 element = len;
2448 noteProp->appendBytes(&element, sizeof(element));
2449 element = crc32(0, smcBytes, len);
2450 noteProp->appendBytes(&element, sizeof(element));
2451
2452 bcopy(smcBytes, noteStore, len);
2453 element = (addr64_t) &noteStore[0];
2454 element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
2455 noteProp->appendBytes(&element, sizeof(element));
2456
2457 if (!gIOOptionsEntry)
2458 {
2459 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
2460 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
2461 if (regEntry && !gIOOptionsEntry)
2462 regEntry->release();
2463 }
2464
2465 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
2466 if (gIOOptionsEntry && sym) gIOOptionsEntry->setProperty(sym, noteProp);
2467 if (noteProp) noteProp->release();
2468 if (sym) sym->release();
2469 }
2470
2471
2472
mirror of XNU