]> git.saurik.com Git - apple/xnu.git/blob - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-3247.1.106.tar.gz
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
1 /*
2 * Copyright (c) 2004-2008 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29
30 /*
31
32 Sleep:
33
34 - PMRootDomain calls IOHibernateSystemSleep() before system sleep
35 (devices awake, normal execution context)
36 - IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
37 grabs its extents and searches for a polling driver willing to work with that IOMedia.
38 The BSD code makes an ioctl to the storage driver to get the partition base offset to
39 the disk, and other ioctls to get the transfer constraints
40 If successful, the file is written to make sure its initially not bootable (in case of
41 later failure) and nvram set to point to the first block of the file. (Has to be done
42 here so blocking is possible in nvram support).
43 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
44 page out any pages it wants to (currently zero, but probably some percentage of memory).
45 Its assumed just allocating pages will cause the VM system to naturally select the best
46 pages for eviction. It also copies processor flags needed for the restore path and sets
47 a flag in the boot processor proc info.
48 gIOHibernateState = kIOHibernateStateHibernating.
49 - Regular sleep progresses - some drivers may inspect the root domain property
50 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
51 as usual but leaves motherboard I/O on.
52 - Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
53 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
54 all ppc RC bits out of the hash table and caches into the mapping structures.
55 - hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
56 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
57 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
58 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
59 The image header and block list are written. The header includes the second file extent so
60 only the header block is needed to read the file, regardless of filesystem.
61 The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
62 (only) is used to decompress the image during wake/boot.
63 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
64 The bitmaps are written to the image.
65 More areas are removed from the bitmaps (after they have been written to the image) - the
66 segment "__HIB" pages and interrupt stack.
67 Each wired page is compressed and written and then each non-wired page. Compression and
68 disk writes are in parallel.
69 The image header is written to the start of the file and the polling driver closed.
70 The machine powers down (or sleeps).
71
72 Boot/Wake:
73
74 - BootX sees the boot-image nvram variable containing the device and block number of the image,
75 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
76 - BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
77 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
78 that is in the kernel's __HIB section.
79 - BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
80 hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
81 only code & data in that section is safe to call since all the other wired pages are still
82 compressed in the image.
83 - hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
84 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
85 location directly, and copies those that can't to interim free pages. When the image has been
86 completed, the copies are uncompressed, overwriting the wired image pages.
87 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
88 is used to get pages into place for 64bit.
89 - the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
90 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
91 are removed from the software strutures, and the hash table is reinitialized.
92 - After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
93 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
94 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
95 for the remaining non wired pages.
96 - The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
97 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
98 is closed via bsd.
99
100 Polled Mode I/O:
101
102 IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
103 registry, specifying an object of calls IOPolledInterface.
104
105 Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
106 partition) that the image is going to live, looking for polled interface properties. If it finds
107 one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
108 interfaces found are kept in an ordered list.
109
110 There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
111 few different contexts things happen in:
112
113 - there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
114 up and running) and after wake - this is safe to allocate memory and do anything. The device
115 ignores sleep requests from that point since its a waste of time if it goes to sleep and
116 immediately wakes back up for the image write.
117
118 - there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
119 immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
120 from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
121 used to flush and set the disk to sleep.
122
123 - there is an Open/Close (AfterSleep) pair made around the image read operations that happen
124 immediately after sleep. These can't block or allocate memory. This is happening after the platform
125 expert has woken the low level bits of the system, but most of the I/O system has not. There is only
126 one thread running.
127
128 For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
129 (prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
130 the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
131 Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
132 that is called for the hardware to check for events, and complete the I/O via the callback.
133 The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
134 to restrict I/O ops.
135 */
136
137 #include <sys/systm.h>
138
139 #include <IOKit/IOWorkLoop.h>
140 #include <IOKit/IOCommandGate.h>
141 #include <IOKit/IOTimerEventSource.h>
142 #include <IOKit/IOPlatformExpert.h>
143 #include <IOKit/IOKitDebug.h>
144 #include <IOKit/IOTimeStamp.h>
145 #include <IOKit/pwr_mgt/RootDomain.h>
146 #include <IOKit/pwr_mgt/IOPMPrivate.h>
147 #include <IOKit/IOMessage.h>
148 #include <IOKit/IODeviceTreeSupport.h>
149 #include <IOKit/IOBSD.h>
150 #include "RootDomainUserClient.h"
151 #include <IOKit/pwr_mgt/IOPowerConnection.h>
152 #include "IOPMPowerStateQueue.h"
153 #include <IOKit/IOBufferMemoryDescriptor.h>
154 #include <IOKit/AppleKeyStoreInterface.h>
155 #include <libkern/crypto/aes.h>
156
157 #include <sys/uio.h>
158 #include <sys/conf.h>
159 #include <sys/stat.h>
160 #include <sys/fcntl.h> // (FWRITE, ...)
161 #include <sys/sysctl.h>
162 #include <sys/kdebug.h>
163 #include <stdint.h>
164
165 #include <IOKit/IOHibernatePrivate.h>
166 #include <IOKit/IOPolledInterface.h>
167 #include <IOKit/IONVRAM.h>
168 #include "IOHibernateInternal.h"
169 #include <vm/WKdm_new.h>
170 #include "IOKitKernelInternal.h"
171 #include <pexpert/device_tree.h>
172
173 #include <machine/pal_routines.h>
174 #include <machine/pal_hibernate.h>
175 #include <i386/tsc.h>
176
177 extern "C" addr64_t kvtophys(vm_offset_t va);
178 extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
179
180 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
181
182 #define DISABLE_TRIM 0
183 #define TRIM_DELAY 5000
184
185 extern boolean_t root_is_CF_drive;
186 extern unsigned int save_kdebug_enable;
187 extern uint32_t gIOHibernateState;
188 uint32_t gIOHibernateMode;
189 static char gIOHibernateBootSignature[256+1];
190 static char gIOHibernateFilename[MAXPATHLEN+1];
191 static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
192 uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
193 static uint64_t gIOHibernateCompression = 0x80; // default compression 50%
194
195 static IODTNVRAM * gIOOptionsEntry;
196 static IORegistryEntry * gIOChosenEntry;
197
198 static const OSSymbol * gIOHibernateBootImageKey;
199
200 #if defined(__i386__) || defined(__x86_64__)
201
202 static const OSSymbol * gIOHibernateRTCVariablesKey;
203 static const OSSymbol * gIOHibernateBoot0082Key;
204 static const OSSymbol * gIOHibernateBootNextKey;
205 static OSData * gIOHibernateBoot0082Data;
206 static OSData * gIOHibernateBootNextData;
207 static OSObject * gIOHibernateBootNextSave;
208
209 static IOPolledFileIOVars * gDebugImageFileVars;
210 static IOLock * gDebugImageLock;
211
212 #endif /* defined(__i386__) || defined(__x86_64__) */
213
214 static IOLock * gFSLock;
215 static uint32_t gFSState;
216 static IOPolledFileIOVars gFileVars;
217 static IOHibernateVars gIOHibernateVars;
218 static IOPolledFileCryptVars gIOHibernateCryptWakeContext;
219 static hibernate_graphics_t _hibernateGraphics;
220 static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
221 static hibernate_statistics_t _hibernateStats;
222 static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats;
223
224 enum
225 {
226 kFSIdle = 0,
227 kFSOpening = 2,
228 kFSOpened = 3,
229 kFSTimedOut = 4,
230 };
231
232 static IOReturn IOHibernateDone(IOHibernateVars * vars);
233 static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature);
234 static void IOSetBootImageNVRAM(OSData * data);
235
236 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
237
238 enum { kDefaultIOSize = 128 * 1024 };
239 enum { kVideoMapSize = 80 * 1024 * 1024 };
240
241 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
242
243 // copy from phys addr to MD
244
245 static IOReturn
246 IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
247 IOByteCount offset, addr64_t bytes, IOByteCount length)
248 {
249 addr64_t srcAddr = bytes;
250 IOByteCount remaining;
251
252 remaining = length = min(length, md->getLength() - offset);
253 while (remaining) { // (process another target segment?)
254 addr64_t dstAddr64;
255 IOByteCount dstLen;
256
257 dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
258 if (!dstAddr64)
259 break;
260
261 // Clip segment length to remaining
262 if (dstLen > remaining)
263 dstLen = remaining;
264
265 #if 1
266 bcopy_phys(srcAddr, dstAddr64, dstLen);
267 #else
268 copypv(srcAddr, dstAddr64, dstLen,
269 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
270 #endif
271 srcAddr += dstLen;
272 offset += dstLen;
273 remaining -= dstLen;
274 }
275
276 assert(!remaining);
277
278 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
279 }
280
281 // copy from MD to phys addr
282
283 static IOReturn
284 IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
285 IOByteCount offset, addr64_t bytes, IOByteCount length)
286 {
287 addr64_t dstAddr = bytes;
288 IOByteCount remaining;
289
290 remaining = length = min(length, md->getLength() - offset);
291 while (remaining) { // (process another target segment?)
292 addr64_t srcAddr64;
293 IOByteCount dstLen;
294
295 srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
296 if (!srcAddr64)
297 break;
298
299 // Clip segment length to remaining
300 if (dstLen > remaining)
301 dstLen = remaining;
302
303 #if 1
304 bcopy_phys(srcAddr64, dstAddr, dstLen);
305 #else
306 copypv(srcAddr, dstAddr64, dstLen,
307 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
308 #endif
309 dstAddr += dstLen;
310 offset += dstLen;
311 remaining -= dstLen;
312 }
313
314 assert(!remaining);
315
316 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
317 }
318
319 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
320
321 void
322 hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
323 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
324 {
325 count += ppnum;
326 switch (kind)
327 {
328 case kIOHibernatePageStateUnwiredSave:
329 // unwired save
330 for (; ppnum < count; ppnum++)
331 {
332 hibernate_page_bitset(page_list, FALSE, ppnum);
333 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
334 }
335 break;
336 case kIOHibernatePageStateWiredSave:
337 // wired save
338 for (; ppnum < count; ppnum++)
339 {
340 hibernate_page_bitset(page_list, FALSE, ppnum);
341 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
342 }
343 break;
344 case kIOHibernatePageStateFree:
345 // free page
346 for (; ppnum < count; ppnum++)
347 {
348 hibernate_page_bitset(page_list, TRUE, ppnum);
349 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
350 }
351 break;
352 default:
353 panic("hibernate_set_page_state");
354 }
355 }
356
357 static vm_offset_t
358 hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
359 {
360 uint32_t page = *pPage;
361 uint32_t count;
362 hibernate_bitmap_t * bitmap;
363
364 while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
365 {
366 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
367 if (!count)
368 break;
369 page += count;
370 if (page <= bitmap->last_page)
371 break;
372 }
373
374 *pPage = page;
375 if (bitmap)
376 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
377 else
378 count = 0;
379
380 return (count);
381 }
382
383 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
384
385 IOReturn
386 IOHibernateSystemSleep(void)
387 {
388 IOReturn err;
389 OSData * nvramData;
390 OSObject * obj;
391 OSString * str;
392 OSNumber * num;
393 bool dsSSD, vmflush;
394 IOHibernateVars * vars;
395 uint64_t setFileSize = 0;
396
397 gIOHibernateState = kIOHibernateStateInactive;
398
399 gIOHibernateDebugFlags = 0;
400 if (kIOLogHibernate & gIOKitDebug)
401 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
402
403 if (IOService::getPMRootDomain()->getHibernateSettings(
404 &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime))
405 {
406 if (kIOHibernateModeSleep & gIOHibernateMode)
407 // default to discard clean for safe sleep
408 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
409 | kIOHibernateModeDiscardCleanActive);
410 }
411
412 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
413 {
414 if ((str = OSDynamicCast(OSString, obj)))
415 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
416 sizeof(gIOHibernateFilename));
417 obj->release();
418 }
419
420 if (!gIOHibernateMode || !gIOHibernateFilename[0])
421 return (kIOReturnUnsupported);
422
423 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
424
425 vars = IONew(IOHibernateVars, 1);
426 if (!vars) return (kIOReturnNoMemory);
427 bzero(vars, sizeof(*vars));
428
429 IOLockLock(gFSLock);
430 if (kFSIdle != gFSState)
431 {
432 HIBLOG("hibernate file busy\n");
433 IOLockUnlock(gFSLock);
434 IODelete(vars, IOHibernateVars, 1);
435 return (kIOReturnBusy);
436 }
437 gFSState = kFSOpening;
438 IOLockUnlock(gFSLock);
439
440 do
441 {
442 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
443 2 * page_size + WKdm_SCRATCH_BUF_SIZE, page_size);
444
445 vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
446 ptoa_64(gIOHibernateHandoffPageCount), page_size);
447
448 if (!vars->srcBuffer || !vars->handoffBuffer)
449 {
450 err = kIOReturnNoMemory;
451 break;
452 }
453
454 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey)))
455 {
456 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMinSize = num->unsigned64BitValue();
457 obj->release();
458 }
459 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey)))
460 {
461 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMaxSize = num->unsigned64BitValue();
462 obj->release();
463 }
464
465 boolean_t encryptedswap = true;
466 uint32_t pageCount;
467 AbsoluteTime startTime, endTime;
468 uint64_t nsec;
469
470 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
471 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
472 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
473
474 vmflush = ((kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)) && root_is_CF_drive == FALSE);
475 err = hibernate_alloc_page_lists(&vars->page_list,
476 &vars->page_list_wired,
477 &vars->page_list_pal);
478 if (KERN_SUCCESS != err)
479 break;
480
481 if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode))
482 {
483 hibernate_page_list_setall(vars->page_list,
484 vars->page_list_wired,
485 vars->page_list_pal,
486 true /* preflight */,
487 vmflush /* discard */,
488 &pageCount);
489 PE_Video consoleInfo;
490 bzero(&consoleInfo, sizeof(consoleInfo));
491 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
492
493 // estimate: 6% increase in pages compressed
494 // screen preview 2 images compressed 0%
495 setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
496 + vars->page_list->list_size
497 + (consoleInfo.v_width * consoleInfo.v_height * 8);
498 enum { setFileRound = 1024*1024ULL };
499 setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
500
501 HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
502 pageCount, (100ULL * gIOHibernateCompression) >> 8,
503 setFileSize, vars->fileMinSize);
504
505 if (!(kIOHibernateModeFileResize & gIOHibernateMode)
506 && (setFileSize < vars->fileMinSize))
507 {
508 setFileSize = vars->fileMinSize;
509 }
510 }
511
512 // Invalidate the image file
513 if (gDebugImageLock) {
514 IOLockLock(gDebugImageLock);
515 if (gDebugImageFileVars != 0) {
516 kprintf("IOHIBSystemSleep: Closing debugdata file\n");
517 IOSetBootImageNVRAM(0);
518 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
519 }
520 IOLockUnlock(gDebugImageLock);
521 }
522
523 err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, 0,
524 gIOHibernateCurrentHeader, sizeof(gIOHibernateCurrentHeader),
525 &vars->fileVars, &nvramData,
526 &vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
527
528 if (KERN_SUCCESS != err)
529 {
530 IOLockLock(gFSLock);
531 if (kFSOpening != gFSState) err = kIOReturnTimeout;
532 IOLockUnlock(gFSLock);
533 }
534
535 if (KERN_SUCCESS != err)
536 {
537 HIBLOG("IOPolledFileOpen(%x)\n", err);
538 break;
539 }
540
541 // write extents for debug data usage in EFI
542 IOWriteExtentsToFile(vars->fileVars, kIOHibernateHeaderOpenSignature);
543
544 err = IOPolledFilePollersSetup(vars->fileVars, kIOPolledPreflightState);
545 if (KERN_SUCCESS != err) break;
546
547 clock_get_uptime(&startTime);
548 err = hibernate_setup(gIOHibernateCurrentHeader,
549 vmflush,
550 vars->page_list, vars->page_list_wired, vars->page_list_pal);
551 clock_get_uptime(&endTime);
552 SUB_ABSOLUTETIME(&endTime, &startTime);
553 absolutetime_to_nanoseconds(endTime, &nsec);
554 HIBLOG("hibernate_setup(%d) took %qd ms\n", err, nsec / 1000000ULL);
555 if (KERN_SUCCESS != err) break;
556
557 dsSSD = ((0 != (kIOPolledFileSSD & vars->fileVars->flags))
558 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
559
560 if (dsSSD) gIOHibernateCurrentHeader->options |= kIOHibernateOptionSSD | kIOHibernateOptionColor;
561 else gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
562
563
564 #if defined(__i386__) || defined(__x86_64__)
565 if (!uuid_is_null(vars->volumeCryptKey) &&
566 (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey)))
567 {
568 uintptr_t smcVars[2];
569 smcVars[0] = sizeof(vars->volumeCryptKey);
570 smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
571
572 IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
573 bzero(smcVars, sizeof(smcVars));
574 }
575 #endif
576
577
578 if (encryptedswap || !uuid_is_null(vars->volumeCryptKey))
579 gIOHibernateMode ^= kIOHibernateModeEncrypt;
580
581 if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
582 {
583 vars->videoAllocSize = kVideoMapSize;
584 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize, VM_KERN_MEMORY_IOKIT))
585 vars->videoMapping = 0;
586 }
587
588 // generate crypt keys
589 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
590 vars->wiredCryptKey[i] = random();
591 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
592 vars->cryptKey[i] = random();
593
594 // set nvram
595
596 IOSetBootImageNVRAM(nvramData);
597 nvramData->release();
598
599 #if defined(__i386__) || defined(__x86_64__)
600 {
601 struct AppleRTCHibernateVars
602 {
603 uint8_t signature[4];
604 uint32_t revision;
605 uint8_t booterSignature[20];
606 uint8_t wiredCryptKey[16];
607 };
608 AppleRTCHibernateVars rtcVars;
609 OSData * data;
610
611 rtcVars.signature[0] = 'A';
612 rtcVars.signature[1] = 'A';
613 rtcVars.signature[2] = 'P';
614 rtcVars.signature[3] = 'L';
615 rtcVars.revision = 1;
616 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
617 if (gIOHibernateBootSignature[0])
618 {
619 char c;
620 uint8_t value = 0;
621 for (uint32_t i = 0;
622 (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
623 i++)
624 {
625 if (c >= 'a') c -= 'a' - 10;
626 else if (c >= 'A') c -= 'A' - 10;
627 else if (c >= '0') c -= '0';
628 else continue;
629 value = (value << 4) | c;
630 if (i & 1) rtcVars.booterSignature[i >> 1] = value;
631 }
632 }
633 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
634 if (data)
635 {
636 if (gIOHibernateRTCVariablesKey)
637 IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
638 data->release();
639 }
640 if (gIOChosenEntry)
641 {
642 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
643 if (data) gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
644 // set BootNext
645 if (!gIOHibernateBoot0082Data)
646 {
647 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
648 if (data)
649 {
650 // AppleNVRAM_EFI_LOAD_OPTION
651 struct {
652 uint32_t Attributes;
653 uint16_t FilePathLength;
654 uint16_t Desc;
655 } loadOptionHeader;
656 loadOptionHeader.Attributes = 1;
657 loadOptionHeader.FilePathLength = data->getLength();
658 loadOptionHeader.Desc = 0;
659 gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
660 if (gIOHibernateBoot0082Data)
661 {
662 gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
663 gIOHibernateBoot0082Data->appendBytes(data);
664 }
665 }
666 }
667 if (!gIOHibernateBootNextData)
668 {
669 uint16_t bits = 0x0082;
670 gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
671 }
672 if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData)
673 {
674 gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
675 gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
676 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
677 }
678 // BootNext
679 }
680 }
681 #endif /* !i386 && !x86_64 */
682 }
683 while (false);
684
685 IOLockLock(gFSLock);
686 if ((kIOReturnSuccess == err) && (kFSOpening != gFSState))
687 {
688 HIBLOG("hibernate file close due timeout\n");
689 err = kIOReturnTimeout;
690 }
691 if (kIOReturnSuccess == err)
692 {
693 gFSState = kFSOpened;
694 gIOHibernateVars = *vars;
695 gFileVars = *vars->fileVars;
696 gFileVars.allocated = false;
697 gIOHibernateVars.fileVars = &gFileVars;
698 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
699 gIOHibernateState = kIOHibernateStateHibernating;
700 }
701 else
702 {
703 IOPolledFileIOVars * fileVars = vars->fileVars;
704 IOHibernateDone(vars);
705 IOPolledFileClose(&fileVars,
706 #if DISABLE_TRIM
707 0, NULL, 0, 0, 0);
708 #else
709 0, NULL, 0, sizeof(IOHibernateImageHeader), setFileSize);
710 #endif
711 gFSState = kFSIdle;
712 }
713 IOLockUnlock(gFSLock);
714
715 if (vars->fileVars) IODelete(vars->fileVars, IOPolledFileIOVars, 1);
716 IODelete(vars, IOHibernateVars, 1);
717
718 return (err);
719 }
720
721 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
722
723 static void
724 IOSetBootImageNVRAM(OSData * data)
725 {
726 IORegistryEntry * regEntry;
727
728 if (!gIOOptionsEntry)
729 {
730 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
731 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
732 if (regEntry && !gIOOptionsEntry)
733 regEntry->release();
734 }
735 if (gIOOptionsEntry && gIOHibernateBootImageKey)
736 {
737 if (data) gIOOptionsEntry->setProperty(gIOHibernateBootImageKey, data);
738 else
739 {
740 gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
741 gIOOptionsEntry->sync();
742 }
743 }
744 }
745
746 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
747 /*
748 * Writes header to disk with signature, block size and file extents data.
749 * If there are more than 2 extents, then they are written on second block.
750 */
751 static IOReturn
752 IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature)
753 {
754 IOHibernateImageHeader hdr;
755 IOItemCount count;
756 IOReturn err = kIOReturnSuccess;
757 int rc;
758 IOPolledFileExtent * fileExtents;
759
760 fileExtents = (typeof(fileExtents)) vars->fileExtents->getBytesNoCopy(),
761
762 memset(&hdr, 0, sizeof(IOHibernateImageHeader));
763 count = vars->fileExtents->getLength();
764 if (count > sizeof(hdr.fileExtentMap))
765 {
766 hdr.fileExtentMapSize = count;
767 count = sizeof(hdr.fileExtentMap);
768 }
769 else
770 hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap);
771
772 bcopy(fileExtents, &hdr.fileExtentMap[0], count);
773
774 // copy file block extent list if larger than header
775 if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap))
776 {
777 count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap);
778 rc = kern_write_file(vars->fileRef, vars->blockSize,
779 (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)),
780 count, IO_SKIP_ENCRYPTION);
781 if (rc != 0) {
782 HIBLOG("kern_write_file returned %d\n", rc);
783 err = kIOReturnIOError;
784 goto exit;
785 }
786 }
787 hdr.signature = signature;
788 hdr.deviceBlockSize = vars->blockSize;
789
790 rc = kern_write_file(vars->fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION);
791 if (rc != 0) {
792 HIBLOG("kern_write_file returned %d\n", rc);
793 err = kIOReturnIOError;
794 goto exit;
795 }
796
797 exit:
798 return err;
799 }
800
801 void
802 IOOpenDebugDataFile(const char *fname, uint64_t size)
803 {
804 IOReturn err;
805 OSData * imagePath = NULL;
806 uint64_t padding;
807
808 if (!gDebugImageLock) {
809 gDebugImageLock = IOLockAlloc();
810 }
811
812 // Try to get a lock, but don't block for getting lock
813 if (!IOLockTryLock(gDebugImageLock)) {
814 HIBLOG("IOOpenDebugDataFile: Failed to get lock\n");
815 return;
816 }
817
818 if (gDebugImageFileVars || !fname || !size) {
819 HIBLOG("IOOpenDebugDataFile: conditions failed\n");
820 goto exit;
821 }
822
823 padding = (PAGE_SIZE*2); // allocate couple more pages for header and fileextents
824 err = IOPolledFileOpen(fname, size+padding, 32ULL*1024*1024*1024,
825 NULL, 0,
826 &gDebugImageFileVars, &imagePath, NULL, 0);
827
828 if ((kIOReturnSuccess == err) && imagePath)
829 {
830 if ((gDebugImageFileVars->fileSize < (size+padding)) ||
831 (gDebugImageFileVars->fileExtents->getLength() > PAGE_SIZE)) {
832 // Can't use the file
833 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
834 HIBLOG("IOOpenDebugDataFile: too many file extents\n");
835 goto exit;
836 }
837
838 // write extents for debug data usage in EFI
839 IOWriteExtentsToFile(gDebugImageFileVars, kIOHibernateHeaderOpenSignature);
840 IOSetBootImageNVRAM(imagePath);
841 kprintf("IOOpenDebugDataFile: opened debugdata file\n");
842 }
843
844 exit:
845 IOLockUnlock(gDebugImageLock);
846
847 if (imagePath) imagePath->release();
848 return;
849 }
850
851 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
852
853 DECLARE_IOHIBERNATEPROGRESSALPHA
854
855 static void
856 ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
857 {
858 uint32_t rowBytes, pixelShift;
859 uint32_t x, y;
860 int32_t blob;
861 uint32_t alpha, in, color, result;
862 uint8_t * out;
863 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
864
865 rowBytes = display->rowBytes;
866 pixelShift = display->depth >> 4;
867 if (pixelShift < 1) return;
868
869 screen += ((display->width
870 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
871 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
872
873 for (y = 0; y < kIOHibernateProgressHeight; y++)
874 {
875 out = screen + y * rowBytes;
876 for (blob = 0; blob < kIOHibernateProgressCount; blob++)
877 {
878 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
879 for (x = 0; x < kIOHibernateProgressWidth; x++)
880 {
881 alpha = gIOHibernateProgressAlpha[y][x];
882 result = color;
883 if (alpha)
884 {
885 if (0xff != alpha)
886 {
887 if (1 == pixelShift)
888 {
889 in = *((uint16_t *)out) & 0x1f; // 16
890 in = (in << 3) | (in >> 2);
891 }
892 else
893 in = *((uint32_t *)out) & 0xff; // 32
894 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
895 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
896 }
897 if (1 == pixelShift)
898 {
899 result >>= 3;
900 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
901 }
902 else
903 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
904 }
905 out += (1 << pixelShift);
906 }
907 out += (kIOHibernateProgressSpacing << pixelShift);
908 }
909 }
910 }
911
912
913 static void
914 ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
915 {
916 uint32_t rowBytes, pixelShift;
917 uint32_t x, y;
918 int32_t blob, lastBlob;
919 uint32_t alpha, in, color, result;
920 uint8_t * out;
921 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
922
923 pixelShift = display->depth >> 4;
924 if (pixelShift < 1)
925 return;
926
927 rowBytes = display->rowBytes;
928
929 screen += ((display->width
930 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
931 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
932
933 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
934
935 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
936
937 for (y = 0; y < kIOHibernateProgressHeight; y++)
938 {
939 out = screen + y * rowBytes;
940 for (blob = firstBlob; blob <= lastBlob; blob++)
941 {
942 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
943 for (x = 0; x < kIOHibernateProgressWidth; x++)
944 {
945 alpha = gIOHibernateProgressAlpha[y][x];
946 result = color;
947 if (alpha)
948 {
949 if (0xff != alpha)
950 {
951 in = display->progressSaveUnder[blob][saveindex[blob]++];
952 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
953 }
954 if (1 == pixelShift)
955 {
956 result >>= 3;
957 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
958 }
959 else
960 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
961 }
962 out += (1 << pixelShift);
963 }
964 out += (kIOHibernateProgressSpacing << pixelShift);
965 }
966 }
967 }
968
969 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
970
971 IOReturn
972 IOHibernateIOKitSleep(void)
973 {
974 IOReturn ret = kIOReturnSuccess;
975 IOLockLock(gFSLock);
976 if (kFSOpening == gFSState)
977 {
978 gFSState = kFSTimedOut;
979 HIBLOG("hibernate file open timed out\n");
980 ret = kIOReturnTimeout;
981 }
982 IOLockUnlock(gFSLock);
983 return (ret);
984 }
985
986 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
987
988 IOReturn
989 IOHibernateSystemHasSlept(void)
990 {
991 IOReturn ret = kIOReturnSuccess;
992 IOHibernateVars * vars = &gIOHibernateVars;
993 OSObject * obj = 0;
994 OSData * data;
995
996 IOLockLock(gFSLock);
997 if ((kFSOpened != gFSState) && gIOHibernateMode)
998 {
999 ret = kIOReturnTimeout;
1000 }
1001 IOLockUnlock(gFSLock);
1002 if (kIOReturnSuccess != ret) return (ret);
1003
1004 if (gIOHibernateMode) obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
1005 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1006 if (obj && !vars->previewBuffer)
1007 obj->release();
1008
1009 vars->consoleMapping = NULL;
1010 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
1011 {
1012 vars->previewBuffer->release();
1013 vars->previewBuffer = 0;
1014 }
1015
1016 if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1017 && vars->previewBuffer
1018 && (data = OSDynamicCast(OSData,
1019 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
1020 {
1021 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
1022 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
1023
1024 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1025
1026 if (kIOHibernatePreviewUpdates & flags)
1027 {
1028 PE_Video consoleInfo;
1029 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1030
1031 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1032
1033 graphicsInfo->width = consoleInfo.v_width;
1034 graphicsInfo->height = consoleInfo.v_height;
1035 graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
1036 graphicsInfo->depth = consoleInfo.v_depth;
1037 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1038
1039 HIBPRINT("video %p %d %d %d\n",
1040 vars->consoleMapping, graphicsInfo->depth,
1041 graphicsInfo->width, graphicsInfo->height);
1042 if (vars->consoleMapping)
1043 ProgressInit(graphicsInfo, vars->consoleMapping,
1044 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1045 }
1046 }
1047
1048 if (gIOOptionsEntry)
1049 gIOOptionsEntry->sync();
1050
1051 return (ret);
1052 }
1053
1054 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1055
1056 static DeviceTreeNode *
1057 MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry)
1058 {
1059 DeviceTreeNodeProperty * prop;
1060 DeviceTreeNode * child;
1061 IORegistryEntry * childRegEntry;
1062 const char * nameProp;
1063 unsigned int propLen, idx;
1064
1065 prop = (DeviceTreeNodeProperty *) (entry + 1);
1066 for (idx = 0; idx < entry->nProperties; idx++)
1067 {
1068 if (regEntry && (0 != strcmp("name", prop->name)))
1069 {
1070 regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
1071 // HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
1072 }
1073 prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
1074 }
1075
1076 child = (DeviceTreeNode *) prop;
1077 for (idx = 0; idx < entry->nChildren; idx++)
1078 {
1079 if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen))
1080 panic("no name");
1081 childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
1082 // HIBPRINT("%s == %p\n", nameProp, childRegEntry);
1083 child = MergeDeviceTree(child, childRegEntry);
1084 }
1085 return (child);
1086 }
1087
1088 IOReturn
1089 IOHibernateSystemWake(void)
1090 {
1091 if (kFSOpened == gFSState)
1092 {
1093 IOPolledFilePollersClose(gIOHibernateVars.fileVars, kIOPolledPostflightState);
1094 IOHibernateDone(&gIOHibernateVars);
1095 }
1096 else
1097 {
1098 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1099 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1100 }
1101 return (kIOReturnSuccess);
1102 }
1103
1104 static IOReturn
1105 IOHibernateDone(IOHibernateVars * vars)
1106 {
1107 hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
1108
1109 if (vars->videoMapping)
1110 {
1111 if (vars->videoMapSize)
1112 // remove mappings
1113 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1114 if (vars->videoAllocSize)
1115 // dealloc range
1116 kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
1117 }
1118
1119 if (vars->previewBuffer)
1120 {
1121 vars->previewBuffer->release();
1122 vars->previewBuffer = 0;
1123 }
1124
1125 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1126 {
1127 IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
1128 gIOHibernateCurrentHeader->options, 32);
1129 }
1130 else
1131 {
1132 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1133 }
1134
1135 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1136 && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus))
1137 {
1138 IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
1139 &gIOHibernateGraphicsInfo->gfxStatus,
1140 sizeof(gIOHibernateGraphicsInfo->gfxStatus));
1141 }
1142 else
1143 {
1144 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1145 }
1146
1147 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1148
1149 #if defined(__i386__) || defined(__x86_64__)
1150 IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
1151 IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
1152
1153 /*
1154 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1155 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1156 */
1157 if (gIOOptionsEntry) {
1158
1159 if (gIOHibernateRTCVariablesKey) {
1160 if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
1161 gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
1162 }
1163 }
1164
1165 if (gIOHibernateBootNextKey)
1166 {
1167 if (gIOHibernateBootNextSave)
1168 {
1169 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
1170 gIOHibernateBootNextSave->release();
1171 gIOHibernateBootNextSave = NULL;
1172 }
1173 else
1174 gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
1175 }
1176 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) gIOOptionsEntry->sync();
1177 }
1178 #endif
1179
1180 if (vars->srcBuffer) vars->srcBuffer->release();
1181 bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
1182 if (vars->handoffBuffer)
1183 {
1184 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1185 {
1186 IOHibernateHandoff * handoff;
1187 bool done = false;
1188 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
1189 !done;
1190 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
1191 {
1192 HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
1193 uint8_t * data = &handoff->data[0];
1194 switch (handoff->type)
1195 {
1196 case kIOHibernateHandoffTypeEnd:
1197 done = true;
1198 break;
1199
1200 case kIOHibernateHandoffTypeDeviceTree:
1201 MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
1202 break;
1203
1204 case kIOHibernateHandoffTypeKeyStore:
1205 #if defined(__i386__) || defined(__x86_64__)
1206 {
1207 IOBufferMemoryDescriptor *
1208 md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
1209 if (md)
1210 {
1211 IOSetKeyStoreData(md);
1212 }
1213 }
1214 #endif
1215 break;
1216
1217 default:
1218 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
1219 break;
1220 }
1221 }
1222 }
1223 vars->handoffBuffer->release();
1224 }
1225
1226 bzero(vars, sizeof(*vars));
1227
1228 // gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1229
1230 return (kIOReturnSuccess);
1231 }
1232
1233 IOReturn
1234 IOHibernateSystemPostWake(void)
1235 {
1236 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1237 if (kFSOpened == gFSState)
1238 {
1239 // invalidate & close the image file
1240 IOSleep(TRIM_DELAY);
1241 IOPolledFileIOVars * vars = &gFileVars;
1242 IOPolledFileClose(&vars,
1243 #if DISABLE_TRIM
1244 0, NULL, 0, 0, 0);
1245 #else
1246 0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader),
1247 sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize);
1248 #endif
1249 }
1250 gFSState = kFSIdle;
1251
1252 IOSetBootImageNVRAM(0);
1253
1254 if (gDebugImageLock) {
1255 IOLockLock(gDebugImageLock);
1256 if (gDebugImageFileVars != 0) {
1257 kprintf("IOHibernateSystemPostWake: Closing debugdata file\n");
1258 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
1259 }
1260 IOLockUnlock(gDebugImageLock);
1261 }
1262
1263 return (kIOReturnSuccess);
1264 }
1265
1266 bool IOHibernateWasScreenLocked(void)
1267 {
1268 bool ret = false;
1269 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry)
1270 {
1271 OSData *
1272 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
1273 if (data) switch (*((uint32_t *)data->getBytesNoCopy()))
1274 {
1275 case kIOScreenLockLocked:
1276 case kIOScreenLockFileVaultDialog:
1277 ret = true;
1278 break;
1279 case kIOScreenLockNoLock:
1280 case kIOScreenLockUnlocked:
1281 default:
1282 ret = false;
1283 break;
1284 }
1285 }
1286 return (ret);
1287 }
1288
1289 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1290
1291 SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
1292 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1293 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
1294 SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
1295 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1296 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
1297 SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
1298 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1299 &gIOHibernateMode, 0, "");
1300 SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
1301 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1302 gIOHibernateStats, hibernate_statistics_t, "");
1303
1304 SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
1305 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1306 &gIOHibernateStats->graphicsReadyTime, 0, "");
1307 SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
1308 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1309 &gIOHibernateStats->wakeNotificationTime, 0, "");
1310 SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
1311 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1312 &gIOHibernateStats->lockScreenReadyTime, 0, "");
1313 SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
1314 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1315 &gIOHibernateStats->hidReadyTime, 0, "");
1316
1317
1318 void
1319 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1320 {
1321 gIOHibernateBootImageKey = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1322
1323 #if defined(__i386__) || defined(__x86_64__)
1324 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1325 gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
1326 gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
1327 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1328 #endif /* defined(__i386__) || defined(__x86_64__) */
1329
1330 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
1331 if (data)
1332 {
1333 rootDomain->setProperty(kIOHibernateStateKey, data);
1334 data->release();
1335 }
1336
1337 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))
1338 gIOHibernateMode = kIOHibernateModeOn;
1339 else
1340 gIOHibernateFilename[0] = 0;
1341
1342 sysctl_register_oid(&sysctl__kern_hibernatefile);
1343 sysctl_register_oid(&sysctl__kern_bootsignature);
1344 sysctl_register_oid(&sysctl__kern_hibernatemode);
1345 sysctl_register_oid(&sysctl__kern_hibernatestatistics);
1346 sysctl_register_oid(&sysctl__kern_hibernategraphicsready);
1347 sysctl_register_oid(&sysctl__kern_hibernatewakenotification);
1348 sysctl_register_oid(&sysctl__kern_hibernatelockscreenready);
1349 sysctl_register_oid(&sysctl__kern_hibernatehidready);
1350
1351 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1352
1353 gFSLock = IOLockAlloc();
1354 }
1355
1356
1357 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1358
1359 static void
1360 hibernate_setup_for_wake(void)
1361 {
1362 }
1363
1364 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1365
1366 static IOReturn
1367 IOHibernatePolledFileWrite(IOPolledFileIOVars * vars,
1368 const uint8_t * bytes, IOByteCount size,
1369 IOPolledFileCryptVars * cryptvars)
1370 {
1371 IOReturn err;
1372
1373 err = IOPolledFileWrite(vars, bytes, size, cryptvars);
1374 if ((kIOReturnSuccess == err) && hibernate_should_abort()) err = kIOReturnAborted;
1375
1376 return (err);
1377 }
1378
1379 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1380
1381 extern "C" uint32_t
1382 hibernate_write_image(void)
1383 {
1384 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
1385 IOHibernateVars * vars = &gIOHibernateVars;
1386 IOPolledFileExtent * fileExtents;
1387
1388 assert_static(sizeof(IOHibernateImageHeader) == 512);
1389
1390 uint32_t pageCount, pagesDone;
1391 IOReturn err;
1392 vm_offset_t ppnum, page;
1393 IOItemCount count;
1394 uint8_t * src;
1395 uint8_t * data;
1396 uint8_t * compressed;
1397 uint8_t * scratch;
1398 IOByteCount pageCompressedSize;
1399 uint64_t compressedSize, uncompressedSize;
1400 uint64_t image1Size = 0;
1401 uint32_t bitmap_size;
1402 bool iterDone, pollerOpen, needEncrypt;
1403 uint32_t restore1Sum, sum, sum1, sum2;
1404 int wkresult;
1405 uint32_t tag;
1406 uint32_t pageType;
1407 uint32_t pageAndCount[2];
1408 addr64_t phys64;
1409 IOByteCount segLen;
1410 uintptr_t hibernateBase;
1411 uintptr_t hibernateEnd;
1412
1413 AbsoluteTime startTime, endTime;
1414 AbsoluteTime allTime, compTime;
1415 uint64_t compBytes;
1416 uint64_t nsec;
1417 uint32_t lastProgressStamp = 0;
1418 uint32_t progressStamp;
1419 uint32_t blob, lastBlob = (uint32_t) -1L;
1420
1421 uint32_t wiredPagesEncrypted;
1422 uint32_t dirtyPagesEncrypted;
1423 uint32_t wiredPagesClear;
1424 uint32_t svPageCount;
1425 uint32_t zvPageCount;
1426
1427 IOPolledFileCryptVars _cryptvars;
1428 IOPolledFileCryptVars * cryptvars = 0;
1429
1430 wiredPagesEncrypted = 0;
1431 dirtyPagesEncrypted = 0;
1432 wiredPagesClear = 0;
1433 svPageCount = 0;
1434 zvPageCount = 0;
1435
1436 if (!vars->fileVars || !vars->fileVars->pollers)
1437 return (false /* sleep */ );
1438
1439 if (kIOHibernateModeSleep & gIOHibernateMode)
1440 kdebug_enable = save_kdebug_enable;
1441
1442 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START, 0, 0, 0, 0, 0);
1443 IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
1444
1445 restore1Sum = sum1 = sum2 = 0;
1446
1447 #if CRYPTO
1448 // encryption data. "iv" is the "initial vector".
1449 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1450 {
1451 static const unsigned char first_iv[AES_BLOCK_SIZE]
1452 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
1453 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
1454
1455 cryptvars = &gIOHibernateCryptWakeContext;
1456 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1457 aes_encrypt_key(vars->cryptKey,
1458 kIOHibernateAESKeySize,
1459 &cryptvars->ctx.encrypt);
1460 aes_decrypt_key(vars->cryptKey,
1461 kIOHibernateAESKeySize,
1462 &cryptvars->ctx.decrypt);
1463
1464 cryptvars = &_cryptvars;
1465 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1466 for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++)
1467 vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
1468 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
1469 aes_encrypt_key(vars->wiredCryptKey,
1470 kIOHibernateAESKeySize,
1471 &cryptvars->ctx.encrypt);
1472
1473 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1474 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1475 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
1476 }
1477 #endif /* CRYPTO */
1478
1479 hibernate_setup_for_wake();
1480
1481 hibernate_page_list_setall(vars->page_list,
1482 vars->page_list_wired,
1483 vars->page_list_pal,
1484 false /* !preflight */,
1485 /* discard_all */
1486 ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
1487 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
1488 &pageCount);
1489
1490 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
1491
1492 fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy();
1493
1494 #if 0
1495 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
1496 for (page = 0; page < count; page++)
1497 {
1498 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
1499 fileExtents[page].start, fileExtents[page].length,
1500 fileExtents[page].start + fileExtents[page].length);
1501 }
1502 #endif
1503
1504 needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
1505 AbsoluteTime_to_scalar(&compTime) = 0;
1506 compBytes = 0;
1507
1508 clock_get_uptime(&allTime);
1509 IOService::getPMRootDomain()->pmStatsRecordEvent(
1510 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
1511 do
1512 {
1513 compressedSize = 0;
1514 uncompressedSize = 0;
1515 svPageCount = 0;
1516 zvPageCount = 0;
1517
1518 IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
1519
1520 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
1521 ml_get_interrupts_enabled());
1522 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState, true);
1523 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
1524 pollerOpen = (kIOReturnSuccess == err);
1525 if (!pollerOpen)
1526 break;
1527
1528 // copy file block extent list if larger than header
1529
1530 count = vars->fileVars->fileExtents->getLength();
1531 if (count > sizeof(header->fileExtentMap))
1532 {
1533 count -= sizeof(header->fileExtentMap);
1534 err = IOHibernatePolledFileWrite(vars->fileVars,
1535 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
1536 if (kIOReturnSuccess != err)
1537 break;
1538 }
1539
1540 hibernateBase = HIB_BASE; /* Defined in PAL headers */
1541 hibernateEnd = (segHIBB + segSizeHIB);
1542
1543 // copy out restore1 code
1544
1545 for (count = 0;
1546 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1547 count += segLen)
1548 {
1549 for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++)
1550 {
1551 gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
1552 }
1553 }
1554
1555 page = atop_32(kvtophys(hibernateBase));
1556 count = atop_32(round_page(hibernateEnd) - hibernateBase);
1557 header->restore1CodePhysPage = page;
1558 header->restore1CodeVirt = hibernateBase;
1559 header->restore1PageCount = count;
1560 header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
1561 header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
1562
1563 // sum __HIB seg, with zeros for the stack
1564 src = (uint8_t *) trunc_page(hibernateBase);
1565 for (page = 0; page < count; page++)
1566 {
1567 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
1568 restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
1569 else
1570 restore1Sum += 0x00000000;
1571 src += page_size;
1572 }
1573 sum1 = restore1Sum;
1574
1575 // write the __HIB seg, with zeros for the stack
1576
1577 src = (uint8_t *) trunc_page(hibernateBase);
1578 count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
1579 if (count)
1580 {
1581 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
1582 if (kIOReturnSuccess != err)
1583 break;
1584 }
1585 err = IOHibernatePolledFileWrite(vars->fileVars,
1586 (uint8_t *) 0,
1587 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
1588 cryptvars);
1589 if (kIOReturnSuccess != err)
1590 break;
1591 src = &gIOHibernateRestoreStackEnd[0];
1592 count = round_page(hibernateEnd) - ((uintptr_t) src);
1593 if (count)
1594 {
1595 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
1596 if (kIOReturnSuccess != err)
1597 break;
1598 }
1599
1600 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1601 {
1602 vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
1603 vars->fileVars->encryptEnd = UINT64_MAX;
1604 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1605 }
1606
1607 // write the preview buffer
1608
1609 if (vars->previewBuffer)
1610 {
1611 ppnum = 0;
1612 count = 0;
1613 do
1614 {
1615 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
1616 pageAndCount[0] = atop_64(phys64);
1617 pageAndCount[1] = atop_32(segLen);
1618 err = IOHibernatePolledFileWrite(vars->fileVars,
1619 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1620 cryptvars);
1621 if (kIOReturnSuccess != err)
1622 break;
1623 count += segLen;
1624 ppnum += sizeof(pageAndCount);
1625 }
1626 while (phys64);
1627 if (kIOReturnSuccess != err)
1628 break;
1629
1630 src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
1631
1632 ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
1633
1634 count = vars->previewBuffer->getLength();
1635
1636 header->previewPageListSize = ppnum;
1637 header->previewSize = count + ppnum;
1638
1639 for (page = 0; page < count; page += page_size)
1640 {
1641 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
1642 sum1 += hibernate_sum_page(src + page, atop_64(phys64));
1643 }
1644 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
1645 if (kIOReturnSuccess != err)
1646 break;
1647 }
1648
1649 // mark areas for no save
1650 IOMemoryDescriptor * ioBuffer;
1651 ioBuffer = IOPolledFileGetIOBuffer(vars->fileVars);
1652 for (count = 0;
1653 (phys64 = ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1654 count += segLen)
1655 {
1656 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1657 atop_64(phys64), atop_32(segLen),
1658 kIOHibernatePageStateFree);
1659 pageCount -= atop_32(segLen);
1660 }
1661
1662 for (count = 0;
1663 (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1664 count += segLen)
1665 {
1666 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1667 atop_64(phys64), atop_32(segLen),
1668 kIOHibernatePageStateFree);
1669 pageCount -= atop_32(segLen);
1670 }
1671
1672 // copy out bitmap of pages available for trashing during restore
1673
1674 bitmap_size = vars->page_list_wired->list_size;
1675 src = (uint8_t *) vars->page_list_wired;
1676 err = IOHibernatePolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
1677 if (kIOReturnSuccess != err)
1678 break;
1679
1680 // mark more areas for no save, but these are not available
1681 // for trashing during restore
1682
1683 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
1684
1685
1686 page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
1687 count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
1688 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1689 page, count,
1690 kIOHibernatePageStateFree);
1691 pageCount -= count;
1692
1693 if (vars->previewBuffer) for (count = 0;
1694 (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1695 count += segLen)
1696 {
1697 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1698 atop_64(phys64), atop_32(segLen),
1699 kIOHibernatePageStateFree);
1700 pageCount -= atop_32(segLen);
1701 }
1702
1703 for (count = 0;
1704 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1705 count += segLen)
1706 {
1707 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1708 atop_64(phys64), atop_32(segLen),
1709 kIOHibernatePageStateFree);
1710 pageCount -= atop_32(segLen);
1711 }
1712
1713 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
1714 compressed = src + page_size;
1715 scratch = compressed + page_size;
1716
1717 pagesDone = 0;
1718 lastBlob = 0;
1719
1720 HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
1721 bitmap_size, header->previewSize,
1722 pageCount, vars->fileVars->position);
1723
1724 enum
1725 // pageType
1726 {
1727 kWired = 0x02,
1728 kEncrypt = 0x01,
1729 kWiredEncrypt = kWired | kEncrypt,
1730 kWiredClear = kWired,
1731 kUnwiredEncrypt = kEncrypt
1732 };
1733
1734 for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)
1735 {
1736 if (kUnwiredEncrypt == pageType)
1737 {
1738 // start unwired image
1739 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1740 {
1741 vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
1742 vars->fileVars->encryptEnd = UINT64_MAX;
1743 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1744 }
1745 bcopy(&cryptvars->aes_iv[0],
1746 &gIOHibernateCryptWakeContext.aes_iv[0],
1747 sizeof(cryptvars->aes_iv));
1748 cryptvars = &gIOHibernateCryptWakeContext;
1749 }
1750 for (iterDone = false, ppnum = 0; !iterDone; )
1751 {
1752 count = hibernate_page_list_iterate((kWired & pageType)
1753 ? vars->page_list_wired : vars->page_list,
1754 &ppnum);
1755 // kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
1756 iterDone = !count;
1757
1758 if (count && (kWired & pageType) && needEncrypt)
1759 {
1760 uint32_t checkIndex;
1761 for (checkIndex = 0;
1762 (checkIndex < count)
1763 && (((kEncrypt & pageType) == 0) == pmap_is_noencrypt(ppnum + checkIndex));
1764 checkIndex++)
1765 {}
1766 if (!checkIndex)
1767 {
1768 ppnum++;
1769 continue;
1770 }
1771 count = checkIndex;
1772 }
1773
1774 switch (pageType)
1775 {
1776 case kWiredEncrypt: wiredPagesEncrypted += count; break;
1777 case kWiredClear: wiredPagesClear += count; break;
1778 case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
1779 }
1780
1781 if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */}
1782 else
1783 {
1784 pageAndCount[0] = ppnum;
1785 pageAndCount[1] = count;
1786 err = IOHibernatePolledFileWrite(vars->fileVars,
1787 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1788 cryptvars);
1789 if (kIOReturnSuccess != err)
1790 break;
1791 }
1792
1793 for (page = ppnum; page < (ppnum + count); page++)
1794 {
1795 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
1796 if (err)
1797 {
1798 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
1799 break;
1800 }
1801
1802 sum = hibernate_sum_page(src, page);
1803 if (kWired & pageType)
1804 sum1 += sum;
1805 else
1806 sum2 += sum;
1807
1808 clock_get_uptime(&startTime);
1809 wkresult = WKdm_compress_new((const WK_word*) src,
1810 (WK_word*) compressed,
1811 (WK_word*) scratch,
1812 page_size - 4);
1813
1814 clock_get_uptime(&endTime);
1815 ADD_ABSOLUTETIME(&compTime, &endTime);
1816 SUB_ABSOLUTETIME(&compTime, &startTime);
1817
1818 compBytes += page_size;
1819 pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
1820
1821 if (pageCompressedSize == 0)
1822 {
1823 pageCompressedSize = 4;
1824 data = src;
1825
1826 if (*(uint32_t *)src)
1827 svPageCount++;
1828 else
1829 zvPageCount++;
1830 }
1831 else
1832 {
1833 if (pageCompressedSize != page_size)
1834 data = compressed;
1835 else
1836 data = src;
1837 }
1838
1839 tag = pageCompressedSize | kIOHibernateTagSignature;
1840 err = IOHibernatePolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
1841 if (kIOReturnSuccess != err)
1842 break;
1843
1844 err = IOHibernatePolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
1845 if (kIOReturnSuccess != err)
1846 break;
1847
1848 compressedSize += pageCompressedSize;
1849 uncompressedSize += page_size;
1850 pagesDone++;
1851
1852 if (vars->consoleMapping && (0 == (1023 & pagesDone)))
1853 {
1854 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
1855 if (blob != lastBlob)
1856 {
1857 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
1858 lastBlob = blob;
1859 }
1860 }
1861 if (0 == (8191 & pagesDone))
1862 {
1863 clock_get_uptime(&endTime);
1864 SUB_ABSOLUTETIME(&endTime, &allTime);
1865 absolutetime_to_nanoseconds(endTime, &nsec);
1866 progressStamp = nsec / 750000000ULL;
1867 if (progressStamp != lastProgressStamp)
1868 {
1869 lastProgressStamp = progressStamp;
1870 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
1871 }
1872 }
1873 }
1874 if (kIOReturnSuccess != err)
1875 break;
1876 ppnum = page;
1877 }
1878
1879 if (kIOReturnSuccess != err)
1880 break;
1881
1882 if ((kEncrypt & pageType) && vars->fileVars->encryptStart)
1883 {
1884 vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
1885 HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
1886 }
1887
1888 if (kWiredEncrypt != pageType)
1889 {
1890 // end of image1/2 - fill to next block
1891 err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
1892 if (kIOReturnSuccess != err)
1893 break;
1894 }
1895 if (kWiredClear == pageType)
1896 {
1897 // enlarge wired image for test
1898 // err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
1899
1900 // end wired image
1901 header->encryptStart = vars->fileVars->encryptStart;
1902 header->encryptEnd = vars->fileVars->encryptEnd;
1903 image1Size = vars->fileVars->position;
1904 HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
1905 image1Size, header->encryptStart, header->encryptEnd);
1906 }
1907 }
1908 if (kIOReturnSuccess != err)
1909 {
1910 if (kIOReturnOverrun == err)
1911 {
1912 // update actual compression ratio on not enough space (for retry)
1913 gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
1914 }
1915
1916 // update partial amount written (for IOPolledFileClose cleanup/unmap)
1917 header->imageSize = vars->fileVars->position;
1918 break;
1919 }
1920
1921 // Header:
1922
1923 header->imageSize = vars->fileVars->position;
1924 header->image1Size = image1Size;
1925 header->bitmapSize = bitmap_size;
1926 header->pageCount = pageCount;
1927
1928 header->restore1Sum = restore1Sum;
1929 header->image1Sum = sum1;
1930 header->image2Sum = sum2;
1931 header->sleepTime = gIOLastSleepTime.tv_sec;
1932
1933 header->compression = (compressedSize << 8) / uncompressedSize;
1934 gIOHibernateCompression = header->compression;
1935
1936 count = vars->fileVars->fileExtents->getLength();
1937 if (count > sizeof(header->fileExtentMap))
1938 {
1939 header->fileExtentMapSize = count;
1940 count = sizeof(header->fileExtentMap);
1941 }
1942 else
1943 header->fileExtentMapSize = sizeof(header->fileExtentMap);
1944 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
1945
1946 header->deviceBase = vars->fileVars->block0;
1947 header->deviceBlockSize = vars->fileVars->blockSize;
1948
1949 IOPolledFileSeek(vars->fileVars, 0);
1950 err = IOHibernatePolledFileWrite(vars->fileVars,
1951 (uint8_t *) header, sizeof(IOHibernateImageHeader),
1952 cryptvars);
1953 if (kIOReturnSuccess != err)
1954 break;
1955 err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
1956 }
1957 while (false);
1958
1959 clock_get_uptime(&endTime);
1960
1961 IOService::getPMRootDomain()->pmStatsRecordEvent(
1962 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
1963
1964 SUB_ABSOLUTETIME(&endTime, &allTime);
1965 absolutetime_to_nanoseconds(endTime, &nsec);
1966 HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
1967
1968 absolutetime_to_nanoseconds(compTime, &nsec);
1969 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
1970 compBytes,
1971 nsec / 1000000ULL,
1972 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
1973
1974 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
1975 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
1976 vars->fileVars->cryptBytes,
1977 nsec / 1000000ULL,
1978 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
1979
1980 HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
1981 header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
1982 uncompressedSize, atop_32(uncompressedSize), compressedSize,
1983 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
1984 sum1, sum2);
1985
1986 HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
1987 svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
1988
1989 if (pollerOpen)
1990 IOPolledFilePollersClose(vars->fileVars, kIOPolledBeforeSleepState);
1991
1992 if (vars->consoleMapping)
1993 ProgressUpdate(gIOHibernateGraphicsInfo,
1994 vars->consoleMapping, 0, kIOHibernateProgressCount);
1995
1996 HIBLOG("hibernate_write_image done(%x)\n", err);
1997
1998 // should we come back via regular wake, set the state in memory.
1999 gIOHibernateState = kIOHibernateStateInactive;
2000
2001 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END,
2002 wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted, 0, 0);
2003
2004 if (kIOReturnSuccess == err)
2005 {
2006 if (kIOHibernateModeSleep & gIOHibernateMode)
2007 {
2008 return (kIOHibernatePostWriteSleep);
2009 }
2010 else if(kIOHibernateModeRestart & gIOHibernateMode)
2011 {
2012 return (kIOHibernatePostWriteRestart);
2013 }
2014 else
2015 {
2016 /* by default, power down */
2017 return (kIOHibernatePostWriteHalt);
2018 }
2019 }
2020 else if (kIOReturnAborted == err)
2021 {
2022 return (kIOHibernatePostWriteWake);
2023 }
2024 else
2025 {
2026 /* on error, sleep */
2027 return (kIOHibernatePostWriteSleep);
2028 }
2029 }
2030
2031 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2032
2033 extern "C" void
2034 hibernate_machine_init(void)
2035 {
2036 IOReturn err;
2037 uint32_t sum;
2038 uint32_t pagesDone;
2039 uint32_t pagesRead = 0;
2040 AbsoluteTime startTime, compTime;
2041 AbsoluteTime allTime, endTime;
2042 AbsoluteTime startIOTime, endIOTime;
2043 uint64_t nsec, nsecIO;
2044 uint64_t compBytes;
2045 uint32_t lastProgressStamp = 0;
2046 uint32_t progressStamp;
2047 IOPolledFileCryptVars * cryptvars = 0;
2048
2049 IOHibernateVars * vars = &gIOHibernateVars;
2050 bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
2051
2052 if (!vars->fileVars || !vars->fileVars->pollers)
2053 return;
2054
2055 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2056 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2057
2058 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
2059 {
2060 HIBLOG("regular wake\n");
2061 return;
2062 }
2063
2064 HIBPRINT("diag %x %x %x %x\n",
2065 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
2066 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
2067
2068 #define t40ms(x) (tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)
2069 #define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
2070 tStat(booterStart, booterStart);
2071 gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart,
2072 tStat(booterDuration0, booterTime0);
2073 tStat(booterDuration1, booterTime1);
2074 tStat(booterDuration2, booterTime2);
2075 tStat(booterDuration, booterTime);
2076 tStat(booterConnectDisplayDuration, connectDisplayTime);
2077 tStat(booterSplashDuration, splashTime);
2078 tStat(trampolineDuration, trampolineTime);
2079
2080 gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size;
2081 gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize;
2082 gIOHibernateStats->image1Pages = pagesDone;
2083
2084 HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
2085 gIOHibernateStats->booterStart,
2086 gIOHibernateStats->smcStart,
2087 gIOHibernateStats->booterDuration0,
2088 gIOHibernateStats->booterDuration1,
2089 gIOHibernateStats->booterDuration2,
2090 gIOHibernateStats->booterDuration,
2091 gIOHibernateStats->booterConnectDisplayDuration,
2092 gIOHibernateStats->booterSplashDuration,
2093 gIOHibernateStats->trampolineDuration);
2094
2095 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
2096 gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
2097 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
2098
2099 if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
2100 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)))
2101 {
2102 hibernate_page_list_discard(vars->page_list);
2103 }
2104
2105 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
2106
2107 if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount)
2108 panic("handoff overflow");
2109
2110 IOHibernateHandoff * handoff;
2111 bool done = false;
2112 bool foundCryptData = false;
2113
2114 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
2115 !done;
2116 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
2117 {
2118 // HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
2119 uint8_t * data = &handoff->data[0];
2120 switch (handoff->type)
2121 {
2122 case kIOHibernateHandoffTypeEnd:
2123 done = true;
2124 break;
2125
2126 case kIOHibernateHandoffTypeGraphicsInfo:
2127 bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
2128 break;
2129
2130 case kIOHibernateHandoffTypeCryptVars:
2131 if (cryptvars)
2132 {
2133 hibernate_cryptwakevars_t *
2134 wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
2135 bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
2136 }
2137 foundCryptData = true;
2138 bzero(data, handoff->bytecount);
2139 break;
2140
2141 case kIOHibernateHandoffTypeMemoryMap:
2142
2143 clock_get_uptime(&allTime);
2144
2145 hibernate_newruntime_map(data, handoff->bytecount,
2146 gIOHibernateCurrentHeader->systemTableOffset);
2147
2148 clock_get_uptime(&endTime);
2149
2150 SUB_ABSOLUTETIME(&endTime, &allTime);
2151 absolutetime_to_nanoseconds(endTime, &nsec);
2152
2153 HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
2154
2155 break;
2156
2157 case kIOHibernateHandoffTypeDeviceTree:
2158 {
2159 // DTEntry chosen = NULL;
2160 // HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen));
2161 }
2162 break;
2163
2164 default:
2165 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
2166 break;
2167 }
2168 }
2169 if (cryptvars && !foundCryptData)
2170 panic("hibernate handoff");
2171
2172 HIBPRINT("video %x %d %d %d status %x\n",
2173 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2174 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
2175
2176 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)
2177 {
2178 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2179 * gIOHibernateGraphicsInfo->rowBytes);
2180 if (vars->videoMapSize > vars->videoAllocSize) vars->videoMapSize = 0;
2181 else
2182 {
2183 IOMapPages(kernel_map,
2184 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2185 vars->videoMapSize, kIOMapInhibitCache );
2186 }
2187 }
2188
2189 if (vars->videoMapSize)
2190 ProgressUpdate(gIOHibernateGraphicsInfo,
2191 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2192
2193 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2194 uint8_t * compressed = src + page_size;
2195 uint8_t * scratch = compressed + page_size;
2196 uint32_t decoOffset;
2197
2198 clock_get_uptime(&allTime);
2199 AbsoluteTime_to_scalar(&compTime) = 0;
2200 compBytes = 0;
2201
2202 HIBLOG("IOPolledFilePollersOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2203 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledAfterSleepState, false);
2204 clock_get_uptime(&startIOTime);
2205 endTime = startIOTime;
2206 SUB_ABSOLUTETIME(&endTime, &allTime);
2207 absolutetime_to_nanoseconds(endTime, &nsec);
2208 HIBLOG("IOPolledFilePollersOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
2209
2210 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2211
2212 // kick off the read ahead
2213 vars->fileVars->bufferHalf = 0;
2214 vars->fileVars->bufferLimit = 0;
2215 vars->fileVars->lastRead = 0;
2216 vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
2217 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2218 vars->fileVars->cryptBytes = 0;
2219 AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
2220
2221 err = IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
2222 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2223 // --
2224
2225 HIBLOG("hibernate_machine_init reading\n");
2226
2227 uint32_t * header = (uint32_t *) src;
2228 sum = 0;
2229
2230 while (kIOReturnSuccess == err)
2231 {
2232 unsigned int count;
2233 unsigned int page;
2234 uint32_t tag;
2235 vm_offset_t ppnum, compressedSize;
2236
2237 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2238 if (kIOReturnSuccess != err)
2239 break;
2240
2241 ppnum = header[0];
2242 count = header[1];
2243
2244 // HIBPRINT("(%x, %x)\n", ppnum, count);
2245
2246 if (!count)
2247 break;
2248
2249 for (page = 0; page < count; page++)
2250 {
2251 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2252 if (kIOReturnSuccess != err)
2253 break;
2254
2255 compressedSize = kIOHibernateTagLength & tag;
2256 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength))
2257 {
2258 err = kIOReturnIPCError;
2259 break;
2260 }
2261
2262 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2263 if (kIOReturnSuccess != err) break;
2264
2265 if (compressedSize < page_size)
2266 {
2267 decoOffset = page_size;
2268 clock_get_uptime(&startTime);
2269
2270 if (compressedSize == 4) {
2271 int i;
2272 uint32_t *s, *d;
2273
2274 s = (uint32_t *)src;
2275 d = (uint32_t *)(uintptr_t)compressed;
2276
2277 for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++)
2278 *d++ = *s;
2279 }
2280 else
2281 WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, compressedSize);
2282 clock_get_uptime(&endTime);
2283 ADD_ABSOLUTETIME(&compTime, &endTime);
2284 SUB_ABSOLUTETIME(&compTime, &startTime);
2285 compBytes += page_size;
2286 }
2287 else decoOffset = 0;
2288
2289 sum += hibernate_sum_page((src + decoOffset), ppnum);
2290 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2291 if (err)
2292 {
2293 HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
2294 break;
2295 }
2296
2297 ppnum++;
2298 pagesDone++;
2299 pagesRead++;
2300
2301 if (0 == (8191 & pagesDone))
2302 {
2303 clock_get_uptime(&endTime);
2304 SUB_ABSOLUTETIME(&endTime, &allTime);
2305 absolutetime_to_nanoseconds(endTime, &nsec);
2306 progressStamp = nsec / 750000000ULL;
2307 if (progressStamp != lastProgressStamp)
2308 {
2309 lastProgressStamp = progressStamp;
2310 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2311 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2312 }
2313 }
2314 }
2315 }
2316 if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages))
2317 err = kIOReturnLockedRead;
2318
2319 if (kIOReturnSuccess != err)
2320 panic("Hibernate restore error %x", err);
2321
2322 gIOHibernateCurrentHeader->actualImage2Sum = sum;
2323 gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
2324
2325 clock_get_uptime(&endIOTime);
2326
2327 err = IOPolledFilePollersClose(vars->fileVars, kIOPolledAfterSleepState);
2328
2329 clock_get_uptime(&endTime);
2330
2331 IOService::getPMRootDomain()->pmStatsRecordEvent(
2332 kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
2333 IOService::getPMRootDomain()->pmStatsRecordEvent(
2334 kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
2335
2336 SUB_ABSOLUTETIME(&endTime, &allTime);
2337 absolutetime_to_nanoseconds(endTime, &nsec);
2338
2339 SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
2340 absolutetime_to_nanoseconds(endIOTime, &nsecIO);
2341
2342 gIOHibernateStats->kernelImageReadDuration = nsec / 1000000ULL;
2343 gIOHibernateStats->imagePages = pagesDone;
2344
2345 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
2346 pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
2347 nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
2348
2349 absolutetime_to_nanoseconds(compTime, &nsec);
2350 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2351 compBytes,
2352 nsec / 1000000ULL,
2353 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2354
2355 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2356 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
2357 vars->fileVars->cryptBytes,
2358 nsec / 1000000ULL,
2359 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2360
2361 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_NONE, pagesRead, pagesDone, 0, 0, 0);
2362 }
2363
2364 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2365
2366 void IOHibernateSetWakeCapabilities(uint32_t capability)
2367 {
2368 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
2369 {
2370 gIOHibernateStats->wakeCapability = capability;
2371
2372 if (kIOPMSystemCapabilityGraphics & capability)
2373 {
2374 vm_compressor_do_warmup();
2375 }
2376 }
2377 }
2378
2379 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2380
2381 void IOHibernateSystemRestart(void)
2382 {
2383 static uint8_t noteStore[32] __attribute__((aligned(32)));
2384 IORegistryEntry * regEntry;
2385 const OSSymbol * sym;
2386 OSData * noteProp;
2387 OSData * data;
2388 uintptr_t * smcVars;
2389 uint8_t * smcBytes;
2390 size_t len;
2391 addr64_t element;
2392
2393 data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
2394 if (!data) return;
2395
2396 smcVars = (typeof(smcVars)) data->getBytesNoCopy();
2397 smcBytes = (typeof(smcBytes)) smcVars[1];
2398 len = smcVars[0];
2399 if (len > sizeof(noteStore)) len = sizeof(noteStore);
2400 noteProp = OSData::withCapacity(3 * sizeof(element));
2401 if (!noteProp) return;
2402 element = len;
2403 noteProp->appendBytes(&element, sizeof(element));
2404 element = crc32(0, smcBytes, len);
2405 noteProp->appendBytes(&element, sizeof(element));
2406
2407 bcopy(smcBytes, noteStore, len);
2408 element = (addr64_t) &noteStore[0];
2409 element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
2410 noteProp->appendBytes(&element, sizeof(element));
2411
2412 if (!gIOOptionsEntry)
2413 {
2414 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
2415 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
2416 if (regEntry && !gIOOptionsEntry)
2417 regEntry->release();
2418 }
2419
2420 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
2421 if (gIOOptionsEntry && sym) gIOOptionsEntry->setProperty(sym, noteProp);
2422 if (noteProp) noteProp->release();
2423 if (sym) sym->release();
2424 }
2425
2426
2427
mirror of XNU