]> git.saurik.com Git - apple/xnu.git/blob - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-4570.1.46.tar.gz
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
1 /*
2 * Copyright (c) 2004-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30
31 Sleep:
32
33 - PMRootDomain calls IOHibernateSystemSleep() before system sleep
34 (devices awake, normal execution context)
35 - IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
36 grabs its extents and searches for a polling driver willing to work with that IOMedia.
37 The BSD code makes an ioctl to the storage driver to get the partition base offset to
38 the disk, and other ioctls to get the transfer constraints
39 If successful, the file is written to make sure its initially not bootable (in case of
40 later failure) and nvram set to point to the first block of the file. (Has to be done
41 here so blocking is possible in nvram support).
42 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
43 page out any pages it wants to (currently zero, but probably some percentage of memory).
44 Its assumed just allocating pages will cause the VM system to naturally select the best
45 pages for eviction. It also copies processor flags needed for the restore path and sets
46 a flag in the boot processor proc info.
47 gIOHibernateState = kIOHibernateStateHibernating.
48 - Regular sleep progresses - some drivers may inspect the root domain property
49 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
50 as usual but leaves motherboard I/O on.
51 - Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
52 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
53 all ppc RC bits out of the hash table and caches into the mapping structures.
54 - hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
55 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
56 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
57 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
58 The image header and block list are written. The header includes the second file extent so
59 only the header block is needed to read the file, regardless of filesystem.
60 The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
61 (only) is used to decompress the image during wake/boot.
62 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
63 The bitmaps are written to the image.
64 More areas are removed from the bitmaps (after they have been written to the image) - the
65 segment "__HIB" pages and interrupt stack.
66 Each wired page is compressed and written and then each non-wired page. Compression and
67 disk writes are in parallel.
68 The image header is written to the start of the file and the polling driver closed.
69 The machine powers down (or sleeps).
70
71 Boot/Wake:
72
73 - BootX sees the boot-image nvram variable containing the device and block number of the image,
74 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
75 - BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
76 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
77 that is in the kernel's __HIB section.
78 - BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
79 hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
80 only code & data in that section is safe to call since all the other wired pages are still
81 compressed in the image.
82 - hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
83 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
84 location directly, and copies those that can't to interim free pages. When the image has been
85 completed, the copies are uncompressed, overwriting the wired image pages.
86 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
87 is used to get pages into place for 64bit.
88 - the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
89 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
90 are removed from the software strutures, and the hash table is reinitialized.
91 - After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
92 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
93 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
94 for the remaining non wired pages.
95 - The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
96 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
97 is closed via bsd.
98
99 Polled Mode I/O:
100
101 IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
102 registry, specifying an object of calls IOPolledInterface.
103
104 Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
105 partition) that the image is going to live, looking for polled interface properties. If it finds
106 one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
107 interfaces found are kept in an ordered list.
108
109 There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
110 few different contexts things happen in:
111
112 - there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
113 up and running) and after wake - this is safe to allocate memory and do anything. The device
114 ignores sleep requests from that point since its a waste of time if it goes to sleep and
115 immediately wakes back up for the image write.
116
117 - there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
118 immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
119 from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
120 used to flush and set the disk to sleep.
121
122 - there is an Open/Close (AfterSleep) pair made around the image read operations that happen
123 immediately after sleep. These can't block or allocate memory. This is happening after the platform
124 expert has woken the low level bits of the system, but most of the I/O system has not. There is only
125 one thread running.
126
127 For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
128 (prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
129 the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
130 Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
131 that is called for the hardware to check for events, and complete the I/O via the callback.
132 The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
133 to restrict I/O ops.
134 */
135
136 #include <sys/systm.h>
137
138 #include <IOKit/IOWorkLoop.h>
139 #include <IOKit/IOCommandGate.h>
140 #include <IOKit/IOTimerEventSource.h>
141 #include <IOKit/IOPlatformExpert.h>
142 #include <IOKit/IOKitDebug.h>
143 #include <IOKit/IOTimeStamp.h>
144 #include <IOKit/pwr_mgt/RootDomain.h>
145 #include <IOKit/pwr_mgt/IOPMPrivate.h>
146 #include <IOKit/IOMessage.h>
147 #include <IOKit/IODeviceTreeSupport.h>
148 #include <IOKit/IOBSD.h>
149 #include "RootDomainUserClient.h"
150 #include <IOKit/pwr_mgt/IOPowerConnection.h>
151 #include "IOPMPowerStateQueue.h"
152 #include <IOKit/IOBufferMemoryDescriptor.h>
153 #include <IOKit/AppleKeyStoreInterface.h>
154 #include <libkern/crypto/aes.h>
155
156 #include <sys/uio.h>
157 #include <sys/conf.h>
158 #include <sys/stat.h>
159 #include <sys/fcntl.h> // (FWRITE, ...)
160 #include <sys/sysctl.h>
161 #include <sys/kdebug.h>
162 #include <stdint.h>
163
164 #include <IOKit/IOHibernatePrivate.h>
165 #include <IOKit/IOPolledInterface.h>
166 #include <IOKit/IONVRAM.h>
167 #include "IOHibernateInternal.h"
168 #include <vm/WKdm_new.h>
169 #include <vm/vm_protos.h>
170 #include "IOKitKernelInternal.h"
171 #include <pexpert/device_tree.h>
172
173 #include <machine/pal_routines.h>
174 #include <machine/pal_hibernate.h>
175 #include <i386/tsc.h>
176 #include <i386/cpuid.h>
177 #include <san/kasan.h>
178
179 extern "C" addr64_t kvtophys(vm_offset_t va);
180 extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
181
182 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
183
184 #define DISABLE_TRIM 0
185 #define TRIM_DELAY 25000
186
187 extern unsigned int save_kdebug_enable;
188 extern uint32_t gIOHibernateState;
189 uint32_t gIOHibernateMode;
190 static char gIOHibernateBootSignature[256+1];
191 static char gIOHibernateFilename[MAXPATHLEN+1];
192 static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
193 uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
194 static uint64_t gIOHibernateCompression = 0x80; // default compression 50%
195 boolean_t gIOHibernateStandbyDisabled;
196
197 static IODTNVRAM * gIOOptionsEntry;
198 static IORegistryEntry * gIOChosenEntry;
199
200 static const OSSymbol * gIOHibernateBootImageKey;
201
202 #if defined(__i386__) || defined(__x86_64__)
203
204 static const OSSymbol * gIOHibernateRTCVariablesKey;
205 static const OSSymbol * gIOHibernateBoot0082Key;
206 static const OSSymbol * gIOHibernateBootNextKey;
207 static OSData * gIOHibernateBoot0082Data;
208 static OSData * gIOHibernateBootNextData;
209 static OSObject * gIOHibernateBootNextSave;
210
211 static IOPolledFileIOVars * gDebugImageFileVars;
212 static IOLock * gDebugImageLock;
213
214 #endif /* defined(__i386__) || defined(__x86_64__) */
215
216 static IOLock * gFSLock;
217 uint32_t gFSState;
218 static thread_call_t gIOHibernateTrimCalloutEntry;
219 static IOPolledFileIOVars gFileVars;
220 static IOHibernateVars gIOHibernateVars;
221 static IOPolledFileCryptVars gIOHibernateCryptWakeContext;
222 static hibernate_graphics_t _hibernateGraphics;
223 static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
224 static hibernate_statistics_t _hibernateStats;
225 static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats;
226
227 enum
228 {
229 kFSIdle = 0,
230 kFSOpening = 2,
231 kFSOpened = 3,
232 kFSTimedOut = 4,
233 kFSTrimDelay = 5
234 };
235
236 static IOReturn IOHibernateDone(IOHibernateVars * vars);
237 static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature);
238 static void IOSetBootImageNVRAM(OSData * data);
239 static void IOHibernateSystemPostWakeTrim(void * p1, void * p2);
240
241 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
242
243 enum { kDefaultIOSize = 128 * 1024 };
244 enum { kVideoMapSize = 80 * 1024 * 1024 };
245
246 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
247
248 // copy from phys addr to MD
249
250 static IOReturn
251 IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
252 IOByteCount offset, addr64_t bytes, IOByteCount length)
253 {
254 addr64_t srcAddr = bytes;
255 IOByteCount remaining;
256
257 remaining = length = min(length, md->getLength() - offset);
258 while (remaining) { // (process another target segment?)
259 addr64_t dstAddr64;
260 IOByteCount dstLen;
261
262 dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
263 if (!dstAddr64)
264 break;
265
266 // Clip segment length to remaining
267 if (dstLen > remaining)
268 dstLen = remaining;
269
270 #if 1
271 bcopy_phys(srcAddr, dstAddr64, dstLen);
272 #else
273 copypv(srcAddr, dstAddr64, dstLen,
274 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
275 #endif
276 srcAddr += dstLen;
277 offset += dstLen;
278 remaining -= dstLen;
279 }
280
281 assert(!remaining);
282
283 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
284 }
285
286 // copy from MD to phys addr
287
288 static IOReturn
289 IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
290 IOByteCount offset, addr64_t bytes, IOByteCount length)
291 {
292 addr64_t dstAddr = bytes;
293 IOByteCount remaining;
294
295 remaining = length = min(length, md->getLength() - offset);
296 while (remaining) { // (process another target segment?)
297 addr64_t srcAddr64;
298 IOByteCount dstLen;
299
300 srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
301 if (!srcAddr64)
302 break;
303
304 // Clip segment length to remaining
305 if (dstLen > remaining)
306 dstLen = remaining;
307
308 #if 1
309 bcopy_phys(srcAddr64, dstAddr, dstLen);
310 #else
311 copypv(srcAddr, dstAddr64, dstLen,
312 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
313 #endif
314 dstAddr += dstLen;
315 offset += dstLen;
316 remaining -= dstLen;
317 }
318
319 assert(!remaining);
320
321 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
322 }
323
324 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
325
326 void
327 hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
328 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
329 {
330 count += ppnum;
331 switch (kind)
332 {
333 case kIOHibernatePageStateUnwiredSave:
334 // unwired save
335 for (; ppnum < count; ppnum++)
336 {
337 hibernate_page_bitset(page_list, FALSE, ppnum);
338 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
339 }
340 break;
341 case kIOHibernatePageStateWiredSave:
342 // wired save
343 for (; ppnum < count; ppnum++)
344 {
345 hibernate_page_bitset(page_list, FALSE, ppnum);
346 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
347 }
348 break;
349 case kIOHibernatePageStateFree:
350 // free page
351 for (; ppnum < count; ppnum++)
352 {
353 hibernate_page_bitset(page_list, TRUE, ppnum);
354 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
355 }
356 break;
357 default:
358 panic("hibernate_set_page_state");
359 }
360 }
361
362 static vm_offset_t
363 hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
364 {
365 uint32_t page = *pPage;
366 uint32_t count;
367 hibernate_bitmap_t * bitmap;
368
369 while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
370 {
371 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
372 if (!count)
373 break;
374 page += count;
375 if (page <= bitmap->last_page)
376 break;
377 }
378
379 *pPage = page;
380 if (bitmap)
381 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
382 else
383 count = 0;
384
385 return (count);
386 }
387
388 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
389
390 IOReturn
391 IOHibernateSystemSleep(void)
392 {
393 IOReturn err;
394 OSData * nvramData;
395 OSObject * obj;
396 OSString * str;
397 OSNumber * num;
398 bool dsSSD, vmflush, swapPinned;
399 IOHibernateVars * vars;
400 uint64_t setFileSize = 0;
401
402 gIOHibernateState = kIOHibernateStateInactive;
403
404 gIOHibernateDebugFlags = 0;
405 if (kIOLogHibernate & gIOKitDebug)
406 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
407
408 if (IOService::getPMRootDomain()->getHibernateSettings(
409 &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime))
410 {
411 if (kIOHibernateModeSleep & gIOHibernateMode)
412 // default to discard clean for safe sleep
413 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
414 | kIOHibernateModeDiscardCleanActive);
415 }
416
417 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
418 {
419 if ((str = OSDynamicCast(OSString, obj)))
420 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
421 sizeof(gIOHibernateFilename));
422 obj->release();
423 }
424
425 if (!gIOHibernateMode || !gIOHibernateFilename[0])
426 return (kIOReturnUnsupported);
427
428 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
429
430 vars = IONew(IOHibernateVars, 1);
431 if (!vars) return (kIOReturnNoMemory);
432 bzero(vars, sizeof(*vars));
433
434 IOLockLock(gFSLock);
435 if (!gIOHibernateTrimCalloutEntry)
436 {
437 gIOHibernateTrimCalloutEntry = thread_call_allocate(&IOHibernateSystemPostWakeTrim, &gFSLock);
438 }
439 IOHibernateSystemPostWakeTrim(NULL, NULL);
440 if (kFSIdle != gFSState)
441 {
442 HIBLOG("hibernate file busy\n");
443 IOLockUnlock(gFSLock);
444 IODelete(vars, IOHibernateVars, 1);
445 return (kIOReturnBusy);
446 }
447 gFSState = kFSOpening;
448 IOLockUnlock(gFSLock);
449
450 swapPinned = false;
451 do
452 {
453 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
454 2 * page_size + WKdm_SCRATCH_BUF_SIZE_INTERNAL, page_size);
455
456 vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
457 ptoa_64(gIOHibernateHandoffPageCount), page_size);
458
459 if (!vars->srcBuffer || !vars->handoffBuffer)
460 {
461 err = kIOReturnNoMemory;
462 break;
463 }
464
465 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey)))
466 {
467 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMinSize = num->unsigned64BitValue();
468 obj->release();
469 }
470 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey)))
471 {
472 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMaxSize = num->unsigned64BitValue();
473 obj->release();
474 }
475
476 boolean_t encryptedswap = true;
477 uint32_t pageCount;
478 AbsoluteTime startTime, endTime;
479 uint64_t nsec;
480
481 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
482 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
483 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
484
485 vmflush = ((kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
486 err = hibernate_alloc_page_lists(&vars->page_list,
487 &vars->page_list_wired,
488 &vars->page_list_pal);
489 if (KERN_SUCCESS != err) break;
490
491 err = hibernate_pin_swap(TRUE);
492 if (KERN_SUCCESS != err) break;
493 swapPinned = true;
494
495 if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode))
496 {
497 hibernate_page_list_setall(vars->page_list,
498 vars->page_list_wired,
499 vars->page_list_pal,
500 true /* preflight */,
501 vmflush /* discard */,
502 &pageCount);
503 PE_Video consoleInfo;
504 bzero(&consoleInfo, sizeof(consoleInfo));
505 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
506
507 // estimate: 6% increase in pages compressed
508 // screen preview 2 images compressed 0%
509 setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
510 + vars->page_list->list_size
511 + (consoleInfo.v_width * consoleInfo.v_height * 8);
512 enum { setFileRound = 1024*1024ULL };
513 setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
514
515 HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
516 pageCount, (100ULL * gIOHibernateCompression) >> 8,
517 setFileSize, vars->fileMinSize);
518
519 if (!(kIOHibernateModeFileResize & gIOHibernateMode)
520 && (setFileSize < vars->fileMinSize))
521 {
522 setFileSize = vars->fileMinSize;
523 }
524 }
525
526 // Invalidate the image file
527 if (gDebugImageLock) {
528 IOLockLock(gDebugImageLock);
529 if (gDebugImageFileVars != 0) {
530 IOSetBootImageNVRAM(0);
531 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
532 }
533 IOLockUnlock(gDebugImageLock);
534 }
535
536 err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, 0,
537 gIOHibernateCurrentHeader, sizeof(gIOHibernateCurrentHeader),
538 &vars->fileVars, &nvramData,
539 &vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
540
541 if (KERN_SUCCESS != err)
542 {
543 IOLockLock(gFSLock);
544 if (kFSOpening != gFSState) err = kIOReturnTimeout;
545 IOLockUnlock(gFSLock);
546 }
547
548 if (KERN_SUCCESS != err)
549 {
550 HIBLOG("IOPolledFileOpen(%x)\n", err);
551 break;
552 }
553
554 // write extents for debug data usage in EFI
555 IOWriteExtentsToFile(vars->fileVars, kIOHibernateHeaderOpenSignature);
556
557 err = IOPolledFilePollersSetup(vars->fileVars, kIOPolledPreflightState);
558 if (KERN_SUCCESS != err) break;
559
560 clock_get_uptime(&startTime);
561 err = hibernate_setup(gIOHibernateCurrentHeader,
562 vmflush,
563 vars->page_list, vars->page_list_wired, vars->page_list_pal);
564 clock_get_uptime(&endTime);
565 SUB_ABSOLUTETIME(&endTime, &startTime);
566 absolutetime_to_nanoseconds(endTime, &nsec);
567
568 boolean_t haveSwapPin, hibFileSSD;
569 haveSwapPin = vm_swap_files_pinned();
570
571 hibFileSSD = (kIOPolledFileSSD & vars->fileVars->flags);
572
573 HIBLOG("hibernate_setup(%d) took %qd ms, swapPin(%d) ssd(%d)\n",
574 err, nsec / 1000000ULL,
575 haveSwapPin, hibFileSSD);
576 if (KERN_SUCCESS != err) break;
577
578 gIOHibernateStandbyDisabled = ((!haveSwapPin || !hibFileSSD));
579
580 dsSSD = ((0 != (kIOPolledFileSSD & vars->fileVars->flags))
581 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
582
583 if (dsSSD) gIOHibernateCurrentHeader->options |= kIOHibernateOptionSSD | kIOHibernateOptionColor;
584 else gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
585
586
587 #if defined(__i386__) || defined(__x86_64__)
588 if (!uuid_is_null(vars->volumeCryptKey) &&
589 (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey)))
590 {
591 uintptr_t smcVars[2];
592 smcVars[0] = sizeof(vars->volumeCryptKey);
593 smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
594
595 IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
596 bzero(smcVars, sizeof(smcVars));
597 }
598 #endif
599
600
601 if (encryptedswap || !uuid_is_null(vars->volumeCryptKey))
602 gIOHibernateMode ^= kIOHibernateModeEncrypt;
603
604 if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
605 {
606 vars->videoAllocSize = kVideoMapSize;
607 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize, VM_KERN_MEMORY_IOKIT))
608 vars->videoMapping = 0;
609 }
610
611 // generate crypt keys
612 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
613 vars->wiredCryptKey[i] = random();
614 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
615 vars->cryptKey[i] = random();
616
617 // set nvram
618
619 IOSetBootImageNVRAM(nvramData);
620 nvramData->release();
621
622 #if defined(__i386__) || defined(__x86_64__)
623 {
624 struct AppleRTCHibernateVars
625 {
626 uint8_t signature[4];
627 uint32_t revision;
628 uint8_t booterSignature[20];
629 uint8_t wiredCryptKey[16];
630 };
631 AppleRTCHibernateVars rtcVars;
632 OSData * data;
633
634 rtcVars.signature[0] = 'A';
635 rtcVars.signature[1] = 'A';
636 rtcVars.signature[2] = 'P';
637 rtcVars.signature[3] = 'L';
638 rtcVars.revision = 1;
639 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
640
641 if (gIOChosenEntry
642 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateBootSignatureKey)))
643 && (sizeof(rtcVars.booterSignature) <= data->getLength()))
644 {
645 bcopy(data->getBytesNoCopy(), &rtcVars.booterSignature[0], sizeof(rtcVars.booterSignature));
646 }
647 else if (gIOHibernateBootSignature[0])
648 {
649 char c;
650 uint8_t value = 0;
651 uint32_t in, out, digits;
652 for (in = out = digits = 0;
653 (c = gIOHibernateBootSignature[in]) && (in < sizeof(gIOHibernateBootSignature));
654 in++)
655 {
656 if ((c >= 'a') && (c <= 'f')) c -= 'a' - 10;
657 else if ((c >= 'A') && (c <= 'F')) c -= 'A' - 10;
658 else if ((c >= '0') && (c <= '9')) c -= '0';
659 else
660 {
661 if (c == '=') out = digits = value = 0;
662 continue;
663 }
664 value = (value << 4) | c;
665 if (digits & 1)
666 {
667 rtcVars.booterSignature[out++] = value;
668 if (out >= sizeof(rtcVars.booterSignature)) break;
669 }
670 digits++;
671 }
672 }
673 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
674 if (data)
675 {
676 if (gIOHibernateRTCVariablesKey)
677 IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
678 data->release();
679 }
680 if (gIOChosenEntry)
681 {
682 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
683 if (data) gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
684 // set BootNext
685 if (!gIOHibernateBoot0082Data)
686 {
687 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
688 if (data)
689 {
690 // AppleNVRAM_EFI_LOAD_OPTION
691 struct {
692 uint32_t Attributes;
693 uint16_t FilePathLength;
694 uint16_t Desc;
695 } loadOptionHeader;
696 loadOptionHeader.Attributes = 1;
697 loadOptionHeader.FilePathLength = data->getLength();
698 loadOptionHeader.Desc = 0;
699 gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
700 if (gIOHibernateBoot0082Data)
701 {
702 gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
703 gIOHibernateBoot0082Data->appendBytes(data);
704 }
705 }
706 }
707 if (!gIOHibernateBootNextData)
708 {
709 uint16_t bits = 0x0082;
710 gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
711 }
712 if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData)
713 {
714 gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
715 gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
716 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
717 }
718 // BootNext
719 }
720 }
721 #endif /* !i386 && !x86_64 */
722 }
723 while (false);
724
725 if (swapPinned) hibernate_pin_swap(FALSE);
726
727 IOLockLock(gFSLock);
728 if ((kIOReturnSuccess == err) && (kFSOpening != gFSState))
729 {
730 HIBLOG("hibernate file close due timeout\n");
731 err = kIOReturnTimeout;
732 }
733 if (kIOReturnSuccess == err)
734 {
735 gFSState = kFSOpened;
736 gIOHibernateVars = *vars;
737 gFileVars = *vars->fileVars;
738 gFileVars.allocated = false;
739 gIOHibernateVars.fileVars = &gFileVars;
740 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
741 gIOHibernateState = kIOHibernateStateHibernating;
742 }
743 else
744 {
745 IOPolledFileIOVars * fileVars = vars->fileVars;
746 IOHibernateDone(vars);
747 IOPolledFileClose(&fileVars,
748 #if DISABLE_TRIM
749 0, NULL, 0, 0, 0);
750 #else
751 0, NULL, 0, sizeof(IOHibernateImageHeader), setFileSize);
752 #endif
753 gFSState = kFSIdle;
754 }
755 IOLockUnlock(gFSLock);
756
757 if (vars->fileVars) IODelete(vars->fileVars, IOPolledFileIOVars, 1);
758 IODelete(vars, IOHibernateVars, 1);
759
760 return (err);
761 }
762
763 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
764
765 static void
766 IOSetBootImageNVRAM(OSData * data)
767 {
768 IORegistryEntry * regEntry;
769
770 if (!gIOOptionsEntry)
771 {
772 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
773 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
774 if (regEntry && !gIOOptionsEntry)
775 regEntry->release();
776 }
777 if (gIOOptionsEntry && gIOHibernateBootImageKey)
778 {
779 if (data) gIOOptionsEntry->setProperty(gIOHibernateBootImageKey, data);
780 else
781 {
782 gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
783 gIOOptionsEntry->sync();
784 }
785 }
786 }
787
788 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
789 /*
790 * Writes header to disk with signature, block size and file extents data.
791 * If there are more than 2 extents, then they are written on second block.
792 */
793 static IOReturn
794 IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature)
795 {
796 IOHibernateImageHeader hdr;
797 IOItemCount count;
798 IOReturn err = kIOReturnSuccess;
799 int rc;
800 IOPolledFileExtent * fileExtents;
801
802 fileExtents = (typeof(fileExtents)) vars->fileExtents->getBytesNoCopy();
803
804 memset(&hdr, 0, sizeof(IOHibernateImageHeader));
805 count = vars->fileExtents->getLength();
806 if (count > sizeof(hdr.fileExtentMap))
807 {
808 hdr.fileExtentMapSize = count;
809 count = sizeof(hdr.fileExtentMap);
810 }
811 else
812 hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap);
813
814 bcopy(fileExtents, &hdr.fileExtentMap[0], count);
815
816 // copy file block extent list if larger than header
817 if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap))
818 {
819 count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap);
820 rc = kern_write_file(vars->fileRef, vars->blockSize,
821 (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)),
822 count, IO_SKIP_ENCRYPTION);
823 if (rc != 0) {
824 HIBLOG("kern_write_file returned %d\n", rc);
825 err = kIOReturnIOError;
826 goto exit;
827 }
828 }
829 hdr.signature = signature;
830 hdr.deviceBlockSize = vars->blockSize;
831
832 rc = kern_write_file(vars->fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION);
833 if (rc != 0) {
834 HIBLOG("kern_write_file returned %d\n", rc);
835 err = kIOReturnIOError;
836 goto exit;
837 }
838
839 exit:
840 return err;
841 }
842
843 extern "C" boolean_t root_is_CF_drive;
844
845 void
846 IOOpenDebugDataFile(const char *fname, uint64_t size)
847 {
848 IOReturn err;
849 OSData * imagePath = NULL;
850 uint64_t padding;
851
852 if (!gDebugImageLock) {
853 gDebugImageLock = IOLockAlloc();
854 }
855
856 if (root_is_CF_drive) return;
857
858 // Try to get a lock, but don't block for getting lock
859 if (!IOLockTryLock(gDebugImageLock)) {
860 HIBLOG("IOOpenDebugDataFile: Failed to get lock\n");
861 return;
862 }
863
864 if (gDebugImageFileVars || !fname || !size) {
865 HIBLOG("IOOpenDebugDataFile: conditions failed\n");
866 goto exit;
867 }
868
869 padding = (PAGE_SIZE*2); // allocate couple more pages for header and fileextents
870 err = IOPolledFileOpen(fname, size+padding, 32ULL*1024*1024*1024,
871 NULL, 0,
872 &gDebugImageFileVars, &imagePath, NULL, 0);
873
874 if ((kIOReturnSuccess == err) && imagePath)
875 {
876 if ((gDebugImageFileVars->fileSize < (size+padding)) ||
877 (gDebugImageFileVars->fileExtents->getLength() > PAGE_SIZE)) {
878 // Can't use the file
879 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
880 HIBLOG("IOOpenDebugDataFile: too many file extents\n");
881 goto exit;
882 }
883
884 // write extents for debug data usage in EFI
885 IOWriteExtentsToFile(gDebugImageFileVars, kIOHibernateHeaderOpenSignature);
886 IOSetBootImageNVRAM(imagePath);
887 }
888
889 exit:
890 IOLockUnlock(gDebugImageLock);
891
892 if (imagePath) imagePath->release();
893 return;
894 }
895
896 void
897 IOCloseDebugDataFile()
898 {
899 IOSetBootImageNVRAM(0);
900
901 if (gDebugImageLock) {
902 IOLockLock(gDebugImageLock);
903 if (gDebugImageFileVars != 0) {
904 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
905 }
906 IOLockUnlock(gDebugImageLock);
907 }
908
909
910 }
911
912 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
913
914 DECLARE_IOHIBERNATEPROGRESSALPHA
915
916 static void
917 ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
918 {
919 uint32_t rowBytes, pixelShift;
920 uint32_t x, y;
921 int32_t blob;
922 uint32_t alpha, in, color, result;
923 uint8_t * out;
924 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
925
926 rowBytes = display->rowBytes;
927 pixelShift = display->depth >> 4;
928 if (pixelShift < 1) return;
929
930 screen += ((display->width
931 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
932 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
933
934 for (y = 0; y < kIOHibernateProgressHeight; y++)
935 {
936 out = screen + y * rowBytes;
937 for (blob = 0; blob < kIOHibernateProgressCount; blob++)
938 {
939 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
940 for (x = 0; x < kIOHibernateProgressWidth; x++)
941 {
942 alpha = gIOHibernateProgressAlpha[y][x];
943 result = color;
944 if (alpha)
945 {
946 if (0xff != alpha)
947 {
948 if (1 == pixelShift)
949 {
950 in = *((uint16_t *)out) & 0x1f; // 16
951 in = (in << 3) | (in >> 2);
952 }
953 else
954 in = *((uint32_t *)out) & 0xff; // 32
955 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
956 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
957 }
958 if (1 == pixelShift)
959 {
960 result >>= 3;
961 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
962 }
963 else
964 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
965 }
966 out += (1 << pixelShift);
967 }
968 out += (kIOHibernateProgressSpacing << pixelShift);
969 }
970 }
971 }
972
973
974 static void
975 ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
976 {
977 uint32_t rowBytes, pixelShift;
978 uint32_t x, y;
979 int32_t blob, lastBlob;
980 uint32_t alpha, in, color, result;
981 uint8_t * out;
982 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
983
984 pixelShift = display->depth >> 4;
985 if (pixelShift < 1)
986 return;
987
988 rowBytes = display->rowBytes;
989
990 screen += ((display->width
991 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
992 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
993
994 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
995
996 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
997
998 for (y = 0; y < kIOHibernateProgressHeight; y++)
999 {
1000 out = screen + y * rowBytes;
1001 for (blob = firstBlob; blob <= lastBlob; blob++)
1002 {
1003 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
1004 for (x = 0; x < kIOHibernateProgressWidth; x++)
1005 {
1006 alpha = gIOHibernateProgressAlpha[y][x];
1007 result = color;
1008 if (alpha)
1009 {
1010 if (0xff != alpha)
1011 {
1012 in = display->progressSaveUnder[blob][saveindex[blob]++];
1013 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
1014 }
1015 if (1 == pixelShift)
1016 {
1017 result >>= 3;
1018 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1019 }
1020 else
1021 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1022 }
1023 out += (1 << pixelShift);
1024 }
1025 out += (kIOHibernateProgressSpacing << pixelShift);
1026 }
1027 }
1028 }
1029
1030 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1031
1032 IOReturn
1033 IOHibernateIOKitSleep(void)
1034 {
1035 IOReturn ret = kIOReturnSuccess;
1036 IOLockLock(gFSLock);
1037 if (kFSOpening == gFSState)
1038 {
1039 gFSState = kFSTimedOut;
1040 HIBLOG("hibernate file open timed out\n");
1041 ret = kIOReturnTimeout;
1042 }
1043 IOLockUnlock(gFSLock);
1044 return (ret);
1045 }
1046
1047 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1048
1049 IOReturn
1050 IOHibernateSystemHasSlept(void)
1051 {
1052 IOReturn ret = kIOReturnSuccess;
1053 IOHibernateVars * vars = &gIOHibernateVars;
1054 OSObject * obj = 0;
1055 OSData * data;
1056
1057 IOLockLock(gFSLock);
1058 if ((kFSOpened != gFSState) && gIOHibernateMode)
1059 {
1060 ret = kIOReturnTimeout;
1061 }
1062 IOLockUnlock(gFSLock);
1063 if (kIOReturnSuccess != ret) return (ret);
1064
1065 if (gIOHibernateMode) obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
1066 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1067 if (obj && !vars->previewBuffer)
1068 obj->release();
1069
1070 vars->consoleMapping = NULL;
1071 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
1072 {
1073 vars->previewBuffer->release();
1074 vars->previewBuffer = 0;
1075 }
1076
1077 if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1078 && vars->previewBuffer
1079 && (data = OSDynamicCast(OSData,
1080 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
1081 {
1082 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
1083 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
1084
1085 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1086
1087 if (kIOHibernatePreviewUpdates & flags)
1088 {
1089 PE_Video consoleInfo;
1090 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1091
1092 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1093
1094 graphicsInfo->width = consoleInfo.v_width;
1095 graphicsInfo->height = consoleInfo.v_height;
1096 graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
1097 graphicsInfo->depth = consoleInfo.v_depth;
1098 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1099
1100 HIBPRINT("video %p %d %d %d\n",
1101 vars->consoleMapping, graphicsInfo->depth,
1102 graphicsInfo->width, graphicsInfo->height);
1103 if (vars->consoleMapping)
1104 ProgressInit(graphicsInfo, vars->consoleMapping,
1105 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1106 }
1107 }
1108
1109 if (gIOOptionsEntry)
1110 gIOOptionsEntry->sync();
1111
1112 return (ret);
1113 }
1114
1115 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1116
1117 static DeviceTreeNode *
1118 MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry)
1119 {
1120 DeviceTreeNodeProperty * prop;
1121 DeviceTreeNode * child;
1122 IORegistryEntry * childRegEntry;
1123 const char * nameProp;
1124 unsigned int propLen, idx;
1125
1126 prop = (DeviceTreeNodeProperty *) (entry + 1);
1127 for (idx = 0; idx < entry->nProperties; idx++)
1128 {
1129 if (regEntry && (0 != strcmp("name", prop->name)))
1130 {
1131 regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
1132 // HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
1133 }
1134 prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
1135 }
1136
1137 child = (DeviceTreeNode *) prop;
1138 for (idx = 0; idx < entry->nChildren; idx++)
1139 {
1140 if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen))
1141 panic("no name");
1142 childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
1143 // HIBPRINT("%s == %p\n", nameProp, childRegEntry);
1144 child = MergeDeviceTree(child, childRegEntry);
1145 }
1146 return (child);
1147 }
1148
1149 IOReturn
1150 IOHibernateSystemWake(void)
1151 {
1152 if (kFSOpened == gFSState)
1153 {
1154 IOPolledFilePollersClose(gIOHibernateVars.fileVars, kIOPolledPostflightState);
1155 IOHibernateDone(&gIOHibernateVars);
1156 }
1157 else
1158 {
1159 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1160 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1161 }
1162 return (kIOReturnSuccess);
1163 }
1164
1165 static IOReturn
1166 IOHibernateDone(IOHibernateVars * vars)
1167 {
1168 hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
1169
1170 if (vars->videoMapping)
1171 {
1172 if (vars->videoMapSize)
1173 // remove mappings
1174 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1175 if (vars->videoAllocSize)
1176 // dealloc range
1177 kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
1178 }
1179
1180 if (vars->previewBuffer)
1181 {
1182 vars->previewBuffer->release();
1183 vars->previewBuffer = 0;
1184 }
1185
1186 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1187 {
1188 IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
1189 gIOHibernateCurrentHeader->options, 32);
1190 }
1191 else
1192 {
1193 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1194 }
1195
1196 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1197 && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus))
1198 {
1199 IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
1200 &gIOHibernateGraphicsInfo->gfxStatus,
1201 sizeof(gIOHibernateGraphicsInfo->gfxStatus));
1202 }
1203 else
1204 {
1205 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1206 }
1207
1208 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1209
1210 #if defined(__i386__) || defined(__x86_64__)
1211 IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
1212 IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
1213
1214 /*
1215 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1216 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1217 */
1218 if (gIOOptionsEntry) {
1219
1220 if (gIOHibernateRTCVariablesKey) {
1221 if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
1222 gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
1223 }
1224 }
1225
1226 if (gIOHibernateBootNextKey)
1227 {
1228 if (gIOHibernateBootNextSave)
1229 {
1230 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
1231 gIOHibernateBootNextSave->release();
1232 gIOHibernateBootNextSave = NULL;
1233 }
1234 else
1235 gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
1236 }
1237 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) gIOOptionsEntry->sync();
1238 }
1239 #endif
1240
1241 if (vars->srcBuffer) vars->srcBuffer->release();
1242 bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
1243 if (vars->handoffBuffer)
1244 {
1245 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1246 {
1247 IOHibernateHandoff * handoff;
1248 bool done = false;
1249 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
1250 !done;
1251 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
1252 {
1253 HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
1254 uint8_t * data = &handoff->data[0];
1255 switch (handoff->type)
1256 {
1257 case kIOHibernateHandoffTypeEnd:
1258 done = true;
1259 break;
1260
1261 case kIOHibernateHandoffTypeDeviceTree:
1262 MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
1263 break;
1264
1265 case kIOHibernateHandoffTypeKeyStore:
1266 #if defined(__i386__) || defined(__x86_64__)
1267 {
1268 IOBufferMemoryDescriptor *
1269 md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
1270 if (md)
1271 {
1272 IOSetKeyStoreData(md);
1273 }
1274 }
1275 #endif
1276 break;
1277
1278 default:
1279 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
1280 break;
1281 }
1282 }
1283 }
1284 vars->handoffBuffer->release();
1285 }
1286
1287 bzero(vars, sizeof(*vars));
1288
1289 // gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1290
1291 return (kIOReturnSuccess);
1292 }
1293
1294 static void
1295 IOHibernateSystemPostWakeTrim(void * p1, void * p2)
1296 {
1297 // invalidate & close the image file
1298 if (p1) IOLockLock(gFSLock);
1299 if (kFSTrimDelay == gFSState)
1300 {
1301 IOPolledFileIOVars * vars = &gFileVars;
1302 IOPolledFileClose(&vars,
1303 #if DISABLE_TRIM
1304 0, NULL, 0, 0, 0);
1305 #else
1306 0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader),
1307 sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize);
1308 #endif
1309 gFSState = kFSIdle;
1310 }
1311 if (p1) IOLockUnlock(gFSLock);
1312 }
1313
1314 IOReturn
1315 IOHibernateSystemPostWake(bool now)
1316 {
1317 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1318 IOLockLock(gFSLock);
1319 if (kFSTrimDelay == gFSState) IOHibernateSystemPostWakeTrim(NULL, NULL);
1320 else if (kFSOpened != gFSState) gFSState = kFSIdle;
1321 else
1322 {
1323 gFSState = kFSTrimDelay;
1324 if (now) IOHibernateSystemPostWakeTrim(NULL, NULL);
1325 else
1326 {
1327 AbsoluteTime deadline;
1328 clock_interval_to_deadline(TRIM_DELAY, kMillisecondScale, &deadline );
1329 thread_call_enter1_delayed(gIOHibernateTrimCalloutEntry, NULL, deadline);
1330 }
1331 }
1332 IOLockUnlock(gFSLock);
1333
1334 // IOCloseDebugDataFile() calls IOSetBootImageNVRAM() unconditionally
1335 IOCloseDebugDataFile( );
1336 return (kIOReturnSuccess);
1337 }
1338
1339 uint32_t IOHibernateWasScreenLocked(void)
1340 {
1341 uint32_t ret = 0;
1342 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry)
1343 {
1344 OSData *
1345 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
1346 if (data)
1347 {
1348 ret = ((uint32_t *)data->getBytesNoCopy())[0];
1349 gIOChosenEntry->setProperty(kIOBooterScreenLockStateKey, data);
1350 }
1351 }
1352 else gIOChosenEntry->removeProperty(kIOBooterScreenLockStateKey);
1353
1354 return (ret);
1355 }
1356
1357 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1358
1359 SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
1360 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1361 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
1362 SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
1363 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1364 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
1365 SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
1366 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1367 &gIOHibernateMode, 0, "");
1368 SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
1369 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1370 &_hibernateStats, hibernate_statistics_t, "");
1371
1372 SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
1373 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1374 &_hibernateStats.graphicsReadyTime, 0, "");
1375 SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
1376 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1377 &_hibernateStats.wakeNotificationTime, 0, "");
1378 SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
1379 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1380 &_hibernateStats.lockScreenReadyTime, 0, "");
1381 SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
1382 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1383 &_hibernateStats.hidReadyTime, 0, "");
1384
1385
1386 void
1387 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1388 {
1389 gIOHibernateBootImageKey = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1390
1391 #if defined(__i386__) || defined(__x86_64__)
1392 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1393 gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
1394 gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
1395 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1396 #endif /* defined(__i386__) || defined(__x86_64__) */
1397
1398 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
1399 if (data)
1400 {
1401 rootDomain->setProperty(kIOHibernateStateKey, data);
1402 data->release();
1403 }
1404
1405 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))
1406 gIOHibernateMode = kIOHibernateModeOn;
1407 else
1408 gIOHibernateFilename[0] = 0;
1409
1410 sysctl_register_oid(&sysctl__kern_hibernatefile);
1411 sysctl_register_oid(&sysctl__kern_bootsignature);
1412 sysctl_register_oid(&sysctl__kern_hibernatemode);
1413 sysctl_register_oid(&sysctl__kern_hibernatestatistics);
1414 sysctl_register_oid(&sysctl__kern_hibernategraphicsready);
1415 sysctl_register_oid(&sysctl__kern_hibernatewakenotification);
1416 sysctl_register_oid(&sysctl__kern_hibernatelockscreenready);
1417 sysctl_register_oid(&sysctl__kern_hibernatehidready);
1418
1419 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1420
1421 gFSLock = IOLockAlloc();
1422 }
1423
1424 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1425
1426 static IOReturn
1427 IOHibernatePolledFileWrite(IOPolledFileIOVars * vars,
1428 const uint8_t * bytes, IOByteCount size,
1429 IOPolledFileCryptVars * cryptvars)
1430 {
1431 IOReturn err;
1432
1433 err = IOPolledFileWrite(vars, bytes, size, cryptvars);
1434 if ((kIOReturnSuccess == err) && hibernate_should_abort()) err = kIOReturnAborted;
1435
1436 return (err);
1437 }
1438
1439 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1440
1441 extern "C" uint32_t
1442 hibernate_write_image(void)
1443 {
1444 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
1445 IOHibernateVars * vars = &gIOHibernateVars;
1446 IOPolledFileExtent * fileExtents;
1447
1448 _static_assert_1_arg(sizeof(IOHibernateImageHeader) == 512);
1449
1450 uint32_t pageCount, pagesDone;
1451 IOReturn err;
1452 vm_offset_t ppnum, page;
1453 IOItemCount count;
1454 uint8_t * src;
1455 uint8_t * data;
1456 uint8_t * compressed;
1457 uint8_t * scratch;
1458 IOByteCount pageCompressedSize;
1459 uint64_t compressedSize, uncompressedSize;
1460 uint64_t image1Size = 0;
1461 uint32_t bitmap_size;
1462 bool iterDone, pollerOpen, needEncrypt;
1463 uint32_t restore1Sum, sum, sum1, sum2;
1464 int wkresult;
1465 uint32_t tag;
1466 uint32_t pageType;
1467 uint32_t pageAndCount[2];
1468 addr64_t phys64;
1469 IOByteCount segLen;
1470 uintptr_t hibernateBase;
1471 uintptr_t hibernateEnd;
1472
1473 AbsoluteTime startTime, endTime;
1474 AbsoluteTime allTime, compTime;
1475 uint64_t compBytes;
1476 uint64_t nsec;
1477 uint32_t lastProgressStamp = 0;
1478 uint32_t progressStamp;
1479 uint32_t blob, lastBlob = (uint32_t) -1L;
1480
1481 uint32_t wiredPagesEncrypted;
1482 uint32_t dirtyPagesEncrypted;
1483 uint32_t wiredPagesClear;
1484 uint32_t svPageCount;
1485 uint32_t zvPageCount;
1486
1487 IOPolledFileCryptVars _cryptvars;
1488 IOPolledFileCryptVars * cryptvars = 0;
1489
1490 wiredPagesEncrypted = 0;
1491 dirtyPagesEncrypted = 0;
1492 wiredPagesClear = 0;
1493 svPageCount = 0;
1494 zvPageCount = 0;
1495
1496 if (!vars->fileVars
1497 || !vars->fileVars->pollers
1498 || !(kIOHibernateModeOn & gIOHibernateMode)) return (kIOHibernatePostWriteSleep);
1499
1500 if (kIOHibernateModeSleep & gIOHibernateMode)
1501 kdebug_enable = save_kdebug_enable;
1502
1503 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START);
1504 IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
1505
1506 restore1Sum = sum1 = sum2 = 0;
1507
1508 #if CRYPTO
1509 // encryption data. "iv" is the "initial vector".
1510 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1511 {
1512 static const unsigned char first_iv[AES_BLOCK_SIZE]
1513 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
1514 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
1515
1516 cryptvars = &gIOHibernateCryptWakeContext;
1517 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1518 aes_encrypt_key(vars->cryptKey,
1519 kIOHibernateAESKeySize,
1520 &cryptvars->ctx.encrypt);
1521 aes_decrypt_key(vars->cryptKey,
1522 kIOHibernateAESKeySize,
1523 &cryptvars->ctx.decrypt);
1524
1525 cryptvars = &_cryptvars;
1526 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1527 for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++)
1528 vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
1529 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
1530 aes_encrypt_key(vars->wiredCryptKey,
1531 kIOHibernateAESKeySize,
1532 &cryptvars->ctx.encrypt);
1533
1534 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1535 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1536 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
1537 }
1538 #endif /* CRYPTO */
1539
1540 hibernate_page_list_setall(vars->page_list,
1541 vars->page_list_wired,
1542 vars->page_list_pal,
1543 false /* !preflight */,
1544 /* discard_all */
1545 ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
1546 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
1547 &pageCount);
1548
1549 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
1550
1551 fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy();
1552
1553 #if 0
1554 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
1555 for (page = 0; page < count; page++)
1556 {
1557 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
1558 fileExtents[page].start, fileExtents[page].length,
1559 fileExtents[page].start + fileExtents[page].length);
1560 }
1561 #endif
1562
1563 needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
1564 AbsoluteTime_to_scalar(&compTime) = 0;
1565 compBytes = 0;
1566
1567 clock_get_uptime(&allTime);
1568 IOService::getPMRootDomain()->pmStatsRecordEvent(
1569 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
1570 do
1571 {
1572 compressedSize = 0;
1573 uncompressedSize = 0;
1574 svPageCount = 0;
1575 zvPageCount = 0;
1576
1577 IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
1578
1579 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
1580 ml_get_interrupts_enabled());
1581 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState,
1582 // abortable if not low battery
1583 !IOService::getPMRootDomain()->mustHibernate());
1584 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
1585 pollerOpen = (kIOReturnSuccess == err);
1586 if (!pollerOpen)
1587 break;
1588
1589 // copy file block extent list if larger than header
1590
1591 count = vars->fileVars->fileExtents->getLength();
1592 if (count > sizeof(header->fileExtentMap))
1593 {
1594 count -= sizeof(header->fileExtentMap);
1595 err = IOHibernatePolledFileWrite(vars->fileVars,
1596 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
1597 if (kIOReturnSuccess != err)
1598 break;
1599 }
1600
1601 hibernateBase = HIB_BASE; /* Defined in PAL headers */
1602 hibernateEnd = (segHIBB + segSizeHIB);
1603
1604 // copy out restore1 code
1605
1606 for (count = 0;
1607 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1608 count += segLen)
1609 {
1610 for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++)
1611 {
1612 gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
1613 }
1614 }
1615
1616 page = atop_32(kvtophys(hibernateBase));
1617 count = atop_32(round_page(hibernateEnd) - hibernateBase);
1618 header->restore1CodePhysPage = page;
1619 header->restore1CodeVirt = hibernateBase;
1620 header->restore1PageCount = count;
1621 header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
1622 header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
1623
1624 // sum __HIB seg, with zeros for the stack
1625 src = (uint8_t *) trunc_page(hibernateBase);
1626 for (page = 0; page < count; page++)
1627 {
1628 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
1629 restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
1630 else
1631 restore1Sum += 0x00000000;
1632 src += page_size;
1633 }
1634 sum1 = restore1Sum;
1635
1636 // write the __HIB seg, with zeros for the stack
1637
1638 src = (uint8_t *) trunc_page(hibernateBase);
1639 count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
1640 if (count)
1641 {
1642 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
1643 if (kIOReturnSuccess != err)
1644 break;
1645 }
1646 err = IOHibernatePolledFileWrite(vars->fileVars,
1647 (uint8_t *) 0,
1648 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
1649 cryptvars);
1650 if (kIOReturnSuccess != err)
1651 break;
1652 src = &gIOHibernateRestoreStackEnd[0];
1653 count = round_page(hibernateEnd) - ((uintptr_t) src);
1654 if (count)
1655 {
1656 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
1657 if (kIOReturnSuccess != err)
1658 break;
1659 }
1660
1661 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1662 {
1663 vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
1664 vars->fileVars->encryptEnd = UINT64_MAX;
1665 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1666 }
1667
1668 // write the preview buffer
1669
1670 if (vars->previewBuffer)
1671 {
1672 ppnum = 0;
1673 count = 0;
1674 do
1675 {
1676 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
1677 pageAndCount[0] = atop_64(phys64);
1678 pageAndCount[1] = atop_32(segLen);
1679 err = IOHibernatePolledFileWrite(vars->fileVars,
1680 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1681 cryptvars);
1682 if (kIOReturnSuccess != err)
1683 break;
1684 count += segLen;
1685 ppnum += sizeof(pageAndCount);
1686 }
1687 while (phys64);
1688 if (kIOReturnSuccess != err)
1689 break;
1690
1691 src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
1692
1693 ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
1694
1695 count = vars->previewBuffer->getLength();
1696
1697 header->previewPageListSize = ppnum;
1698 header->previewSize = count + ppnum;
1699
1700 for (page = 0; page < count; page += page_size)
1701 {
1702 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
1703 sum1 += hibernate_sum_page(src + page, atop_64(phys64));
1704 }
1705 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
1706 if (kIOReturnSuccess != err)
1707 break;
1708 }
1709
1710 // mark areas for no save
1711 IOMemoryDescriptor * ioBuffer;
1712 ioBuffer = IOPolledFileGetIOBuffer(vars->fileVars);
1713 for (count = 0;
1714 (phys64 = ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1715 count += segLen)
1716 {
1717 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1718 atop_64(phys64), atop_32(segLen),
1719 kIOHibernatePageStateFree);
1720 pageCount -= atop_32(segLen);
1721 }
1722
1723 for (count = 0;
1724 (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1725 count += segLen)
1726 {
1727 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1728 atop_64(phys64), atop_32(segLen),
1729 kIOHibernatePageStateFree);
1730 pageCount -= atop_32(segLen);
1731 }
1732
1733 // copy out bitmap of pages available for trashing during restore
1734
1735 bitmap_size = vars->page_list_wired->list_size;
1736 src = (uint8_t *) vars->page_list_wired;
1737 err = IOHibernatePolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
1738 if (kIOReturnSuccess != err)
1739 break;
1740
1741 // mark more areas for no save, but these are not available
1742 // for trashing during restore
1743
1744 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
1745
1746
1747 page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
1748 count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
1749 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1750 page, count,
1751 kIOHibernatePageStateFree);
1752 pageCount -= count;
1753
1754 if (vars->previewBuffer) for (count = 0;
1755 (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1756 count += segLen)
1757 {
1758 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1759 atop_64(phys64), atop_32(segLen),
1760 kIOHibernatePageStateFree);
1761 pageCount -= atop_32(segLen);
1762 }
1763
1764 for (count = 0;
1765 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1766 count += segLen)
1767 {
1768 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1769 atop_64(phys64), atop_32(segLen),
1770 kIOHibernatePageStateFree);
1771 pageCount -= atop_32(segLen);
1772 }
1773
1774 #if KASAN
1775 vm_size_t shadow_pages_free = atop_64(shadow_ptop) - atop_64(shadow_pnext);
1776
1777 /* no need to save unused shadow pages */
1778 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1779 atop_64(shadow_pnext),
1780 shadow_pages_free,
1781 kIOHibernatePageStateFree);
1782 #endif
1783
1784 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
1785 compressed = src + page_size;
1786 scratch = compressed + page_size;
1787
1788 pagesDone = 0;
1789 lastBlob = 0;
1790
1791 HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
1792 bitmap_size, header->previewSize,
1793 pageCount, vars->fileVars->position);
1794
1795 enum
1796 // pageType
1797 {
1798 kWired = 0x02,
1799 kEncrypt = 0x01,
1800 kWiredEncrypt = kWired | kEncrypt,
1801 kWiredClear = kWired,
1802 kUnwiredEncrypt = kEncrypt
1803 };
1804
1805 bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features()));
1806 #define _pmap_is_noencrypt(x) (cpuAES ? false : pmap_is_noencrypt((x)))
1807
1808 for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)
1809 {
1810 if (kUnwiredEncrypt == pageType)
1811 {
1812 // start unwired image
1813 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1814 {
1815 vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
1816 vars->fileVars->encryptEnd = UINT64_MAX;
1817 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1818 }
1819 bcopy(&cryptvars->aes_iv[0],
1820 &gIOHibernateCryptWakeContext.aes_iv[0],
1821 sizeof(cryptvars->aes_iv));
1822 cryptvars = &gIOHibernateCryptWakeContext;
1823 }
1824 for (iterDone = false, ppnum = 0; !iterDone; )
1825 {
1826 count = hibernate_page_list_iterate((kWired & pageType)
1827 ? vars->page_list_wired : vars->page_list,
1828 &ppnum);
1829 // kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
1830 iterDone = !count;
1831
1832 if (count && (kWired & pageType) && needEncrypt)
1833 {
1834 uint32_t checkIndex;
1835 for (checkIndex = 0;
1836 (checkIndex < count)
1837 && (((kEncrypt & pageType) == 0) == _pmap_is_noencrypt(ppnum + checkIndex));
1838 checkIndex++)
1839 {}
1840 if (!checkIndex)
1841 {
1842 ppnum++;
1843 continue;
1844 }
1845 count = checkIndex;
1846 }
1847
1848 switch (pageType)
1849 {
1850 case kWiredEncrypt: wiredPagesEncrypted += count; break;
1851 case kWiredClear: wiredPagesClear += count; break;
1852 case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
1853 }
1854
1855 if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */}
1856 else
1857 {
1858 pageAndCount[0] = ppnum;
1859 pageAndCount[1] = count;
1860 err = IOHibernatePolledFileWrite(vars->fileVars,
1861 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1862 cryptvars);
1863 if (kIOReturnSuccess != err)
1864 break;
1865 }
1866
1867 for (page = ppnum; page < (ppnum + count); page++)
1868 {
1869 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
1870 if (err)
1871 {
1872 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
1873 break;
1874 }
1875
1876 sum = hibernate_sum_page(src, page);
1877 if (kWired & pageType)
1878 sum1 += sum;
1879 else
1880 sum2 += sum;
1881
1882 clock_get_uptime(&startTime);
1883 wkresult = WKdm_compress_new((const WK_word*) src,
1884 (WK_word*) compressed,
1885 (WK_word*) scratch,
1886 page_size - 4);
1887
1888 clock_get_uptime(&endTime);
1889 ADD_ABSOLUTETIME(&compTime, &endTime);
1890 SUB_ABSOLUTETIME(&compTime, &startTime);
1891
1892 compBytes += page_size;
1893 pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
1894
1895 if (pageCompressedSize == 0)
1896 {
1897 pageCompressedSize = 4;
1898 data = src;
1899
1900 if (*(uint32_t *)src)
1901 svPageCount++;
1902 else
1903 zvPageCount++;
1904 }
1905 else
1906 {
1907 if (pageCompressedSize != page_size)
1908 data = compressed;
1909 else
1910 data = src;
1911 }
1912
1913 tag = pageCompressedSize | kIOHibernateTagSignature;
1914 err = IOHibernatePolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
1915 if (kIOReturnSuccess != err)
1916 break;
1917
1918 err = IOHibernatePolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
1919 if (kIOReturnSuccess != err)
1920 break;
1921
1922 compressedSize += pageCompressedSize;
1923 uncompressedSize += page_size;
1924 pagesDone++;
1925
1926 if (vars->consoleMapping && (0 == (1023 & pagesDone)))
1927 {
1928 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
1929 if (blob != lastBlob)
1930 {
1931 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
1932 lastBlob = blob;
1933 }
1934 }
1935 if (0 == (8191 & pagesDone))
1936 {
1937 clock_get_uptime(&endTime);
1938 SUB_ABSOLUTETIME(&endTime, &allTime);
1939 absolutetime_to_nanoseconds(endTime, &nsec);
1940 progressStamp = nsec / 750000000ULL;
1941 if (progressStamp != lastProgressStamp)
1942 {
1943 lastProgressStamp = progressStamp;
1944 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
1945 }
1946 }
1947 }
1948 if (kIOReturnSuccess != err)
1949 break;
1950 ppnum = page;
1951 }
1952
1953 if (kIOReturnSuccess != err)
1954 break;
1955
1956 if ((kEncrypt & pageType) && vars->fileVars->encryptStart)
1957 {
1958 vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
1959 HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
1960 }
1961
1962 if (kWiredEncrypt != pageType)
1963 {
1964 // end of image1/2 - fill to next block
1965 err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
1966 if (kIOReturnSuccess != err)
1967 break;
1968 }
1969 if (kWiredClear == pageType)
1970 {
1971 // enlarge wired image for test
1972 // err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
1973
1974 // end wired image
1975 header->encryptStart = vars->fileVars->encryptStart;
1976 header->encryptEnd = vars->fileVars->encryptEnd;
1977 image1Size = vars->fileVars->position;
1978 HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
1979 image1Size, header->encryptStart, header->encryptEnd);
1980 }
1981 }
1982 if (kIOReturnSuccess != err)
1983 {
1984 if (kIOReturnOverrun == err)
1985 {
1986 // update actual compression ratio on not enough space (for retry)
1987 gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
1988 }
1989
1990 // update partial amount written (for IOPolledFileClose cleanup/unmap)
1991 header->imageSize = vars->fileVars->position;
1992 break;
1993 }
1994
1995 // Header:
1996
1997 header->imageSize = vars->fileVars->position;
1998 header->image1Size = image1Size;
1999 header->bitmapSize = bitmap_size;
2000 header->pageCount = pageCount;
2001
2002 header->restore1Sum = restore1Sum;
2003 header->image1Sum = sum1;
2004 header->image2Sum = sum2;
2005 header->sleepTime = gIOLastSleepTime.tv_sec;
2006
2007 header->compression = (compressedSize << 8) / uncompressedSize;
2008 gIOHibernateCompression = header->compression;
2009
2010 count = vars->fileVars->fileExtents->getLength();
2011 if (count > sizeof(header->fileExtentMap))
2012 {
2013 header->fileExtentMapSize = count;
2014 count = sizeof(header->fileExtentMap);
2015 }
2016 else
2017 header->fileExtentMapSize = sizeof(header->fileExtentMap);
2018 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
2019
2020 header->deviceBase = vars->fileVars->block0;
2021 header->deviceBlockSize = vars->fileVars->blockSize;
2022
2023 IOPolledFileSeek(vars->fileVars, 0);
2024 err = IOHibernatePolledFileWrite(vars->fileVars,
2025 (uint8_t *) header, sizeof(IOHibernateImageHeader),
2026 cryptvars);
2027 if (kIOReturnSuccess != err)
2028 break;
2029 err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2030 }
2031 while (false);
2032
2033 clock_get_uptime(&endTime);
2034
2035 IOService::getPMRootDomain()->pmStatsRecordEvent(
2036 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
2037
2038 SUB_ABSOLUTETIME(&endTime, &allTime);
2039 absolutetime_to_nanoseconds(endTime, &nsec);
2040 HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
2041
2042 absolutetime_to_nanoseconds(compTime, &nsec);
2043 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2044 compBytes,
2045 nsec / 1000000ULL,
2046 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2047
2048 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2049 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
2050 vars->fileVars->cryptBytes,
2051 nsec / 1000000ULL,
2052 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2053
2054 HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
2055 header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
2056 uncompressedSize, atop_32(uncompressedSize), compressedSize,
2057 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
2058 sum1, sum2);
2059
2060 HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
2061 svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
2062
2063 if (pollerOpen)
2064 IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted );
2065
2066 if (vars->consoleMapping)
2067 ProgressUpdate(gIOHibernateGraphicsInfo,
2068 vars->consoleMapping, 0, kIOHibernateProgressCount);
2069
2070 HIBLOG("hibernate_write_image done(%x)\n", err);
2071
2072 // should we come back via regular wake, set the state in memory.
2073 gIOHibernateState = kIOHibernateStateInactive;
2074
2075 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, wiredPagesEncrypted,
2076 wiredPagesClear, dirtyPagesEncrypted);
2077
2078 if (kIOReturnSuccess == err)
2079 {
2080 if (kIOHibernateModeSleep & gIOHibernateMode)
2081 {
2082 return (kIOHibernatePostWriteSleep);
2083 }
2084 else if(kIOHibernateModeRestart & gIOHibernateMode)
2085 {
2086 return (kIOHibernatePostWriteRestart);
2087 }
2088 else
2089 {
2090 /* by default, power down */
2091 return (kIOHibernatePostWriteHalt);
2092 }
2093 }
2094 else if (kIOReturnAborted == err)
2095 {
2096 return (kIOHibernatePostWriteWake);
2097 }
2098 else
2099 {
2100 /* on error, sleep */
2101 return (kIOHibernatePostWriteSleep);
2102 }
2103 }
2104
2105 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2106
2107 extern "C" void
2108 hibernate_machine_init(void)
2109 {
2110 IOReturn err;
2111 uint32_t sum;
2112 uint32_t pagesDone;
2113 uint32_t pagesRead = 0;
2114 AbsoluteTime startTime, compTime;
2115 AbsoluteTime allTime, endTime;
2116 AbsoluteTime startIOTime, endIOTime;
2117 uint64_t nsec, nsecIO;
2118 uint64_t compBytes;
2119 uint32_t lastProgressStamp = 0;
2120 uint32_t progressStamp;
2121 IOPolledFileCryptVars * cryptvars = 0;
2122
2123 IOHibernateVars * vars = &gIOHibernateVars;
2124 bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
2125
2126 if (!vars->fileVars || !vars->fileVars->pollers)
2127 return;
2128
2129 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2130 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2131
2132 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
2133 {
2134 HIBLOG("regular wake\n");
2135 return;
2136 }
2137
2138 HIBPRINT("diag %x %x %x %x\n",
2139 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
2140 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
2141
2142 #define t40ms(x) (tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)
2143 #define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
2144 tStat(booterStart, booterStart);
2145 gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart;
2146 tStat(booterDuration0, booterTime0);
2147 tStat(booterDuration1, booterTime1);
2148 tStat(booterDuration2, booterTime2);
2149 tStat(booterDuration, booterTime);
2150 tStat(booterConnectDisplayDuration, connectDisplayTime);
2151 tStat(booterSplashDuration, splashTime);
2152 tStat(trampolineDuration, trampolineTime);
2153
2154 gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size;
2155 gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize;
2156 gIOHibernateStats->image1Pages = pagesDone;
2157
2158 /* HIBERNATE_stats */
2159 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 14), gIOHibernateStats->smcStart,
2160 gIOHibernateStats->booterStart, gIOHibernateStats->booterDuration,
2161 gIOHibernateStats->trampolineDuration);
2162
2163 HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
2164 gIOHibernateStats->booterStart,
2165 gIOHibernateStats->smcStart,
2166 gIOHibernateStats->booterDuration0,
2167 gIOHibernateStats->booterDuration1,
2168 gIOHibernateStats->booterDuration2,
2169 gIOHibernateStats->booterDuration,
2170 gIOHibernateStats->booterConnectDisplayDuration,
2171 gIOHibernateStats->booterSplashDuration,
2172 gIOHibernateStats->trampolineDuration);
2173
2174 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
2175 gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
2176 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
2177
2178 if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
2179 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)))
2180 {
2181 hibernate_page_list_discard(vars->page_list);
2182 }
2183
2184 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
2185
2186 if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount)
2187 panic("handoff overflow");
2188
2189 IOHibernateHandoff * handoff;
2190 bool done = false;
2191 bool foundCryptData = false;
2192
2193 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
2194 !done;
2195 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
2196 {
2197 // HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
2198 uint8_t * data = &handoff->data[0];
2199 switch (handoff->type)
2200 {
2201 case kIOHibernateHandoffTypeEnd:
2202 done = true;
2203 break;
2204
2205 case kIOHibernateHandoffTypeGraphicsInfo:
2206 if (handoff->bytecount == sizeof(*gIOHibernateGraphicsInfo))
2207 {
2208 bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
2209 }
2210 break;
2211
2212 case kIOHibernateHandoffTypeCryptVars:
2213 if (cryptvars)
2214 {
2215 hibernate_cryptwakevars_t *
2216 wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
2217 bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
2218 }
2219 foundCryptData = true;
2220 bzero(data, handoff->bytecount);
2221 break;
2222
2223 case kIOHibernateHandoffTypeMemoryMap:
2224
2225 clock_get_uptime(&allTime);
2226
2227 hibernate_newruntime_map(data, handoff->bytecount,
2228 gIOHibernateCurrentHeader->systemTableOffset);
2229
2230 clock_get_uptime(&endTime);
2231
2232 SUB_ABSOLUTETIME(&endTime, &allTime);
2233 absolutetime_to_nanoseconds(endTime, &nsec);
2234
2235 HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
2236
2237 break;
2238
2239 case kIOHibernateHandoffTypeDeviceTree:
2240 {
2241 // DTEntry chosen = NULL;
2242 // HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen));
2243 }
2244 break;
2245
2246 default:
2247 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
2248 break;
2249 }
2250 }
2251 if (cryptvars && !foundCryptData)
2252 panic("hibernate handoff");
2253
2254 HIBPRINT("video 0x%llx %d %d %d status %x\n",
2255 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2256 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
2257
2258 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)
2259 {
2260 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2261 * gIOHibernateGraphicsInfo->rowBytes);
2262 if (vars->videoMapSize > vars->videoAllocSize) vars->videoMapSize = 0;
2263 else
2264 {
2265 IOMapPages(kernel_map,
2266 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2267 vars->videoMapSize, kIOMapInhibitCache );
2268 }
2269 }
2270
2271 if (vars->videoMapSize)
2272 ProgressUpdate(gIOHibernateGraphicsInfo,
2273 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2274
2275 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2276 uint8_t * compressed = src + page_size;
2277 uint8_t * scratch = compressed + page_size;
2278 uint32_t decoOffset;
2279
2280 clock_get_uptime(&allTime);
2281 AbsoluteTime_to_scalar(&compTime) = 0;
2282 compBytes = 0;
2283
2284 HIBLOG("IOPolledFilePollersOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2285 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledAfterSleepState, false);
2286 clock_get_uptime(&startIOTime);
2287 endTime = startIOTime;
2288 SUB_ABSOLUTETIME(&endTime, &allTime);
2289 absolutetime_to_nanoseconds(endTime, &nsec);
2290 HIBLOG("IOPolledFilePollersOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
2291
2292 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2293
2294 // kick off the read ahead
2295 vars->fileVars->bufferHalf = 0;
2296 vars->fileVars->bufferLimit = 0;
2297 vars->fileVars->lastRead = 0;
2298 vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
2299 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2300 vars->fileVars->cryptBytes = 0;
2301 AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
2302
2303 err = IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
2304 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2305 // --
2306
2307 HIBLOG("hibernate_machine_init reading\n");
2308
2309 uint32_t * header = (uint32_t *) src;
2310 sum = 0;
2311
2312 while (kIOReturnSuccess == err)
2313 {
2314 unsigned int count;
2315 unsigned int page;
2316 uint32_t tag;
2317 vm_offset_t ppnum, compressedSize;
2318
2319 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2320 if (kIOReturnSuccess != err)
2321 break;
2322
2323 ppnum = header[0];
2324 count = header[1];
2325
2326 // HIBPRINT("(%x, %x)\n", ppnum, count);
2327
2328 if (!count)
2329 break;
2330
2331 for (page = 0; page < count; page++)
2332 {
2333 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2334 if (kIOReturnSuccess != err)
2335 break;
2336
2337 compressedSize = kIOHibernateTagLength & tag;
2338 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength))
2339 {
2340 err = kIOReturnIPCError;
2341 break;
2342 }
2343
2344 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2345 if (kIOReturnSuccess != err) break;
2346
2347 if (compressedSize < page_size)
2348 {
2349 decoOffset = page_size;
2350 clock_get_uptime(&startTime);
2351
2352 if (compressedSize == 4) {
2353 int i;
2354 uint32_t *s, *d;
2355
2356 s = (uint32_t *)src;
2357 d = (uint32_t *)(uintptr_t)compressed;
2358
2359 for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++)
2360 *d++ = *s;
2361 }
2362 else
2363 WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, compressedSize);
2364 clock_get_uptime(&endTime);
2365 ADD_ABSOLUTETIME(&compTime, &endTime);
2366 SUB_ABSOLUTETIME(&compTime, &startTime);
2367 compBytes += page_size;
2368 }
2369 else decoOffset = 0;
2370
2371 sum += hibernate_sum_page((src + decoOffset), ppnum);
2372 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2373 if (err)
2374 {
2375 HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
2376 break;
2377 }
2378
2379 ppnum++;
2380 pagesDone++;
2381 pagesRead++;
2382
2383 if (0 == (8191 & pagesDone))
2384 {
2385 clock_get_uptime(&endTime);
2386 SUB_ABSOLUTETIME(&endTime, &allTime);
2387 absolutetime_to_nanoseconds(endTime, &nsec);
2388 progressStamp = nsec / 750000000ULL;
2389 if (progressStamp != lastProgressStamp)
2390 {
2391 lastProgressStamp = progressStamp;
2392 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2393 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2394 }
2395 }
2396 }
2397 }
2398 if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages))
2399 err = kIOReturnLockedRead;
2400
2401 if (kIOReturnSuccess != err)
2402 panic("Hibernate restore error %x", err);
2403
2404 gIOHibernateCurrentHeader->actualImage2Sum = sum;
2405 gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
2406
2407 clock_get_uptime(&endIOTime);
2408
2409 err = IOPolledFilePollersClose(vars->fileVars, kIOPolledAfterSleepState);
2410
2411 clock_get_uptime(&endTime);
2412
2413 IOService::getPMRootDomain()->pmStatsRecordEvent(
2414 kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
2415 IOService::getPMRootDomain()->pmStatsRecordEvent(
2416 kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
2417
2418 SUB_ABSOLUTETIME(&endTime, &allTime);
2419 absolutetime_to_nanoseconds(endTime, &nsec);
2420
2421 SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
2422 absolutetime_to_nanoseconds(endIOTime, &nsecIO);
2423
2424 gIOHibernateStats->kernelImageReadDuration = nsec / 1000000ULL;
2425 gIOHibernateStats->imagePages = pagesDone;
2426
2427 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
2428 pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
2429 nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
2430
2431 absolutetime_to_nanoseconds(compTime, &nsec);
2432 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2433 compBytes,
2434 nsec / 1000000ULL,
2435 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2436
2437 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2438 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
2439 vars->fileVars->cryptBytes,
2440 nsec / 1000000ULL,
2441 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2442
2443 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 2), pagesRead, pagesDone);
2444 }
2445
2446 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2447
2448 void IOHibernateSetWakeCapabilities(uint32_t capability)
2449 {
2450 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
2451 {
2452 gIOHibernateStats->wakeCapability = capability;
2453
2454 if (kIOPMSystemCapabilityGraphics & capability)
2455 {
2456 vm_compressor_do_warmup();
2457 }
2458 }
2459 }
2460
2461 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2462
2463 void IOHibernateSystemRestart(void)
2464 {
2465 static uint8_t noteStore[32] __attribute__((aligned(32)));
2466 IORegistryEntry * regEntry;
2467 const OSSymbol * sym;
2468 OSData * noteProp;
2469 OSData * data;
2470 uintptr_t * smcVars;
2471 uint8_t * smcBytes;
2472 size_t len;
2473 addr64_t element;
2474
2475 data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
2476 if (!data) return;
2477
2478 smcVars = (typeof(smcVars)) data->getBytesNoCopy();
2479 smcBytes = (typeof(smcBytes)) smcVars[1];
2480 len = smcVars[0];
2481 if (len > sizeof(noteStore)) len = sizeof(noteStore);
2482 noteProp = OSData::withCapacity(3 * sizeof(element));
2483 if (!noteProp) return;
2484 element = len;
2485 noteProp->appendBytes(&element, sizeof(element));
2486 element = crc32(0, smcBytes, len);
2487 noteProp->appendBytes(&element, sizeof(element));
2488
2489 bcopy(smcBytes, noteStore, len);
2490 element = (addr64_t) &noteStore[0];
2491 element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
2492 noteProp->appendBytes(&element, sizeof(element));
2493
2494 if (!gIOOptionsEntry)
2495 {
2496 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
2497 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
2498 if (regEntry && !gIOOptionsEntry)
2499 regEntry->release();
2500 }
2501
2502 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
2503 if (gIOOptionsEntry && sym) gIOOptionsEntry->setProperty(sym, noteProp);
2504 if (noteProp) noteProp->release();
2505 if (sym) sym->release();
2506 }
2507
2508
2509
mirror of XNU