]> git.saurik.com Git - apple/xnu.git/blob - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blob
? search:


851adac6035d329bc50785a11bf56e6cfbac0d96
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
1 /*
2 * Copyright (c) 2004-2008 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29
30 /*
31
32 Sleep:
33
34 - PMRootDomain calls IOHibernateSystemSleep() before system sleep
35 (devices awake, normal execution context)
36 - IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
37 grabs its extents and searches for a polling driver willing to work with that IOMedia.
38 The BSD code makes an ioctl to the storage driver to get the partition base offset to
39 the disk, and other ioctls to get the transfer constraints
40 If successful, the file is written to make sure its initially not bootable (in case of
41 later failure) and nvram set to point to the first block of the file. (Has to be done
42 here so blocking is possible in nvram support).
43 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
44 page out any pages it wants to (currently zero, but probably some percentage of memory).
45 Its assumed just allocating pages will cause the VM system to naturally select the best
46 pages for eviction. It also copies processor flags needed for the restore path and sets
47 a flag in the boot processor proc info.
48 gIOHibernateState = kIOHibernateStateHibernating.
49 - Regular sleep progresses - some drivers may inspect the root domain property
50 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
51 as usual but leaves motherboard I/O on.
52 - Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
53 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
54 all ppc RC bits out of the hash table and caches into the mapping structures.
55 - hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
56 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
57 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
58 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
59 The image header and block list are written. The header includes the second file extent so
60 only the header block is needed to read the file, regardless of filesystem.
61 The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
62 (only) is used to decompress the image during wake/boot.
63 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
64 The bitmaps are written to the image.
65 More areas are removed from the bitmaps (after they have been written to the image) - the
66 segment "__HIB" pages and interrupt stack.
67 Each wired page is compressed and written and then each non-wired page. Compression and
68 disk writes are in parallel.
69 The image header is written to the start of the file and the polling driver closed.
70 The machine powers down (or sleeps).
71
72 Boot/Wake:
73
74 - BootX sees the boot-image nvram variable containing the device and block number of the image,
75 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
76 - BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
77 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
78 that is in the kernel's __HIB section.
79 - BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
80 hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
81 only code & data in that section is safe to call since all the other wired pages are still
82 compressed in the image.
83 - hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
84 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
85 location directly, and copies those that can't to interim free pages. When the image has been
86 completed, the copies are uncompressed, overwriting the wired image pages.
87 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
88 is used to get pages into place for 64bit.
89 - the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
90 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
91 are removed from the software strutures, and the hash table is reinitialized.
92 - After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
93 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
94 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
95 for the remaining non wired pages.
96 - The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
97 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
98 is closed via bsd.
99
100 Polled Mode I/O:
101
102 IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
103 registry, specifying an object of calls IOPolledInterface.
104
105 Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
106 partition) that the image is going to live, looking for polled interface properties. If it finds
107 one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
108 interfaces found are kept in an ordered list.
109
110 There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
111 few different contexts things happen in:
112
113 - there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
114 up and running) and after wake - this is safe to allocate memory and do anything. The device
115 ignores sleep requests from that point since its a waste of time if it goes to sleep and
116 immediately wakes back up for the image write.
117
118 - there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
119 immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
120 from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
121 used to flush and set the disk to sleep.
122
123 - there is an Open/Close (AfterSleep) pair made around the image read operations that happen
124 immediately after sleep. These can't block or allocate memory. This is happening after the platform
125 expert has woken the low level bits of the system, but most of the I/O system has not. There is only
126 one thread running.
127
128 For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
129 (prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
130 the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
131 Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
132 that is called for the hardware to check for events, and complete the I/O via the callback.
133 The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
134 to restrict I/O ops.
135 */
136
137 #include <sys/systm.h>
138
139 #include <IOKit/IOWorkLoop.h>
140 #include <IOKit/IOCommandGate.h>
141 #include <IOKit/IOTimerEventSource.h>
142 #include <IOKit/IOPlatformExpert.h>
143 #include <IOKit/IOKitDebug.h>
144 #include <IOKit/IOTimeStamp.h>
145 #include <IOKit/pwr_mgt/RootDomain.h>
146 #include <IOKit/pwr_mgt/IOPMPrivate.h>
147 #include <IOKit/IOMessage.h>
148 #include <IOKit/IODeviceTreeSupport.h>
149 #include <IOKit/IOBSD.h>
150 #include "RootDomainUserClient.h"
151 #include <IOKit/pwr_mgt/IOPowerConnection.h>
152 #include "IOPMPowerStateQueue.h"
153 #include <IOKit/IOBufferMemoryDescriptor.h>
154 #include <IOKit/AppleKeyStoreInterface.h>
155 #include <libkern/crypto/aes.h>
156
157 #include <sys/uio.h>
158 #include <sys/conf.h>
159 #include <sys/stat.h>
160 #include <sys/fcntl.h> // (FWRITE, ...)
161 #include <sys/sysctl.h>
162 #include <sys/kdebug.h>
163
164 #include <IOKit/IOHibernatePrivate.h>
165 #include <IOKit/IOPolledInterface.h>
166 #include <IOKit/IONVRAM.h>
167 #include "IOHibernateInternal.h"
168 #include <vm/WKdm_new.h>
169 #include "IOKitKernelInternal.h"
170 #include <pexpert/device_tree.h>
171
172 #include <machine/pal_routines.h>
173 #include <machine/pal_hibernate.h>
174 #include <i386/tsc.h>
175
176 extern "C" addr64_t kvtophys(vm_offset_t va);
177 extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
178
179 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
180
181 #define DISABLE_TRIM 0
182 #define TRIM_DELAY 5000
183
184 extern unsigned int save_kdebug_enable;
185 extern uint32_t gIOHibernateState;
186 uint32_t gIOHibernateMode;
187 static char gIOHibernateBootSignature[256+1];
188 static char gIOHibernateFilename[MAXPATHLEN+1];
189 static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
190 uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
191 static uint64_t gIOHibernateCompression = 0x80; // default compression 50%
192
193 static IODTNVRAM * gIOOptionsEntry;
194 static IORegistryEntry * gIOChosenEntry;
195 #if defined(__i386__) || defined(__x86_64__)
196 static const OSSymbol * gIOCreateEFIDevicePathSymbol;
197 static const OSSymbol * gIOHibernateRTCVariablesKey;
198 static const OSSymbol * gIOHibernateBoot0082Key;
199 static const OSSymbol * gIOHibernateBootNextKey;
200 static OSData * gIOHibernateBoot0082Data;
201 static OSData * gIOHibernateBootNextData;
202 static OSObject * gIOHibernateBootNextSave;
203 #endif
204
205 static IOLock * gFSLock;
206 static uint32_t gFSState;
207 static IOPolledFileIOVars gFileVars;
208 static IOHibernateVars gIOHibernateVars;
209 static struct kern_direct_file_io_ref_t * gIOHibernateFileRef;
210 static hibernate_cryptvars_t gIOHibernateCryptWakeContext;
211 static hibernate_graphics_t _hibernateGraphics;
212 static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
213 static hibernate_statistics_t _hibernateStats;
214 static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats;
215
216 enum
217 {
218 kFSIdle = 0,
219 kFSOpening = 2,
220 kFSOpened = 3,
221 kFSTimedOut = 4,
222 };
223
224 static IOReturn IOHibernateDone(IOHibernateVars * vars);
225
226 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
227
228 enum { kXPRamAudioVolume = 8 };
229 enum { kDefaultIOSize = 128 * 1024 };
230 enum { kVideoMapSize = 80 * 1024 * 1024 };
231
232 #ifndef kIOMediaPreferredBlockSizeKey
233 #define kIOMediaPreferredBlockSizeKey "Preferred Block Size"
234 #endif
235
236 #ifndef kIOBootPathKey
237 #define kIOBootPathKey "bootpath"
238 #endif
239 #ifndef kIOSelectedBootDeviceKey
240 #define kIOSelectedBootDeviceKey "boot-device"
241 #endif
242
243 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
244
245 // copy from phys addr to MD
246
247 static IOReturn
248 IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
249 IOByteCount offset, addr64_t bytes, IOByteCount length)
250 {
251 addr64_t srcAddr = bytes;
252 IOByteCount remaining;
253
254 remaining = length = min(length, md->getLength() - offset);
255 while (remaining) { // (process another target segment?)
256 addr64_t dstAddr64;
257 IOByteCount dstLen;
258
259 dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
260 if (!dstAddr64)
261 break;
262
263 // Clip segment length to remaining
264 if (dstLen > remaining)
265 dstLen = remaining;
266
267 #if 1
268 bcopy_phys(srcAddr, dstAddr64, dstLen);
269 #else
270 copypv(srcAddr, dstAddr64, dstLen,
271 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
272 #endif
273 srcAddr += dstLen;
274 offset += dstLen;
275 remaining -= dstLen;
276 }
277
278 assert(!remaining);
279
280 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
281 }
282
283 // copy from MD to phys addr
284
285 static IOReturn
286 IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
287 IOByteCount offset, addr64_t bytes, IOByteCount length)
288 {
289 addr64_t dstAddr = bytes;
290 IOByteCount remaining;
291
292 remaining = length = min(length, md->getLength() - offset);
293 while (remaining) { // (process another target segment?)
294 addr64_t srcAddr64;
295 IOByteCount dstLen;
296
297 srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
298 if (!srcAddr64)
299 break;
300
301 // Clip segment length to remaining
302 if (dstLen > remaining)
303 dstLen = remaining;
304
305 #if 1
306 bcopy_phys(srcAddr64, dstAddr, dstLen);
307 #else
308 copypv(srcAddr, dstAddr64, dstLen,
309 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
310 #endif
311 dstAddr += dstLen;
312 offset += dstLen;
313 remaining -= dstLen;
314 }
315
316 assert(!remaining);
317
318 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
319 }
320
321 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
322
323 void
324 hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
325 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
326 {
327 count += ppnum;
328 switch (kind)
329 {
330 case kIOHibernatePageStateUnwiredSave:
331 // unwired save
332 for (; ppnum < count; ppnum++)
333 {
334 hibernate_page_bitset(page_list, FALSE, ppnum);
335 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
336 }
337 break;
338 case kIOHibernatePageStateWiredSave:
339 // wired save
340 for (; ppnum < count; ppnum++)
341 {
342 hibernate_page_bitset(page_list, FALSE, ppnum);
343 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
344 }
345 break;
346 case kIOHibernatePageStateFree:
347 // free page
348 for (; ppnum < count; ppnum++)
349 {
350 hibernate_page_bitset(page_list, TRUE, ppnum);
351 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
352 }
353 break;
354 default:
355 panic("hibernate_set_page_state");
356 }
357 }
358
359 static vm_offset_t
360 hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
361 {
362 uint32_t page = *pPage;
363 uint32_t count;
364 hibernate_bitmap_t * bitmap;
365
366 while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
367 {
368 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
369 if (!count)
370 break;
371 page += count;
372 if (page <= bitmap->last_page)
373 break;
374 }
375
376 *pPage = page;
377 if (bitmap)
378 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
379 else
380 count = 0;
381
382 return (count);
383 }
384
385 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
386
387 static IOReturn
388 IOHibernatePollerProbe(IOPolledFileIOVars * vars, IOService * target)
389 {
390 IOReturn err = kIOReturnError;
391 int32_t idx;
392 IOPolledInterface * poller;
393
394 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
395 {
396 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
397 err = poller->probe(target);
398 if (err)
399 {
400 HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
401 break;
402 }
403 }
404
405 return (err);
406 }
407
408 static IOReturn
409 IOHibernatePollerOpen(IOPolledFileIOVars * vars, uint32_t state, IOMemoryDescriptor * md)
410 {
411 IOReturn err = kIOReturnError;
412 int32_t idx;
413 IOPolledInterface * poller;
414
415 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
416 {
417 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
418 err = poller->open(state, md);
419 if (err)
420 {
421 HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
422 break;
423 }
424 }
425
426 return (err);
427 }
428
429 static IOReturn
430 IOHibernatePollerClose(IOPolledFileIOVars * vars, uint32_t state)
431 {
432 IOReturn err = kIOReturnError;
433 int32_t idx;
434 IOPolledInterface * poller;
435
436 for (idx = 0;
437 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
438 idx++)
439 {
440 err = poller->close(state);
441 if (err)
442 HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
443 }
444
445 return (err);
446 }
447
448 static void
449 IOHibernatePollerIOComplete(void * target,
450 void * parameter,
451 IOReturn status,
452 UInt64 actualByteCount)
453 {
454 IOPolledFileIOVars * vars = (IOPolledFileIOVars *) parameter;
455
456 vars->ioStatus = status;
457 }
458
459 static IOReturn
460 IOHibernatePollerIO(IOPolledFileIOVars * vars,
461 uint32_t operation, uint32_t bufferOffset,
462 uint64_t deviceOffset, uint64_t length)
463 {
464
465 IOReturn err = kIOReturnError;
466 IOPolledInterface * poller;
467 IOPolledCompletion completion;
468
469 completion.target = 0;
470 completion.action = &IOHibernatePollerIOComplete;
471 completion.parameter = vars;
472
473 vars->ioStatus = -1;
474
475 poller = (IOPolledInterface *) vars->pollers->getObject(0);
476 err = poller->startIO(operation, bufferOffset, deviceOffset + vars->block0, length, completion);
477 if (err)
478 HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
479
480 return (err);
481 }
482
483 static IOReturn
484 IOHibernatePollerIODone(IOPolledFileIOVars * vars, bool abortable)
485 {
486 IOReturn err = kIOReturnSuccess;
487 int32_t idx = 0;
488 IOPolledInterface * poller;
489
490 while (-1 == vars->ioStatus)
491 {
492 for (idx = 0;
493 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
494 idx++)
495 {
496 IOReturn newErr;
497 newErr = poller->checkForWork();
498 if ((newErr == kIOReturnAborted) && !abortable)
499 newErr = kIOReturnSuccess;
500 if (kIOReturnSuccess == err)
501 err = newErr;
502 }
503 }
504
505 if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort())
506 {
507 err = kIOReturnAborted;
508 HIBLOG("IOPolledInterface::checkForWork sw abort\n");
509 }
510
511 if (err)
512 {
513 HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
514 }
515 else
516 {
517 err = vars->ioStatus;
518 if (kIOReturnSuccess != err)
519 HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
520 }
521
522 return (err);
523 }
524
525 IOReturn
526 IOPolledInterface::checkAllForWork(void)
527 {
528 IOReturn err = kIOReturnNotReady;
529 int32_t idx;
530 IOPolledInterface * poller;
531
532 IOHibernateVars * vars = &gIOHibernateVars;
533
534 if (!vars->fileVars || !vars->fileVars->pollers)
535 return (err);
536
537 for (idx = 0;
538 (poller = (IOPolledInterface *) vars->fileVars->pollers->getObject(idx));
539 idx++)
540 {
541 err = poller->checkForWork();
542 if (err)
543 HIBLOG("IOPolledInterface::checkAllForWork[%d] 0x%x\n", idx, err);
544 }
545
546 return (err);
547 }
548
549 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
550
551 struct _OpenFileContext
552 {
553 OSData * extents;
554 uint64_t size;
555 };
556
557 static void
558 file_extent_callback(void * ref, uint64_t start, uint64_t length)
559 {
560 _OpenFileContext * ctx = (_OpenFileContext *) ref;
561 IOPolledFileExtent extent;
562
563 extent.start = start;
564 extent.length = length;
565
566 HIBLOG("[0x%qx, 0x%qx]\n", start, length);
567
568 ctx->extents->appendBytes(&extent, sizeof(extent));
569 ctx->size += length;
570 }
571
572 static IOService *
573 IOCopyMediaForDev(dev_t device)
574 {
575 OSDictionary * matching;
576 OSNumber * num;
577 OSIterator * iter;
578 IOService * result = 0;
579
580 matching = IOService::serviceMatching("IOMedia");
581 if (!matching)
582 return (0);
583 do
584 {
585 num = OSNumber::withNumber(major(device), 32);
586 if (!num)
587 break;
588 matching->setObject(kIOBSDMajorKey, num);
589 num->release();
590 num = OSNumber::withNumber(minor(device), 32);
591 if (!num)
592 break;
593 matching->setObject(kIOBSDMinorKey, num);
594 num->release();
595 if (!num)
596 break;
597 iter = IOService::getMatchingServices(matching);
598 if (iter)
599 {
600 result = (IOService *) iter->getNextObject();
601 result->retain();
602 iter->release();
603 }
604 }
605 while (false);
606 matching->release();
607
608 return (result);
609 }
610
611 IOReturn
612 IOPolledFileOpen( const char * filename, uint64_t setFileSize,
613 IOBufferMemoryDescriptor * ioBuffer,
614 IOPolledFileIOVars ** fileVars, OSData ** fileExtents,
615 OSData ** imagePath, uint8_t * volumeCryptKey)
616 {
617 IOReturn err = kIOReturnSuccess;
618 IOPolledFileIOVars * vars;
619 _OpenFileContext ctx;
620 OSData * extentsData;
621 OSNumber * num;
622 IOService * part = 0;
623 OSString * keyUUID = 0;
624 OSString * keyStoreUUID = 0;
625 dev_t block_dev;
626 dev_t hibernate_image_dev;
627 uint64_t maxiobytes;
628 AbsoluteTime startTime, endTime;
629 uint64_t nsec;
630
631 vars = IONew(IOPolledFileIOVars, 1);
632 if (!vars) return (kIOReturnNoMemory);
633 bzero(vars, sizeof(*vars));
634
635 do
636 {
637 vars->io = false;
638 vars->buffer = (uint8_t *) ioBuffer->getBytesNoCopy();
639 vars->bufferHalf = 0;
640 vars->bufferOffset = 0;
641 vars->bufferSize = ioBuffer->getLength() >> 1;
642
643 extentsData = OSData::withCapacity(32);
644 ctx.extents = extentsData;
645 ctx.size = 0;
646 clock_get_uptime(&startTime);
647 vars->fileRef = kern_open_file_for_direct_io(filename,
648 &file_extent_callback, &ctx,
649 setFileSize,
650 // write file:
651 0, (caddr_t) gIOHibernateCurrentHeader,
652 sizeof(IOHibernateImageHeader),
653 // results
654 &block_dev,
655 &hibernate_image_dev,
656 &vars->block0,
657 &maxiobytes,
658 &vars->flags);
659 #if 0
660 uint32_t msDelay = (131071 & random());
661 HIBLOG("sleep %d\n", msDelay);
662 IOSleep(msDelay);
663 #endif
664 clock_get_uptime(&endTime);
665 SUB_ABSOLUTETIME(&endTime, &startTime);
666 absolutetime_to_nanoseconds(endTime, &nsec);
667
668 if (!vars->fileRef) err = kIOReturnNoSpace;
669
670 IOLockLock(gFSLock);
671 if (kFSOpening != gFSState) err = kIOReturnTimeout;
672 IOLockUnlock(gFSLock);
673
674 HIBLOG("kern_open_file_for_direct_io(%d) took %qd ms\n", err, nsec / 1000000ULL);
675 if (kIOReturnSuccess != err) break;
676
677 if (kIOHibernateModeSSDInvert & gIOHibernateMode)
678 vars->flags ^= kIOHibernateOptionSSD;
679
680 HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx ssd %d\n", filename, ctx.size,
681 vars->block0, maxiobytes, kIOHibernateOptionSSD & vars->flags);
682 if (ctx.size < 1*1024*1024) // check against image size estimate!
683 {
684 err = kIOReturnNoSpace;
685 break;
686 }
687
688 vars->fileSize = ctx.size;
689 if (maxiobytes < vars->bufferSize) vars->bufferSize = maxiobytes;
690
691 vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
692
693 part = IOCopyMediaForDev(block_dev);
694 if (!part)
695 {
696 err = kIOReturnNotFound;
697 break;
698 }
699 err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false,
700 (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
701 if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID)
702 {
703 // IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
704 uuid_t volumeKeyUUID;
705 aks_volume_key_t vek;
706 static IOService * sKeyStore;
707 static const OSSymbol * sAKSGetKey;
708
709 if (!sAKSGetKey)
710 sAKSGetKey = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY);
711 if (!sKeyStore)
712 sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
713 if (sKeyStore)
714 err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
715 else
716 err = kIOReturnNoResources;
717 if (kIOReturnSuccess == err)
718 err = sKeyStore->callPlatformFunction(sAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
719 if (kIOReturnSuccess != err)
720 IOLog("volume key err 0x%x\n", err);
721 else
722 {
723 size_t bytes = (kIOHibernateAESKeySize / 8);
724 if (vek.key.keybytecount < bytes)
725 bytes = vek.key.keybytecount;
726 bcopy(&vek.key.keybytes[0], volumeCryptKey, bytes);
727 }
728 bzero(&vek, sizeof(vek));
729 }
730 part->release();
731
732 part = IOCopyMediaForDev(hibernate_image_dev);
733 if (!part)
734 {
735 err = kIOReturnNotFound;
736 break;
737 }
738
739 IORegistryEntry * next;
740 IORegistryEntry * child;
741 IOService * service;
742 OSData * data;
743
744 vars->pollers = OSArray::withCapacity(4);
745 if (!vars->pollers)
746 {
747 err = kIOReturnNoMemory;
748 break;
749 }
750
751 vars->blockSize = 512;
752 next = part;
753 do
754 {
755 IOPolledInterface * poller;
756 OSObject * obj;
757
758 obj = next->getProperty(kIOPolledInterfaceSupportKey);
759 if (kOSBooleanFalse == obj)
760 {
761 vars->pollers->flushCollection();
762 break;
763 }
764 else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
765 vars->pollers->setObject(poller);
766
767 if ((service = OSDynamicCast(IOService, next))
768 && service->getDeviceMemory()
769 && !vars->pollers->getCount()) break;
770
771 if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey))))
772 vars->blockSize = num->unsigned32BitValue();
773 child = next;
774 }
775 while ((next = child->getParentEntry(gIOServicePlane))
776 && child->isParent(next, gIOServicePlane, true));
777
778 if (vars->blockSize < 4096) vars->blockSize = 4096;
779
780 HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
781 major(hibernate_image_dev), minor(hibernate_image_dev), (long)vars->blockSize,
782 vars->pollers->getCount());
783
784 if (!vars->pollers->getCount())
785 {
786 err = kIOReturnUnsupported;
787 continue;
788 }
789 if (vars->blockSize < sizeof(IOHibernateImageHeader))
790 {
791 err = kIOReturnError;
792 continue;
793 }
794
795 err = IOHibernatePollerProbe(vars, (IOService *) part);
796 if (kIOReturnSuccess != err) break;
797
798 err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer);
799 if (kIOReturnSuccess != err) break;
800
801 vars->media = part;
802 next = part;
803 while (next)
804 {
805 next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue);
806 next = next->getParentEntry(gIOServicePlane);
807 }
808
809 *fileVars = vars;
810 *fileExtents = extentsData;
811
812 // make imagePath
813
814 if ((extentsData->getLength() >= sizeof(IOPolledFileExtent)))
815 {
816 char str2[24 + sizeof(uuid_string_t) + 2];
817
818 #if defined(__i386__) || defined(__x86_64__)
819 if (!gIOCreateEFIDevicePathSymbol)
820 gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
821
822 if (keyUUID)
823 snprintf(str2, sizeof(str2), "%qx:%s",
824 vars->extentMap[0].start, keyUUID->getCStringNoCopy());
825 else
826 snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
827
828 err = IOService::getPlatform()->callPlatformFunction(
829 gIOCreateEFIDevicePathSymbol, false,
830 (void *) part, (void *) str2,
831 (void *) (uintptr_t) true, (void *) &data);
832 #else
833 char str1[256];
834 int len = sizeof(str1);
835
836 if (!part->getPath(str1, &len, gIODTPlane))
837 err = kIOReturnNotFound;
838 else
839 {
840 snprintf(str2, sizeof(str2), ",%qx", vars->extentMap[0].start);
841 // (strip the plane name)
842 char * tail = strchr(str1, ':');
843 if (!tail)
844 tail = str1 - 1;
845 data = OSData::withBytes(tail + 1, strlen(tail + 1));
846 data->appendBytes(str2, strlen(str2));
847 }
848 #endif
849 if (kIOReturnSuccess == err)
850 *imagePath = data;
851 else
852 HIBLOG("error 0x%x getting path\n", err);
853 }
854 }
855 while (false);
856
857 if (kIOReturnSuccess != err)
858 {
859 HIBLOG("error 0x%x opening hibernation file\n", err);
860 if (vars->fileRef)
861 {
862 kern_close_file_for_direct_io(vars->fileRef, 0, 0, 0, 0, 0);
863 vars->fileRef = NULL;
864 }
865 }
866
867 if (part)
868 part->release();
869
870 return (err);
871 }
872
873 IOReturn
874 IOPolledFileClose( IOPolledFileIOVars * vars )
875 {
876 if (vars->pollers)
877 {
878 IOHibernatePollerClose(vars, kIOPolledPostflightState);
879 vars->pollers->release();
880 }
881
882 bzero(vars, sizeof(IOPolledFileIOVars));
883
884 return (kIOReturnSuccess);
885 }
886
887 static IOReturn
888 IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
889 {
890 IOPolledFileExtent * extentMap;
891
892 extentMap = vars->extentMap;
893
894 vars->position = position;
895
896 while (position >= extentMap->length)
897 {
898 position -= extentMap->length;
899 extentMap++;
900 }
901
902 vars->currentExtent = extentMap;
903 vars->extentRemaining = extentMap->length - position;
904 vars->extentPosition = vars->position - position;
905
906 if (vars->bufferSize <= vars->extentRemaining)
907 vars->bufferLimit = vars->bufferSize;
908 else
909 vars->bufferLimit = vars->extentRemaining;
910
911 return (kIOReturnSuccess);
912 }
913
914 static IOReturn
915 IOPolledFileWrite(IOPolledFileIOVars * vars,
916 const uint8_t * bytes, IOByteCount size,
917 hibernate_cryptvars_t * cryptvars)
918 {
919 IOReturn err = kIOReturnSuccess;
920 IOByteCount copy;
921 bool flush = false;
922
923 do
924 {
925 if (!bytes && !size)
926 {
927 // seek to end of block & flush
928 size = vars->position & (vars->blockSize - 1);
929 if (size)
930 size = vars->blockSize - size;
931 flush = true;
932 // use some garbage for the fill
933 bytes = vars->buffer + vars->bufferOffset;
934 }
935
936 copy = vars->bufferLimit - vars->bufferOffset;
937 if (copy > size)
938 copy = size;
939 else
940 flush = true;
941
942 if (bytes)
943 {
944 bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
945 bytes += copy;
946 }
947 else
948 bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
949
950 size -= copy;
951 vars->bufferOffset += copy;
952 vars->position += copy;
953
954 if (flush && vars->bufferOffset)
955 {
956 uint64_t offset = (vars->position - vars->bufferOffset
957 - vars->extentPosition + vars->currentExtent->start);
958 uint32_t length = (vars->bufferOffset);
959
960 #if CRYPTO
961 if (cryptvars && vars->encryptStart
962 && (vars->position > vars->encryptStart)
963 && ((vars->position - length) < vars->encryptEnd))
964 {
965 AbsoluteTime startTime, endTime;
966
967 uint64_t encryptLen, encryptStart;
968 encryptLen = vars->position - vars->encryptStart;
969 if (encryptLen > length)
970 encryptLen = length;
971 encryptStart = length - encryptLen;
972 if (vars->position > vars->encryptEnd)
973 encryptLen -= (vars->position - vars->encryptEnd);
974
975 clock_get_uptime(&startTime);
976
977 // encrypt the buffer
978 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
979 &cryptvars->aes_iv[0],
980 encryptLen / AES_BLOCK_SIZE,
981 vars->buffer + vars->bufferHalf + encryptStart,
982 &cryptvars->ctx.encrypt);
983
984 clock_get_uptime(&endTime);
985 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
986 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
987 vars->cryptBytes += encryptLen;
988
989 // save initial vector for following encrypts
990 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
991 &cryptvars->aes_iv[0],
992 AES_BLOCK_SIZE);
993 }
994 #endif /* CRYPTO */
995
996 if (vars->io)
997 {
998 err = IOHibernatePollerIODone(vars, true);
999 if (kIOReturnSuccess != err)
1000 break;
1001 }
1002
1003 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
1004 //if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
1005
1006 err = IOHibernatePollerIO(vars, kIOPolledWrite, vars->bufferHalf, offset, length);
1007 if (kIOReturnSuccess != err)
1008 break;
1009 vars->io = true;
1010
1011 vars->extentRemaining -= vars->bufferOffset;
1012 if (!vars->extentRemaining)
1013 {
1014 vars->currentExtent++;
1015 vars->extentRemaining = vars->currentExtent->length;
1016 vars->extentPosition = vars->position;
1017 if (!vars->extentRemaining)
1018 {
1019 err = kIOReturnOverrun;
1020 break;
1021 }
1022 }
1023
1024 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1025 vars->bufferOffset = 0;
1026 if (vars->bufferSize <= vars->extentRemaining)
1027 vars->bufferLimit = vars->bufferSize;
1028 else
1029 vars->bufferLimit = vars->extentRemaining;
1030
1031 flush = false;
1032 }
1033 }
1034 while (size);
1035
1036 return (err);
1037 }
1038
1039 static IOReturn
1040 IOPolledFileRead(IOPolledFileIOVars * vars,
1041 uint8_t * bytes, IOByteCount size,
1042 hibernate_cryptvars_t * cryptvars)
1043 {
1044 IOReturn err = kIOReturnSuccess;
1045 IOByteCount copy;
1046
1047 // bytesWritten += size;
1048
1049 do
1050 {
1051 copy = vars->bufferLimit - vars->bufferOffset;
1052 if (copy > size)
1053 copy = size;
1054
1055 if (bytes)
1056 {
1057 bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
1058 bytes += copy;
1059 }
1060 size -= copy;
1061 vars->bufferOffset += copy;
1062 // vars->position += copy;
1063
1064 if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd))
1065 {
1066 if (vars->io)
1067 {
1068 err = IOHibernatePollerIODone(vars, false);
1069 if (kIOReturnSuccess != err)
1070 break;
1071 }
1072 else
1073 cryptvars = 0;
1074
1075 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
1076
1077 vars->position += vars->lastRead;
1078 vars->extentRemaining -= vars->lastRead;
1079 vars->bufferLimit = vars->lastRead;
1080
1081 if (!vars->extentRemaining)
1082 {
1083 vars->currentExtent++;
1084 vars->extentRemaining = vars->currentExtent->length;
1085 vars->extentPosition = vars->position;
1086 if (!vars->extentRemaining)
1087 {
1088 err = kIOReturnOverrun;
1089 break;
1090 }
1091 }
1092
1093 uint64_t length;
1094 uint64_t lastReadLength = vars->lastRead;
1095 uint64_t offset = (vars->position
1096 - vars->extentPosition + vars->currentExtent->start);
1097 if (vars->extentRemaining <= vars->bufferSize)
1098 length = vars->extentRemaining;
1099 else
1100 length = vars->bufferSize;
1101 if ((length + vars->position) > vars->readEnd)
1102 length = vars->readEnd - vars->position;
1103
1104 vars->lastRead = length;
1105 if (length)
1106 {
1107 //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
1108 err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
1109 if (kIOReturnSuccess != err)
1110 break;
1111 vars->io = true;
1112 }
1113
1114 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1115 vars->bufferOffset = 0;
1116
1117 #if CRYPTO
1118 if (cryptvars)
1119 {
1120 uint8_t thisVector[AES_BLOCK_SIZE];
1121 AbsoluteTime startTime, endTime;
1122
1123 // save initial vector for following decrypts
1124 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
1125 bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
1126 &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1127
1128 // decrypt the buffer
1129 clock_get_uptime(&startTime);
1130
1131 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
1132 &thisVector[0],
1133 lastReadLength / AES_BLOCK_SIZE,
1134 vars->buffer + vars->bufferHalf,
1135 &cryptvars->ctx.decrypt);
1136
1137 clock_get_uptime(&endTime);
1138 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
1139 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
1140 vars->cryptBytes += lastReadLength;
1141 }
1142 #endif /* CRYPTO */
1143 }
1144 }
1145 while (size);
1146
1147 return (err);
1148 }
1149
1150 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1151
1152 IOReturn
1153 IOHibernateSystemSleep(void)
1154 {
1155 IOReturn err;
1156 OSData * data;
1157 OSObject * obj;
1158 OSString * str;
1159 OSNumber * num;
1160 bool dsSSD, vmflush;
1161 IOHibernateVars * vars;
1162
1163 gIOHibernateState = kIOHibernateStateInactive;
1164
1165 if (!gIOChosenEntry)
1166 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1167
1168 gIOHibernateDebugFlags = 0;
1169 if (kIOLogHibernate & gIOKitDebug)
1170 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
1171
1172 if (IOService::getPMRootDomain()->getHibernateSettings(
1173 &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime))
1174 {
1175 if (kIOHibernateModeSleep & gIOHibernateMode)
1176 // default to discard clean for safe sleep
1177 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
1178 | kIOHibernateModeDiscardCleanActive);
1179 }
1180
1181 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
1182 {
1183 if ((str = OSDynamicCast(OSString, obj)))
1184 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
1185 sizeof(gIOHibernateFilename));
1186 obj->release();
1187 }
1188
1189 if (!gIOHibernateMode || !gIOHibernateFilename[0])
1190 return (kIOReturnUnsupported);
1191
1192 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
1193
1194 vars = IONew(IOHibernateVars, 1);
1195 if (!vars) return (kIOReturnNoMemory);
1196 bzero(vars, sizeof(*vars));
1197
1198 IOLockLock(gFSLock);
1199 if (kFSIdle != gFSState)
1200 {
1201 HIBLOG("hibernate file busy\n");
1202 IOLockUnlock(gFSLock);
1203 IODelete(vars, IOHibernateVars, 1);
1204 return (kIOReturnBusy);
1205 }
1206 gFSState = kFSOpening;
1207 IOLockUnlock(gFSLock);
1208
1209 do
1210 {
1211 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1212 2 * page_size + WKdm_SCRATCH_BUF_SIZE, page_size);
1213 vars->ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1214 2 * kDefaultIOSize, page_size);
1215
1216 vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1217 ptoa_64(gIOHibernateHandoffPageCount), page_size);
1218
1219 if (!vars->srcBuffer || !vars->ioBuffer || !vars->handoffBuffer)
1220 {
1221 err = kIOReturnNoMemory;
1222 break;
1223 }
1224
1225 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey)))
1226 {
1227 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMinSize = num->unsigned64BitValue();
1228 obj->release();
1229 }
1230 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey)))
1231 {
1232 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMaxSize = num->unsigned64BitValue();
1233 obj->release();
1234 }
1235
1236 boolean_t encryptedswap = true;
1237 uint32_t pageCount;
1238 AbsoluteTime startTime, endTime;
1239 uint64_t nsec;
1240
1241 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1242 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
1243 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1244
1245 vmflush = (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey));
1246 uint64_t setFileSize = 0;
1247 err = hibernate_alloc_page_lists(&vars->page_list,
1248 &vars->page_list_wired,
1249 &vars->page_list_pal);
1250 if (KERN_SUCCESS != err)
1251 break;
1252
1253 if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode))
1254 {
1255 hibernate_page_list_setall(vars->page_list,
1256 vars->page_list_wired,
1257 vars->page_list_pal,
1258 true /* preflight */,
1259 vmflush /* discard */,
1260 &pageCount);
1261 PE_Video consoleInfo;
1262 bzero(&consoleInfo, sizeof(consoleInfo));
1263 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1264
1265 // estimate: 6% increase in pages compressed
1266 // screen preview 2 images compressed 50%
1267 setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
1268 + vars->page_list->list_size
1269 + (consoleInfo.v_width * consoleInfo.v_height * 8);
1270 enum { setFileRound = 1024*1024ULL };
1271 setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
1272
1273 HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
1274 pageCount, (100ULL * gIOHibernateCompression) >> 8,
1275 setFileSize, vars->fileMinSize);
1276
1277 if (!(kIOHibernateModeFileResize & gIOHibernateMode)
1278 && (setFileSize < vars->fileMinSize))
1279 {
1280 setFileSize = vars->fileMinSize;
1281 }
1282 }
1283
1284 // open & invalidate the image file
1285
1286 err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, vars->ioBuffer,
1287 &vars->fileVars, &vars->fileExtents, &data,
1288 &vars->volumeCryptKey[0]);
1289
1290 if (KERN_SUCCESS != err)
1291 {
1292 HIBLOG("IOPolledFileOpen(%x)\n", err);
1293 break;
1294 }
1295
1296 clock_get_uptime(&startTime);
1297 err = hibernate_setup(gIOHibernateCurrentHeader,
1298 gIOHibernateFreeRatio, gIOHibernateFreeTime,
1299 vmflush,
1300 vars->page_list, vars->page_list_wired, vars->page_list_pal);
1301 clock_get_uptime(&endTime);
1302 SUB_ABSOLUTETIME(&endTime, &startTime);
1303 absolutetime_to_nanoseconds(endTime, &nsec);
1304 HIBLOG("hibernate_setup(%d) took %qd ms\n", err, nsec / 1000000ULL);
1305
1306 dsSSD = ((0 != (kIOHibernateOptionSSD & vars->fileVars->flags))
1307 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
1308 if (dsSSD)
1309 {
1310 gIOHibernateCurrentHeader->options |=
1311 kIOHibernateOptionSSD
1312 | kIOHibernateOptionColor;
1313
1314 #if defined(__i386__) || defined(__x86_64__)
1315 if (!uuid_is_null(vars->volumeCryptKey) &&
1316 (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey)))
1317 {
1318 uintptr_t smcVars[2];
1319 smcVars[0] = sizeof(vars->volumeCryptKey);
1320 smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
1321
1322 IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
1323 bzero(smcVars, sizeof(smcVars));
1324 }
1325 #endif
1326 }
1327 else
1328 {
1329 gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
1330 }
1331
1332
1333 if (KERN_SUCCESS != err)
1334 break;
1335
1336 if (encryptedswap || !uuid_is_null(vars->volumeCryptKey))
1337 gIOHibernateMode ^= kIOHibernateModeEncrypt;
1338
1339 if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1340 {
1341 vars->videoAllocSize = kVideoMapSize;
1342 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
1343 vars->videoMapping = 0;
1344 }
1345
1346 // generate crypt keys
1347 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
1348 vars->wiredCryptKey[i] = random();
1349 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
1350 vars->cryptKey[i] = random();
1351
1352 // set nvram
1353
1354 IORegistryEntry * regEntry;
1355 if (!gIOOptionsEntry)
1356 {
1357 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
1358 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
1359 if (regEntry && !gIOOptionsEntry)
1360 regEntry->release();
1361 }
1362
1363 if (gIOOptionsEntry)
1364 {
1365 const OSSymbol * sym;
1366
1367 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1368 if (sym)
1369 {
1370 gIOOptionsEntry->setProperty(sym, data);
1371 sym->release();
1372 }
1373 data->release();
1374
1375 #if defined(__i386__) || defined(__x86_64__)
1376 struct AppleRTCHibernateVars
1377 {
1378 uint8_t signature[4];
1379 uint32_t revision;
1380 uint8_t booterSignature[20];
1381 uint8_t wiredCryptKey[16];
1382 };
1383 AppleRTCHibernateVars rtcVars;
1384
1385 rtcVars.signature[0] = 'A';
1386 rtcVars.signature[1] = 'A';
1387 rtcVars.signature[2] = 'P';
1388 rtcVars.signature[3] = 'L';
1389 rtcVars.revision = 1;
1390 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
1391 if (gIOHibernateBootSignature[0])
1392 {
1393 char c;
1394 uint8_t value = 0;
1395 for (uint32_t i = 0;
1396 (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
1397 i++)
1398 {
1399 if (c >= 'a')
1400 c -= 'a' - 10;
1401 else if (c >= 'A')
1402 c -= 'A' - 10;
1403 else if (c >= '0')
1404 c -= '0';
1405 else
1406 continue;
1407 value = (value << 4) | c;
1408 if (i & 1)
1409 rtcVars.booterSignature[i >> 1] = value;
1410 }
1411 }
1412 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
1413 if (data)
1414 {
1415 if (!gIOHibernateRTCVariablesKey)
1416 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1417 if (gIOHibernateRTCVariablesKey)
1418 IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
1419
1420 if( gIOOptionsEntry )
1421 {
1422 if( gIOHibernateMode & kIOHibernateModeSwitch )
1423 {
1424 const OSSymbol *sym;
1425 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSwitchVarsKey);
1426 if( sym )
1427 {
1428 gIOOptionsEntry->setProperty(sym, data); /* intentional insecure backup of rtc boot vars */
1429 sym->release();
1430 }
1431 }
1432 }
1433
1434 data->release();
1435 }
1436 if (gIOChosenEntry)
1437 {
1438 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
1439 if (data)
1440 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
1441 {
1442 // set BootNext
1443
1444 if (!gIOHibernateBoot0082Data)
1445 {
1446 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
1447 if (data)
1448 {
1449 // AppleNVRAM_EFI_LOAD_OPTION
1450 struct {
1451 uint32_t Attributes;
1452 uint16_t FilePathLength;
1453 uint16_t Desc;
1454 } loadOptionHeader;
1455 loadOptionHeader.Attributes = 1;
1456 loadOptionHeader.FilePathLength = data->getLength();
1457 loadOptionHeader.Desc = 0;
1458 gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
1459 if (gIOHibernateBoot0082Data)
1460 {
1461 gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
1462 gIOHibernateBoot0082Data->appendBytes(data);
1463 }
1464 }
1465 }
1466 if (!gIOHibernateBoot0082Key)
1467 gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
1468 if (!gIOHibernateBootNextKey)
1469 gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
1470 if (!gIOHibernateBootNextData)
1471 {
1472 uint16_t bits = 0x0082;
1473 gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
1474 }
1475 if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData)
1476 {
1477 gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
1478 gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
1479 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
1480 }
1481 }
1482 }
1483 #else /* !i386 && !x86_64 */
1484 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1485 {
1486 data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1487 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
1488 if (sym && data)
1489 gIOOptionsEntry->setProperty(sym, data);
1490 if (sym)
1491 sym->release();
1492 if (data)
1493 data->release();
1494 if (false && gIOHibernateBootSignature[0])
1495 {
1496 data = OSData::withCapacity(16);
1497 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
1498 if (sym && data)
1499 {
1500 char c;
1501 uint8_t value = 0;
1502 for (uint32_t i = 0; (c = gIOHibernateBootSignature[i]); i++)
1503 {
1504 if (c >= 'a')
1505 c -= 'a' - 10;
1506 else if (c >= 'A')
1507 c -= 'A' - 10;
1508 else if (c >= '0')
1509 c -= '0';
1510 else
1511 continue;
1512 value = (value << 4) | c;
1513 if (i & 1)
1514 data->appendBytes(&value, sizeof(value));
1515 }
1516 gIOOptionsEntry->setProperty(sym, data);
1517 }
1518 if (sym)
1519 sym->release();
1520 if (data)
1521 data->release();
1522 }
1523 }
1524 if (!vars->haveFastBoot)
1525 {
1526 // set boot volume to zero
1527 IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
1528 if (platform && (kIOReturnSuccess == platform->readXPRAM(kXPRamAudioVolume,
1529 &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume))))
1530 {
1531 uint8_t newVolume;
1532 newVolume = vars->saveBootAudioVolume & 0xf8;
1533 platform->writeXPRAM(kXPRamAudioVolume,
1534 &newVolume, sizeof(newVolume));
1535 }
1536 }
1537 #endif /* !i386 && !x86_64 */
1538 }
1539 // --
1540
1541 }
1542 while (false);
1543
1544 IOLockLock(gFSLock);
1545 if ((kIOReturnSuccess == err) && (kFSOpening == gFSState))
1546 {
1547 gFSState = kFSOpened;
1548 gIOHibernateVars = *vars;
1549 gFileVars = *vars->fileVars;
1550 gIOHibernateVars.fileVars = &gFileVars;
1551 gIOHibernateFileRef = gFileVars.fileRef;
1552 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
1553 gIOHibernateState = kIOHibernateStateHibernating;
1554 }
1555 else
1556 {
1557 HIBLOG("hibernate file close due timeout\n");
1558 if (vars->fileVars && vars->fileVars->fileRef) kern_close_file_for_direct_io(vars->fileVars->fileRef, 0, 0, 0, 0, 0);
1559 IOHibernateDone(vars);
1560 gFSState = kFSIdle;
1561 }
1562 IOLockUnlock(gFSLock);
1563
1564 if (vars->fileVars) IODelete(vars->fileVars, IOPolledFileIOVars, 1);
1565 IODelete(vars, IOHibernateVars, 1);
1566
1567 return (err);
1568 }
1569
1570 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1571
1572 DECLARE_IOHIBERNATEPROGRESSALPHA
1573
1574 static void
1575 ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
1576 {
1577 uint32_t rowBytes, pixelShift;
1578 uint32_t x, y;
1579 int32_t blob;
1580 uint32_t alpha, in, color, result;
1581 uint8_t * out;
1582 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1583
1584 rowBytes = display->rowBytes;
1585 pixelShift = display->depth >> 4;
1586 if (pixelShift < 1) return;
1587
1588 screen += ((display->width
1589 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1590 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1591
1592 for (y = 0; y < kIOHibernateProgressHeight; y++)
1593 {
1594 out = screen + y * rowBytes;
1595 for (blob = 0; blob < kIOHibernateProgressCount; blob++)
1596 {
1597 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
1598 for (x = 0; x < kIOHibernateProgressWidth; x++)
1599 {
1600 alpha = gIOHibernateProgressAlpha[y][x];
1601 result = color;
1602 if (alpha)
1603 {
1604 if (0xff != alpha)
1605 {
1606 if (1 == pixelShift)
1607 {
1608 in = *((uint16_t *)out) & 0x1f; // 16
1609 in = (in << 3) | (in >> 2);
1610 }
1611 else
1612 in = *((uint32_t *)out) & 0xff; // 32
1613 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
1614 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
1615 }
1616 if (1 == pixelShift)
1617 {
1618 result >>= 3;
1619 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1620 }
1621 else
1622 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1623 }
1624 out += (1 << pixelShift);
1625 }
1626 out += (kIOHibernateProgressSpacing << pixelShift);
1627 }
1628 }
1629 }
1630
1631
1632 static void
1633 ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
1634 {
1635 uint32_t rowBytes, pixelShift;
1636 uint32_t x, y;
1637 int32_t blob, lastBlob;
1638 uint32_t alpha, in, color, result;
1639 uint8_t * out;
1640 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1641
1642 pixelShift = display->depth >> 4;
1643 if (pixelShift < 1)
1644 return;
1645
1646 rowBytes = display->rowBytes;
1647
1648 screen += ((display->width
1649 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1650 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1651
1652 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
1653
1654 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
1655
1656 for (y = 0; y < kIOHibernateProgressHeight; y++)
1657 {
1658 out = screen + y * rowBytes;
1659 for (blob = firstBlob; blob <= lastBlob; blob++)
1660 {
1661 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
1662 for (x = 0; x < kIOHibernateProgressWidth; x++)
1663 {
1664 alpha = gIOHibernateProgressAlpha[y][x];
1665 result = color;
1666 if (alpha)
1667 {
1668 if (0xff != alpha)
1669 {
1670 in = display->progressSaveUnder[blob][saveindex[blob]++];
1671 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
1672 }
1673 if (1 == pixelShift)
1674 {
1675 result >>= 3;
1676 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1677 }
1678 else
1679 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1680 }
1681 out += (1 << pixelShift);
1682 }
1683 out += (kIOHibernateProgressSpacing << pixelShift);
1684 }
1685 }
1686 }
1687
1688 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1689
1690 IOReturn
1691 IOHibernateIOKitSleep(void)
1692 {
1693 IOReturn ret = kIOReturnSuccess;
1694 IOLockLock(gFSLock);
1695 if (kFSOpening == gFSState)
1696 {
1697 gFSState = kFSTimedOut;
1698 HIBLOG("hibernate file open timed out\n");
1699 ret = kIOReturnTimeout;
1700 }
1701 IOLockUnlock(gFSLock);
1702 return (ret);
1703 }
1704
1705 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1706
1707 IOReturn
1708 IOHibernateSystemHasSlept(void)
1709 {
1710 IOReturn ret = kIOReturnSuccess;
1711 IOHibernateVars * vars = &gIOHibernateVars;
1712 OSObject * obj = 0;
1713 OSData * data;
1714
1715 IOLockLock(gFSLock);
1716 if ((kFSOpened != gFSState) && gIOHibernateMode)
1717 {
1718 ret = kIOReturnTimeout;
1719 }
1720 IOLockUnlock(gFSLock);
1721 if (kIOReturnSuccess != ret) return (ret);
1722
1723 if (gIOHibernateMode) obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
1724 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1725 if (obj && !vars->previewBuffer)
1726 obj->release();
1727
1728 vars->consoleMapping = NULL;
1729 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
1730 {
1731 vars->previewBuffer->release();
1732 vars->previewBuffer = 0;
1733 }
1734
1735 if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1736 && vars->previewBuffer
1737 && (data = OSDynamicCast(OSData,
1738 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
1739 {
1740 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
1741 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
1742
1743 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1744
1745 if (kIOHibernatePreviewUpdates & flags)
1746 {
1747 PE_Video consoleInfo;
1748 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1749
1750 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1751
1752 graphicsInfo->width = consoleInfo.v_width;
1753 graphicsInfo->height = consoleInfo.v_height;
1754 graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
1755 graphicsInfo->depth = consoleInfo.v_depth;
1756 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1757
1758 HIBPRINT("video %p %d %d %d\n",
1759 vars->consoleMapping, graphicsInfo->depth,
1760 graphicsInfo->width, graphicsInfo->height);
1761 if (vars->consoleMapping)
1762 ProgressInit(graphicsInfo, vars->consoleMapping,
1763 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1764 }
1765 }
1766
1767 if (gIOOptionsEntry)
1768 gIOOptionsEntry->sync();
1769
1770 return (ret);
1771 }
1772
1773 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1774
1775 static DeviceTreeNode *
1776 MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry)
1777 {
1778 DeviceTreeNodeProperty * prop;
1779 DeviceTreeNode * child;
1780 IORegistryEntry * childRegEntry;
1781 const char * nameProp;
1782 unsigned int propLen, idx;
1783
1784 prop = (DeviceTreeNodeProperty *) (entry + 1);
1785 for (idx = 0; idx < entry->nProperties; idx++)
1786 {
1787 if (regEntry && (0 != strcmp("name", prop->name)))
1788 {
1789 regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
1790 // HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
1791 }
1792 prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
1793 }
1794
1795 child = (DeviceTreeNode *) prop;
1796 for (idx = 0; idx < entry->nChildren; idx++)
1797 {
1798 if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen))
1799 panic("no name");
1800 childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
1801 // HIBPRINT("%s == %p\n", nameProp, childRegEntry);
1802 child = MergeDeviceTree(child, childRegEntry);
1803 }
1804 return (child);
1805 }
1806
1807 IOReturn
1808 IOHibernateSystemWake(void)
1809 {
1810 if (kFSOpened == gFSState)
1811 {
1812 IOHibernateDone(&gIOHibernateVars);
1813 }
1814 else
1815 {
1816 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1817 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1818 }
1819 return (kIOReturnSuccess);
1820 }
1821
1822 static IOReturn
1823 IOHibernateDone(IOHibernateVars * vars)
1824 {
1825 IORegistryEntry * next;
1826
1827 hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
1828
1829 if (vars->videoMapping)
1830 {
1831 if (vars->videoMapSize)
1832 // remove mappings
1833 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1834 if (vars->videoAllocSize)
1835 // dealloc range
1836 kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
1837 }
1838
1839 if (vars->previewBuffer)
1840 {
1841 vars->previewBuffer->release();
1842 vars->previewBuffer = 0;
1843 }
1844
1845 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1846 {
1847 IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
1848 gIOHibernateCurrentHeader->options, 32);
1849 }
1850 else
1851 {
1852 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1853 }
1854
1855 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1856 && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus))
1857 {
1858 IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
1859 &gIOHibernateGraphicsInfo->gfxStatus,
1860 sizeof(gIOHibernateGraphicsInfo->gfxStatus));
1861 }
1862 else
1863 {
1864 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1865 }
1866
1867 if (vars->fileVars)
1868 {
1869 if ((next = vars->fileVars->media)) do
1870 {
1871 next->removeProperty(kIOPolledInterfaceActiveKey);
1872 next = next->getParentEntry(gIOServicePlane);
1873 }
1874 while (next);
1875 IOPolledFileClose(vars->fileVars);
1876 }
1877
1878 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1879
1880 #if defined(__i386__) || defined(__x86_64__)
1881 IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
1882 IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
1883
1884 /*
1885 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1886 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1887 */
1888 if (gIOOptionsEntry) {
1889
1890 if (gIOHibernateRTCVariablesKey) {
1891 if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
1892 gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
1893 }
1894 }
1895
1896 if (gIOHibernateBootNextKey)
1897 {
1898 if (gIOHibernateBootNextSave)
1899 {
1900 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
1901 gIOHibernateBootNextSave->release();
1902 gIOHibernateBootNextSave = NULL;
1903 }
1904 else
1905 gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
1906 }
1907 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) gIOOptionsEntry->sync();
1908 }
1909 #endif
1910
1911 if (vars->srcBuffer)
1912 vars->srcBuffer->release();
1913 if (vars->ioBuffer)
1914 vars->ioBuffer->release();
1915 bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
1916 if (vars->handoffBuffer)
1917 {
1918 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1919 {
1920 IOHibernateHandoff * handoff;
1921 bool done = false;
1922 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
1923 !done;
1924 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
1925 {
1926 HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
1927 uint8_t * data = &handoff->data[0];
1928 switch (handoff->type)
1929 {
1930 case kIOHibernateHandoffTypeEnd:
1931 done = true;
1932 break;
1933
1934 case kIOHibernateHandoffTypeDeviceTree:
1935 MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
1936 break;
1937
1938 case kIOHibernateHandoffTypeKeyStore:
1939 #if defined(__i386__) || defined(__x86_64__)
1940 {
1941 IOBufferMemoryDescriptor *
1942 md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
1943 if (md)
1944 {
1945 IOSetKeyStoreData(md);
1946 }
1947 }
1948 #endif
1949 break;
1950
1951 default:
1952 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
1953 break;
1954 }
1955 }
1956 }
1957 vars->handoffBuffer->release();
1958 }
1959 if (vars->fileExtents)
1960 vars->fileExtents->release();
1961
1962 bzero(vars, sizeof(*vars));
1963
1964 // gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1965
1966 return (kIOReturnSuccess);
1967 }
1968
1969 IOReturn
1970 IOHibernateSystemPostWake(void)
1971 {
1972 struct kern_direct_file_io_ref_t * fileRef;
1973
1974 if (kFSOpened == gFSState)
1975 {
1976 // invalidate & close the image file
1977 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1978 if ((fileRef = gIOHibernateFileRef))
1979 {
1980 gIOHibernateFileRef = 0;
1981 IOSleep(TRIM_DELAY);
1982 kern_close_file_for_direct_io(fileRef,
1983 #if DISABLE_TRIM
1984 0, 0, 0, 0, 0);
1985 #else
1986 0, (caddr_t) gIOHibernateCurrentHeader,
1987 sizeof(IOHibernateImageHeader),
1988 0,
1989 gIOHibernateCurrentHeader->imageSize);
1990 #endif
1991 }
1992 gFSState = kFSIdle;
1993 }
1994 return (kIOReturnSuccess);
1995 }
1996
1997 bool IOHibernateWasScreenLocked(void)
1998 {
1999 bool ret = false;
2000 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry)
2001 {
2002 OSData *
2003 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
2004 if (data) switch (*((uint32_t *)data->getBytesNoCopy()))
2005 {
2006 case kIOScreenLockLocked:
2007 case kIOScreenLockFileVaultDialog:
2008 ret = true;
2009 break;
2010 case kIOScreenLockNoLock:
2011 case kIOScreenLockUnlocked:
2012 default:
2013 ret = false;
2014 break;
2015 }
2016 }
2017 return (ret);
2018 }
2019
2020 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2021
2022 SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
2023 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2024 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
2025 SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
2026 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2027 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
2028 SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
2029 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2030 &gIOHibernateMode, 0, "");
2031 SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
2032 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2033 gIOHibernateStats, hibernate_statistics_t, "");
2034
2035 SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
2036 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
2037 &gIOHibernateStats->graphicsReadyTime, 0, "");
2038 SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
2039 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
2040 &gIOHibernateStats->wakeNotificationTime, 0, "");
2041 SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
2042 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
2043 &gIOHibernateStats->lockScreenReadyTime, 0, "");
2044 SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
2045 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
2046 &gIOHibernateStats->hidReadyTime, 0, "");
2047
2048
2049 void
2050 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
2051 {
2052 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
2053 if (data)
2054 {
2055 rootDomain->setProperty(kIOHibernateStateKey, data);
2056 data->release();
2057 }
2058
2059 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))
2060 gIOHibernateMode = kIOHibernateModeOn;
2061 else
2062 gIOHibernateFilename[0] = 0;
2063
2064 sysctl_register_oid(&sysctl__kern_hibernatefile);
2065 sysctl_register_oid(&sysctl__kern_bootsignature);
2066 sysctl_register_oid(&sysctl__kern_hibernatemode);
2067 sysctl_register_oid(&sysctl__kern_hibernatestatistics);
2068 sysctl_register_oid(&sysctl__kern_hibernategraphicsready);
2069 sysctl_register_oid(&sysctl__kern_hibernatewakenotification);
2070 sysctl_register_oid(&sysctl__kern_hibernatelockscreenready);
2071 sysctl_register_oid(&sysctl__kern_hibernatehidready);
2072
2073 gFSLock = IOLockAlloc();
2074 }
2075
2076
2077 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2078
2079 static void
2080 hibernate_setup_for_wake(void)
2081 {
2082 }
2083
2084 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2085
2086 #define C_ASSERT(e) typedef char __C_ASSERT__[(e) ? 1 : -1]
2087
2088 static bool
2089 no_encrypt_page(vm_offset_t ppnum)
2090 {
2091 if (pmap_is_noencrypt((ppnum_t)ppnum) == TRUE)
2092 {
2093 return true;
2094 }
2095 return false;
2096 }
2097
2098 static void
2099 hibernate_pal_callback(void *vars_arg, vm_offset_t addr)
2100 {
2101 IOHibernateVars *vars = (IOHibernateVars *)vars_arg;
2102 /* Make sure it's not in either of the save lists */
2103 hibernate_set_page_state(vars->page_list, vars->page_list_wired, atop_64(addr), 1, kIOHibernatePageStateFree);
2104
2105 /* Set it in the bitmap of pages owned by the PAL */
2106 hibernate_page_bitset(vars->page_list_pal, TRUE, atop_64(addr));
2107 }
2108
2109 static struct hibernate_cryptvars_t *local_cryptvars;
2110
2111 extern "C" int
2112 hibernate_pal_write(void *buffer, size_t size)
2113 {
2114 IOHibernateVars * vars = &gIOHibernateVars;
2115
2116 IOReturn err = IOPolledFileWrite(vars->fileVars, (const uint8_t *)buffer, size, local_cryptvars);
2117 if (kIOReturnSuccess != err) {
2118 kprintf("epic hibernate fail! %d\n", err);
2119 return err;
2120 }
2121
2122 return 0;
2123 }
2124
2125
2126 extern "C" uint32_t
2127 hibernate_write_image(void)
2128 {
2129 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
2130 IOHibernateVars * vars = &gIOHibernateVars;
2131 IOPolledFileExtent * fileExtents;
2132
2133 C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
2134
2135 uint32_t pageCount, pagesDone;
2136 IOReturn err;
2137 vm_offset_t ppnum, page;
2138 IOItemCount count;
2139 uint8_t * src;
2140 uint8_t * data;
2141 uint8_t * compressed;
2142 uint8_t * scratch;
2143 void * zerosCompressed;
2144 IOByteCount pageCompressedSize, zerosCompressedLen;
2145 uint64_t compressedSize, uncompressedSize;
2146 uint64_t image1Size = 0;
2147 uint32_t bitmap_size;
2148 bool iterDone, pollerOpen, needEncrypt;
2149 uint32_t restore1Sum, sum, sum1, sum2;
2150 int wkresult;
2151 uint32_t tag;
2152 uint32_t pageType;
2153 uint32_t pageAndCount[2];
2154 addr64_t phys64;
2155 IOByteCount segLen;
2156
2157 AbsoluteTime startTime, endTime;
2158 AbsoluteTime allTime, compTime;
2159 uint64_t compBytes;
2160 uint64_t nsec;
2161 uint32_t lastProgressStamp = 0;
2162 uint32_t progressStamp;
2163 uint32_t blob, lastBlob = (uint32_t) -1L;
2164
2165 uint32_t wiredPagesEncrypted;
2166 uint32_t dirtyPagesEncrypted;
2167 uint32_t wiredPagesClear;
2168 uint32_t zeroPageCount;
2169
2170 hibernate_cryptvars_t _cryptvars;
2171 hibernate_cryptvars_t * cryptvars = 0;
2172
2173 wiredPagesEncrypted = 0;
2174 dirtyPagesEncrypted = 0;
2175 wiredPagesClear = 0;
2176 zeroPageCount = 0;
2177
2178 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
2179 return (false /* sleep */ );
2180
2181 if (kIOHibernateModeSleep & gIOHibernateMode)
2182 kdebug_enable = save_kdebug_enable;
2183
2184 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START, 0, 0, 0, 0, 0);
2185 IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
2186
2187 restore1Sum = sum1 = sum2 = 0;
2188
2189 hibernate_pal_prepare();
2190
2191 #if CRYPTO
2192 // encryption data. "iv" is the "initial vector".
2193 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2194 {
2195 static const unsigned char first_iv[AES_BLOCK_SIZE]
2196 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
2197 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
2198
2199 cryptvars = &gIOHibernateCryptWakeContext;
2200 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
2201 aes_encrypt_key(vars->cryptKey,
2202 kIOHibernateAESKeySize,
2203 &cryptvars->ctx.encrypt);
2204 aes_decrypt_key(vars->cryptKey,
2205 kIOHibernateAESKeySize,
2206 &cryptvars->ctx.decrypt);
2207
2208 cryptvars = &_cryptvars;
2209 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
2210 for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++)
2211 vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
2212 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
2213 aes_encrypt_key(vars->wiredCryptKey,
2214 kIOHibernateAESKeySize,
2215 &cryptvars->ctx.encrypt);
2216
2217 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
2218 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
2219 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
2220
2221 local_cryptvars = cryptvars;
2222 }
2223 #endif /* CRYPTO */
2224
2225 hibernate_setup_for_wake();
2226
2227 hibernate_page_list_setall(vars->page_list,
2228 vars->page_list_wired,
2229 vars->page_list_pal,
2230 false /* !preflight */,
2231 /* discard_all */
2232 ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
2233 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
2234 &pageCount);
2235
2236 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
2237
2238 fileExtents = (IOPolledFileExtent *) vars->fileExtents->getBytesNoCopy();
2239
2240 #if 0
2241 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
2242 for (page = 0; page < count; page++)
2243 {
2244 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
2245 fileExtents[page].start, fileExtents[page].length,
2246 fileExtents[page].start + fileExtents[page].length);
2247 }
2248 #endif
2249
2250 needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
2251 AbsoluteTime_to_scalar(&compTime) = 0;
2252 compBytes = 0;
2253
2254 clock_get_uptime(&allTime);
2255 IOService::getPMRootDomain()->pmStatsRecordEvent(
2256 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
2257 do
2258 {
2259 compressedSize = 0;
2260 uncompressedSize = 0;
2261 zeroPageCount = 0;
2262
2263 IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
2264
2265 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
2266 ml_get_interrupts_enabled());
2267 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledBeforeSleepState, vars->ioBuffer);
2268 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
2269 pollerOpen = (kIOReturnSuccess == err);
2270 if (!pollerOpen)
2271 break;
2272
2273 // copy file block extent list if larger than header
2274
2275 count = vars->fileExtents->getLength();
2276 if (count > sizeof(header->fileExtentMap))
2277 {
2278 count -= sizeof(header->fileExtentMap);
2279 err = IOPolledFileWrite(vars->fileVars,
2280 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
2281 if (kIOReturnSuccess != err)
2282 break;
2283 }
2284
2285 uintptr_t hibernateBase;
2286 uintptr_t hibernateEnd;
2287
2288 hibernateBase = HIB_BASE; /* Defined in PAL headers */
2289
2290 hibernateEnd = (segHIBB + segSizeHIB);
2291
2292 // copy out restore1 code
2293
2294 for (count = 0;
2295 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2296 count += segLen)
2297 {
2298 for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++)
2299 {
2300 gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
2301 }
2302 }
2303
2304 page = atop_32(kvtophys(hibernateBase));
2305 count = atop_32(round_page(hibernateEnd) - hibernateBase);
2306 header->restore1CodePhysPage = page;
2307 header->restore1CodeVirt = hibernateBase;
2308 header->restore1PageCount = count;
2309 header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
2310 header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
2311
2312 // sum __HIB seg, with zeros for the stack
2313 src = (uint8_t *) trunc_page(hibernateBase);
2314 for (page = 0; page < count; page++)
2315 {
2316 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
2317 restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
2318 else
2319 restore1Sum += 0x00000000;
2320 src += page_size;
2321 }
2322 sum1 = restore1Sum;
2323
2324 // write the __HIB seg, with zeros for the stack
2325
2326 src = (uint8_t *) trunc_page(hibernateBase);
2327 count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
2328 if (count)
2329 {
2330 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
2331 if (kIOReturnSuccess != err)
2332 break;
2333 }
2334 err = IOPolledFileWrite(vars->fileVars,
2335 (uint8_t *) 0,
2336 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
2337 cryptvars);
2338 if (kIOReturnSuccess != err)
2339 break;
2340 src = &gIOHibernateRestoreStackEnd[0];
2341 count = round_page(hibernateEnd) - ((uintptr_t) src);
2342 if (count)
2343 {
2344 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
2345 if (kIOReturnSuccess != err)
2346 break;
2347 }
2348
2349 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2350 {
2351 vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
2352 vars->fileVars->encryptEnd = UINT64_MAX;
2353 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
2354 }
2355
2356 // write the preview buffer
2357
2358 if (vars->previewBuffer)
2359 {
2360 ppnum = 0;
2361 count = 0;
2362 do
2363 {
2364 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
2365 pageAndCount[0] = atop_64(phys64);
2366 pageAndCount[1] = atop_32(segLen);
2367 err = IOPolledFileWrite(vars->fileVars,
2368 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
2369 cryptvars);
2370 if (kIOReturnSuccess != err)
2371 break;
2372 count += segLen;
2373 ppnum += sizeof(pageAndCount);
2374 }
2375 while (phys64);
2376 if (kIOReturnSuccess != err)
2377 break;
2378
2379 src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
2380
2381 ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
2382
2383 count = vars->previewBuffer->getLength();
2384
2385 header->previewPageListSize = ppnum;
2386 header->previewSize = count + ppnum;
2387
2388 for (page = 0; page < count; page += page_size)
2389 {
2390 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
2391 sum1 += hibernate_sum_page(src + page, atop_64(phys64));
2392 }
2393 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
2394 if (kIOReturnSuccess != err)
2395 break;
2396 }
2397
2398 // mark areas for no save
2399
2400 for (count = 0;
2401 (phys64 = vars->ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2402 count += segLen)
2403 {
2404 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2405 atop_64(phys64), atop_32(segLen),
2406 kIOHibernatePageStateFree);
2407 pageCount -= atop_32(segLen);
2408 }
2409
2410 for (count = 0;
2411 (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2412 count += segLen)
2413 {
2414 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2415 atop_64(phys64), atop_32(segLen),
2416 kIOHibernatePageStateFree);
2417 pageCount -= atop_32(segLen);
2418 }
2419
2420 // copy out bitmap of pages available for trashing during restore
2421
2422 bitmap_size = vars->page_list_wired->list_size;
2423 src = (uint8_t *) vars->page_list_wired;
2424 err = IOPolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
2425 if (kIOReturnSuccess != err)
2426 break;
2427
2428 // mark more areas for no save, but these are not available
2429 // for trashing during restore
2430
2431 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
2432
2433
2434 page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
2435 count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
2436 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2437 page, count,
2438 kIOHibernatePageStateFree);
2439 pageCount -= count;
2440
2441 if (vars->previewBuffer) for (count = 0;
2442 (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2443 count += segLen)
2444 {
2445 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2446 atop_64(phys64), atop_32(segLen),
2447 kIOHibernatePageStateFree);
2448 pageCount -= atop_32(segLen);
2449 }
2450
2451 for (count = 0;
2452 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2453 count += segLen)
2454 {
2455 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2456 atop_64(phys64), atop_32(segLen),
2457 kIOHibernatePageStateFree);
2458 pageCount -= atop_32(segLen);
2459 }
2460
2461 (void)hibernate_pal_callback;
2462
2463 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2464 compressed = src + page_size;
2465 scratch = compressed + page_size;
2466
2467 // compress a zero page
2468 bzero(src, page_size);
2469 zerosCompressed = vars->handoffBuffer->getBytesNoCopy();
2470 zerosCompressedLen = WKdm_compress_new((WK_word*) src,
2471 (WK_word*) zerosCompressed,
2472 (WK_word*) scratch,
2473 page_size - 4);
2474
2475 pagesDone = 0;
2476 lastBlob = 0;
2477
2478 HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
2479 bitmap_size, header->previewSize,
2480 pageCount, vars->fileVars->position);
2481
2482 enum
2483 // pageType
2484 {
2485 kWired = 0x02,
2486 kEncrypt = 0x01,
2487 kWiredEncrypt = kWired | kEncrypt,
2488 kWiredClear = kWired,
2489 kUnwiredEncrypt = kEncrypt
2490 };
2491
2492 for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)
2493 {
2494 if (kUnwiredEncrypt == pageType)
2495 {
2496 // start unwired image
2497 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2498 {
2499 vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
2500 vars->fileVars->encryptEnd = UINT64_MAX;
2501 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
2502 }
2503 bcopy(&cryptvars->aes_iv[0],
2504 &gIOHibernateCryptWakeContext.aes_iv[0],
2505 sizeof(cryptvars->aes_iv));
2506 cryptvars = &gIOHibernateCryptWakeContext;
2507 }
2508 for (iterDone = false, ppnum = 0; !iterDone; )
2509 {
2510 count = hibernate_page_list_iterate((kWired & pageType)
2511 ? vars->page_list_wired : vars->page_list,
2512 &ppnum);
2513 // kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
2514 iterDone = !count;
2515
2516 if (count && (kWired & pageType) && needEncrypt)
2517 {
2518 uint32_t checkIndex;
2519 for (checkIndex = 0;
2520 (checkIndex < count)
2521 && (((kEncrypt & pageType) == 0) == no_encrypt_page(ppnum + checkIndex));
2522 checkIndex++)
2523 {}
2524 if (!checkIndex)
2525 {
2526 ppnum++;
2527 continue;
2528 }
2529 count = checkIndex;
2530 }
2531
2532 switch (pageType)
2533 {
2534 case kWiredEncrypt: wiredPagesEncrypted += count; break;
2535 case kWiredClear: wiredPagesClear += count; break;
2536 case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
2537 }
2538
2539 if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */}
2540 else
2541 {
2542 pageAndCount[0] = ppnum;
2543 pageAndCount[1] = count;
2544 err = IOPolledFileWrite(vars->fileVars,
2545 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
2546 cryptvars);
2547 if (kIOReturnSuccess != err)
2548 break;
2549 }
2550
2551 for (page = ppnum; page < (ppnum + count); page++)
2552 {
2553 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
2554 if (err)
2555 {
2556 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
2557 break;
2558 }
2559
2560 sum = hibernate_sum_page(src, page);
2561 if (kWired & pageType)
2562 sum1 += sum;
2563 else
2564 sum2 += sum;
2565
2566 clock_get_uptime(&startTime);
2567 wkresult = WKdm_compress_new((WK_word*) src,
2568 (WK_word*) compressed,
2569 (WK_word*) scratch,
2570 page_size - 4);
2571
2572 clock_get_uptime(&endTime);
2573 ADD_ABSOLUTETIME(&compTime, &endTime);
2574 SUB_ABSOLUTETIME(&compTime, &startTime);
2575
2576 compBytes += page_size;
2577 pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
2578
2579 if ((pageCompressedSize == zerosCompressedLen)
2580 && !bcmp(compressed, zerosCompressed, zerosCompressedLen))
2581 {
2582 pageCompressedSize = 0;
2583 zeroPageCount++;
2584 }
2585
2586 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2587 pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
2588
2589 if (pageCompressedSize != page_size)
2590 data = compressed;
2591 else
2592 data = src;
2593
2594 tag = pageCompressedSize | kIOHibernateTagSignature;
2595 err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
2596 if (kIOReturnSuccess != err)
2597 break;
2598
2599 err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
2600 if (kIOReturnSuccess != err)
2601 break;
2602
2603 compressedSize += pageCompressedSize;
2604 uncompressedSize += page_size;
2605 pagesDone++;
2606
2607 if (vars->consoleMapping && (0 == (1023 & pagesDone)))
2608 {
2609 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
2610 if (blob != lastBlob)
2611 {
2612 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
2613 lastBlob = blob;
2614 }
2615 }
2616 if (0 == (8191 & pagesDone))
2617 {
2618 clock_get_uptime(&endTime);
2619 SUB_ABSOLUTETIME(&endTime, &allTime);
2620 absolutetime_to_nanoseconds(endTime, &nsec);
2621 progressStamp = nsec / 750000000ULL;
2622 if (progressStamp != lastProgressStamp)
2623 {
2624 lastProgressStamp = progressStamp;
2625 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
2626 }
2627 }
2628 }
2629 if (kIOReturnSuccess != err)
2630 break;
2631 ppnum = page;
2632 }
2633
2634 if (kIOReturnSuccess != err)
2635 break;
2636
2637 if ((kEncrypt & pageType) && vars->fileVars->encryptStart)
2638 {
2639 vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
2640 HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
2641 }
2642
2643 if (kWiredEncrypt != pageType)
2644 {
2645 // end of image1/2 - fill to next block
2646 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2647 if (kIOReturnSuccess != err)
2648 break;
2649 }
2650 if (kWiredClear == pageType)
2651 {
2652 // enlarge wired image for test
2653 // err = IOPolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
2654
2655 // end wired image
2656 header->encryptStart = vars->fileVars->encryptStart;
2657 header->encryptEnd = vars->fileVars->encryptEnd;
2658 image1Size = vars->fileVars->position;
2659 HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
2660 image1Size, header->encryptStart, header->encryptEnd);
2661 }
2662 }
2663 if (kIOReturnSuccess != err)
2664 {
2665 if (kIOReturnOverrun == err)
2666 {
2667 // update actual compression ratio on not enough space
2668 gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
2669 }
2670 break;
2671 }
2672
2673 // Header:
2674
2675 header->imageSize = vars->fileVars->position;
2676 header->image1Size = image1Size;
2677 header->bitmapSize = bitmap_size;
2678 header->pageCount = pageCount;
2679
2680 header->restore1Sum = restore1Sum;
2681 header->image1Sum = sum1;
2682 header->image2Sum = sum2;
2683 header->sleepTime = gIOLastSleepTime.tv_sec;
2684
2685 header->compression = (compressedSize << 8) / uncompressedSize;
2686 gIOHibernateCompression = header->compression;
2687
2688 count = vars->fileExtents->getLength();
2689 if (count > sizeof(header->fileExtentMap))
2690 {
2691 header->fileExtentMapSize = count;
2692 count = sizeof(header->fileExtentMap);
2693 }
2694 else
2695 header->fileExtentMapSize = sizeof(header->fileExtentMap);
2696 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
2697
2698 header->deviceBase = vars->fileVars->block0;
2699 header->deviceBlockSize = vars->fileVars->blockSize;
2700
2701 IOPolledFileSeek(vars->fileVars, 0);
2702 err = IOPolledFileWrite(vars->fileVars,
2703 (uint8_t *) header, sizeof(IOHibernateImageHeader),
2704 cryptvars);
2705 if (kIOReturnSuccess != err)
2706 break;
2707 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2708 if (kIOReturnSuccess != err)
2709 break;
2710 err = IOHibernatePollerIODone(vars->fileVars, true);
2711 if (kIOReturnSuccess != err)
2712 break;
2713 }
2714 while (false);
2715
2716 clock_get_uptime(&endTime);
2717
2718 IOService::getPMRootDomain()->pmStatsRecordEvent(
2719 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
2720
2721 SUB_ABSOLUTETIME(&endTime, &allTime);
2722 absolutetime_to_nanoseconds(endTime, &nsec);
2723 HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
2724
2725 absolutetime_to_nanoseconds(compTime, &nsec);
2726 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2727 compBytes,
2728 nsec / 1000000ULL,
2729 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2730
2731 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2732 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
2733 vars->fileVars->cryptBytes,
2734 nsec / 1000000ULL,
2735 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2736
2737 HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
2738 header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
2739 uncompressedSize, atop_32(uncompressedSize), compressedSize,
2740 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
2741 sum1, sum2);
2742
2743 HIBLOG("zeroPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
2744 zeroPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
2745
2746 if (vars->fileVars->io)
2747 (void) IOHibernatePollerIODone(vars->fileVars, false);
2748
2749 if (pollerOpen)
2750 IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
2751
2752 if (vars->consoleMapping)
2753 ProgressUpdate(gIOHibernateGraphicsInfo,
2754 vars->consoleMapping, 0, kIOHibernateProgressCount);
2755
2756 HIBLOG("hibernate_write_image done(%x)\n", err);
2757
2758 // should we come back via regular wake, set the state in memory.
2759 gIOHibernateState = kIOHibernateStateInactive;
2760
2761 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END,
2762 wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted, 0, 0);
2763
2764 if (kIOReturnSuccess == err)
2765 {
2766 if (kIOHibernateModeSleep & gIOHibernateMode)
2767 {
2768 return (kIOHibernatePostWriteSleep);
2769 }
2770 else if(kIOHibernateModeRestart & gIOHibernateMode)
2771 {
2772 return (kIOHibernatePostWriteRestart);
2773 }
2774 else
2775 {
2776 /* by default, power down */
2777 return (kIOHibernatePostWriteHalt);
2778 }
2779 }
2780 else if (kIOReturnAborted == err)
2781 {
2782 return (kIOHibernatePostWriteWake);
2783 }
2784 else
2785 {
2786 /* on error, sleep */
2787 return (kIOHibernatePostWriteSleep);
2788 }
2789 }
2790
2791 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2792
2793 extern "C" void
2794 hibernate_machine_init(void)
2795 {
2796 IOReturn err;
2797 uint32_t sum;
2798 uint32_t pagesDone;
2799 uint32_t pagesRead = 0;
2800 AbsoluteTime startTime, compTime;
2801 AbsoluteTime allTime, endTime;
2802 AbsoluteTime startIOTime, endIOTime;
2803 uint64_t nsec, nsecIO;
2804 uint64_t compBytes;
2805 uint32_t lastProgressStamp = 0;
2806 uint32_t progressStamp;
2807 hibernate_cryptvars_t * cryptvars = 0;
2808
2809 IOHibernateVars * vars = &gIOHibernateVars;
2810 bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
2811
2812 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
2813 return;
2814
2815 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2816 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2817
2818 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
2819 {
2820 HIBLOG("regular wake\n");
2821 return;
2822 }
2823
2824 HIBPRINT("diag %x %x %x %x\n",
2825 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
2826 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
2827
2828 #define t40ms(x) (tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)
2829 #define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
2830 tStat(booterStart, booterStart);
2831 gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart,
2832 tStat(booterDuration0, booterTime0);
2833 tStat(booterDuration1, booterTime1);
2834 tStat(booterDuration2, booterTime2);
2835 tStat(booterDuration, booterTime);
2836 tStat(booterConnectDisplayDuration, connectDisplayTime);
2837 tStat(booterSplashDuration, splashTime);
2838 tStat(trampolineDuration, trampolineTime);
2839
2840 gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size;
2841 gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize;
2842 gIOHibernateStats->image1Pages = pagesDone;
2843
2844 HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
2845 gIOHibernateStats->booterStart,
2846 gIOHibernateStats->smcStart,
2847 gIOHibernateStats->booterDuration0,
2848 gIOHibernateStats->booterDuration1,
2849 gIOHibernateStats->booterDuration2,
2850 gIOHibernateStats->booterDuration,
2851 gIOHibernateStats->booterConnectDisplayDuration,
2852 gIOHibernateStats->booterSplashDuration,
2853 gIOHibernateStats->trampolineDuration);
2854
2855 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
2856 gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
2857 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
2858
2859 if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
2860 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)))
2861 {
2862 hibernate_page_list_discard(vars->page_list);
2863 }
2864
2865 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
2866
2867 if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount)
2868 panic("handoff overflow");
2869
2870 IOHibernateHandoff * handoff;
2871 bool done = false;
2872 bool foundCryptData = false;
2873
2874 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
2875 !done;
2876 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
2877 {
2878 // HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
2879 uint8_t * data = &handoff->data[0];
2880 switch (handoff->type)
2881 {
2882 case kIOHibernateHandoffTypeEnd:
2883 done = true;
2884 break;
2885
2886 case kIOHibernateHandoffTypeGraphicsInfo:
2887 bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
2888 break;
2889
2890 case kIOHibernateHandoffTypeCryptVars:
2891 if (cryptvars)
2892 {
2893 hibernate_cryptwakevars_t *
2894 wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
2895 bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
2896 }
2897 foundCryptData = true;
2898 bzero(data, handoff->bytecount);
2899 break;
2900
2901 case kIOHibernateHandoffTypeMemoryMap:
2902
2903 clock_get_uptime(&allTime);
2904
2905 hibernate_newruntime_map(data, handoff->bytecount,
2906 gIOHibernateCurrentHeader->systemTableOffset);
2907
2908 clock_get_uptime(&endTime);
2909
2910 SUB_ABSOLUTETIME(&endTime, &allTime);
2911 absolutetime_to_nanoseconds(endTime, &nsec);
2912
2913 HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
2914
2915 break;
2916
2917 case kIOHibernateHandoffTypeDeviceTree:
2918 {
2919 // DTEntry chosen = NULL;
2920 // HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen));
2921 }
2922 break;
2923
2924 default:
2925 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
2926 break;
2927 }
2928 }
2929 if (cryptvars && !foundCryptData)
2930 panic("hibernate handoff");
2931
2932 HIBPRINT("video %x %d %d %d status %x\n",
2933 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2934 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
2935
2936 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)
2937 {
2938 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2939 * gIOHibernateGraphicsInfo->rowBytes);
2940 if (vars->videoMapSize > vars->videoAllocSize) vars->videoMapSize = 0;
2941 else
2942 {
2943 IOMapPages(kernel_map,
2944 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2945 vars->videoMapSize, kIOMapInhibitCache );
2946 }
2947 }
2948
2949 if (vars->videoMapSize)
2950 ProgressUpdate(gIOHibernateGraphicsInfo,
2951 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2952
2953 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2954 uint8_t * compressed = src + page_size;
2955 uint8_t * scratch = compressed + page_size;
2956 uint32_t decoOffset;
2957
2958 clock_get_uptime(&allTime);
2959 AbsoluteTime_to_scalar(&compTime) = 0;
2960 compBytes = 0;
2961
2962 HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2963 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0);
2964 clock_get_uptime(&startIOTime);
2965 endTime = startIOTime;
2966 SUB_ABSOLUTETIME(&endTime, &allTime);
2967 absolutetime_to_nanoseconds(endTime, &nsec);
2968 HIBLOG("IOHibernatePollerOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
2969
2970 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2971
2972 // kick off the read ahead
2973 vars->fileVars->io = false;
2974 vars->fileVars->bufferHalf = 0;
2975 vars->fileVars->bufferLimit = 0;
2976 vars->fileVars->lastRead = 0;
2977 vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
2978 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2979 vars->fileVars->cryptBytes = 0;
2980 AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
2981
2982 err = IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
2983 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2984 // --
2985
2986 HIBLOG("hibernate_machine_init reading\n");
2987
2988 uint32_t * header = (uint32_t *) src;
2989 sum = 0;
2990
2991 while (kIOReturnSuccess == err)
2992 {
2993 unsigned int count;
2994 unsigned int page;
2995 uint32_t tag;
2996 vm_offset_t ppnum, compressedSize;
2997
2998 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2999 if (kIOReturnSuccess != err)
3000 break;
3001
3002 ppnum = header[0];
3003 count = header[1];
3004
3005 // HIBPRINT("(%x, %x)\n", ppnum, count);
3006
3007 if (!count)
3008 break;
3009
3010 for (page = 0; page < count; page++)
3011 {
3012 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
3013 if (kIOReturnSuccess != err)
3014 break;
3015
3016 compressedSize = kIOHibernateTagLength & tag;
3017 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength))
3018 {
3019 err = kIOReturnIPCError;
3020 break;
3021 }
3022
3023 if (!compressedSize) bzero_phys(ptoa_64(ppnum), page_size);
3024 else
3025 {
3026 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
3027 if (kIOReturnSuccess != err) break;
3028 if (compressedSize < page_size)
3029 {
3030 decoOffset = page_size;
3031 clock_get_uptime(&startTime);
3032 WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, page_size);
3033 clock_get_uptime(&endTime);
3034 ADD_ABSOLUTETIME(&compTime, &endTime);
3035 SUB_ABSOLUTETIME(&compTime, &startTime);
3036 compBytes += page_size;
3037 }
3038 else decoOffset = 0;
3039
3040 sum += hibernate_sum_page((src + decoOffset), ppnum);
3041 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
3042 if (err)
3043 {
3044 HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
3045 break;
3046 }
3047 }
3048
3049 ppnum++;
3050 pagesDone++;
3051 pagesRead++;
3052
3053 if (0 == (8191 & pagesDone))
3054 {
3055 clock_get_uptime(&endTime);
3056 SUB_ABSOLUTETIME(&endTime, &allTime);
3057 absolutetime_to_nanoseconds(endTime, &nsec);
3058 progressStamp = nsec / 750000000ULL;
3059 if (progressStamp != lastProgressStamp)
3060 {
3061 lastProgressStamp = progressStamp;
3062 HIBPRINT("pages %d (%d%%)\n", pagesDone,
3063 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
3064 }
3065 }
3066 }
3067 }
3068 if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages))
3069 err = kIOReturnLockedRead;
3070
3071 if (kIOReturnSuccess != err)
3072 panic("Hibernate restore error %x", err);
3073
3074 gIOHibernateCurrentHeader->actualImage2Sum = sum;
3075 gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
3076
3077 if (vars->fileVars->io)
3078 (void) IOHibernatePollerIODone(vars->fileVars, false);
3079
3080 clock_get_uptime(&endIOTime);
3081
3082 err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
3083
3084 clock_get_uptime(&endTime);
3085
3086 IOService::getPMRootDomain()->pmStatsRecordEvent(
3087 kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
3088 IOService::getPMRootDomain()->pmStatsRecordEvent(
3089 kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
3090
3091 SUB_ABSOLUTETIME(&endTime, &allTime);
3092 absolutetime_to_nanoseconds(endTime, &nsec);
3093
3094 SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
3095 absolutetime_to_nanoseconds(endIOTime, &nsecIO);
3096
3097 gIOHibernateStats->kernelImageReadDuration = nsec / 1000000ULL;
3098 gIOHibernateStats->imagePages = pagesDone;
3099
3100 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
3101 pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
3102 nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
3103
3104 absolutetime_to_nanoseconds(compTime, &nsec);
3105 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
3106 compBytes,
3107 nsec / 1000000ULL,
3108 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
3109
3110 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
3111 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
3112 vars->fileVars->cryptBytes,
3113 nsec / 1000000ULL,
3114 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
3115
3116 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_NONE, pagesRead, pagesDone, 0, 0, 0);
3117 }
3118
3119 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
3120
3121 void IOHibernateSetWakeCapabilities(uint32_t capability)
3122 {
3123 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
3124 {
3125 gIOHibernateStats->wakeCapability = capability;
3126
3127 if (kIOPMSystemCapabilityGraphics & capability)
3128 {
3129 vm_compressor_do_warmup();
3130 }
3131
3132 }
3133 }
3134
3135 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
3136
3137 void IOHibernateSystemRestart(void)
3138 {
3139 static uint8_t noteStore[32] __attribute__((aligned(32)));
3140 IORegistryEntry * regEntry;
3141 const OSSymbol * sym;
3142 OSData * noteProp;
3143 OSData * data;
3144 uintptr_t * smcVars;
3145 uint8_t * smcBytes;
3146 size_t len;
3147 addr64_t element;
3148
3149 data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
3150 if (!data) return;
3151
3152 smcVars = (typeof(smcVars)) data->getBytesNoCopy();
3153 smcBytes = (typeof(smcBytes)) smcVars[1];
3154 len = smcVars[0];
3155 if (len > sizeof(noteStore)) len = sizeof(noteStore);
3156 noteProp = OSData::withCapacity(3 * sizeof(element));
3157 if (!noteProp) return;
3158 element = len;
3159 noteProp->appendBytes(&element, sizeof(element));
3160 element = crc32(0, smcBytes, len);
3161 noteProp->appendBytes(&element, sizeof(element));
3162
3163 bcopy(smcBytes, noteStore, len);
3164 element = (addr64_t) &noteStore[0];
3165 element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
3166 noteProp->appendBytes(&element, sizeof(element));
3167
3168 if (!gIOOptionsEntry)
3169 {
3170 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
3171 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
3172 if (regEntry && !gIOOptionsEntry)
3173 regEntry->release();
3174 }
3175
3176 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
3177 if (gIOOptionsEntry && sym) gIOOptionsEntry->setProperty(sym, noteProp);
3178 if (noteProp) noteProp->release();
3179 if (sym) sym->release();
3180 }
3181
3182
3183
mirror of XNU