]> git.saurik.com Git - apple/xnu.git/blob - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blob
? search:


3bd133ebac90398348579beafb3e70f45c899272
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
1 /*
2 * Copyright (c) 2004-2008 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29
30 /*
31
32 Sleep:
33
34 - PMRootDomain calls IOHibernateSystemSleep() before system sleep
35 (devices awake, normal execution context)
36 - IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
37 grabs its extents and searches for a polling driver willing to work with that IOMedia.
38 The BSD code makes an ioctl to the storage driver to get the partition base offset to
39 the disk, and other ioctls to get the transfer constraints
40 If successful, the file is written to make sure its initially not bootable (in case of
41 later failure) and nvram set to point to the first block of the file. (Has to be done
42 here so blocking is possible in nvram support).
43 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
44 page out any pages it wants to (currently zero, but probably some percentage of memory).
45 Its assumed just allocating pages will cause the VM system to naturally select the best
46 pages for eviction. It also copies processor flags needed for the restore path and sets
47 a flag in the boot processor proc info.
48 gIOHibernateState = kIOHibernateStateHibernating.
49 - Regular sleep progresses - some drivers may inspect the root domain property
50 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
51 as usual but leaves motherboard I/O on.
52 - Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
53 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
54 all ppc RC bits out of the hash table and caches into the mapping structures.
55 - hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
56 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
57 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
58 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
59 The image header and block list are written. The header includes the second file extent so
60 only the header block is needed to read the file, regardless of filesystem.
61 The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
62 (only) is used to decompress the image during wake/boot.
63 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
64 The bitmaps are written to the image.
65 More areas are removed from the bitmaps (after they have been written to the image) - the
66 segment "__HIB" pages and interrupt stack.
67 Each wired page is compressed and written and then each non-wired page. Compression and
68 disk writes are in parallel.
69 The image header is written to the start of the file and the polling driver closed.
70 The machine powers down (or sleeps).
71
72 Boot/Wake:
73
74 - BootX sees the boot-image nvram variable containing the device and block number of the image,
75 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
76 - BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
77 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
78 that is in the kernel's __HIB section.
79 - BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
80 hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
81 only code & data in that section is safe to call since all the other wired pages are still
82 compressed in the image.
83 - hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
84 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
85 location directly, and copies those that can't to interim free pages. When the image has been
86 completed, the copies are uncompressed, overwriting the wired image pages.
87 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
88 is used to get pages into place for 64bit.
89 - the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
90 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
91 are removed from the software strutures, and the hash table is reinitialized.
92 - After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
93 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
94 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
95 for the remaining non wired pages.
96 - The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
97 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
98 is closed via bsd.
99
100 Polled Mode I/O:
101
102 IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
103 registry, specifying an object of calls IOPolledInterface.
104
105 Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
106 partition) that the image is going to live, looking for polled interface properties. If it finds
107 one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
108 interfaces found are kept in an ordered list.
109
110 There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
111 few different contexts things happen in:
112
113 - there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
114 up and running) and after wake - this is safe to allocate memory and do anything. The device
115 ignores sleep requests from that point since its a waste of time if it goes to sleep and
116 immediately wakes back up for the image write.
117
118 - there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
119 immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
120 from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
121 used to flush and set the disk to sleep.
122
123 - there is an Open/Close (AfterSleep) pair made around the image read operations that happen
124 immediately after sleep. These can't block or allocate memory. This is happening after the platform
125 expert has woken the low level bits of the system, but most of the I/O system has not. There is only
126 one thread running.
127
128 For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
129 (prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
130 the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
131 Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
132 that is called for the hardware to check for events, and complete the I/O via the callback.
133 The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
134 to restrict I/O ops.
135 */
136
137 #include <sys/systm.h>
138
139 #include <IOKit/IOWorkLoop.h>
140 #include <IOKit/IOCommandGate.h>
141 #include <IOKit/IOTimerEventSource.h>
142 #include <IOKit/IOPlatformExpert.h>
143 #include <IOKit/IOKitDebug.h>
144 #include <IOKit/IOTimeStamp.h>
145 #include <IOKit/pwr_mgt/RootDomain.h>
146 #include <IOKit/pwr_mgt/IOPMPrivate.h>
147 #include <IOKit/IOMessage.h>
148 #include <IOKit/IODeviceTreeSupport.h>
149 #include <IOKit/IOBSD.h>
150 #include "RootDomainUserClient.h"
151 #include <IOKit/pwr_mgt/IOPowerConnection.h>
152 #include "IOPMPowerStateQueue.h"
153 #include <IOKit/IOBufferMemoryDescriptor.h>
154 #include <IOKit/AppleKeyStoreInterface.h>
155 #include <libkern/crypto/aes.h>
156
157 #include <sys/uio.h>
158 #include <sys/conf.h>
159 #include <sys/stat.h>
160 #include <sys/fcntl.h> // (FWRITE, ...)
161 #include <sys/sysctl.h>
162 #include <sys/kdebug.h>
163
164 #include <IOKit/IOHibernatePrivate.h>
165 #include <IOKit/IOPolledInterface.h>
166 #include <IOKit/IONVRAM.h>
167 #include "IOHibernateInternal.h"
168 #include <vm/WKdm_new.h>
169 #include "IOKitKernelInternal.h"
170 #include <pexpert/device_tree.h>
171
172 #include <machine/pal_routines.h>
173 #include <machine/pal_hibernate.h>
174 #include <i386/tsc.h>
175
176 extern "C" addr64_t kvtophys(vm_offset_t va);
177 extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
178
179 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
180
181 #define DISABLE_TRIM 0
182 #define TRIM_DELAY 5000
183
184 extern unsigned int save_kdebug_enable;
185 extern uint32_t gIOHibernateState;
186 uint32_t gIOHibernateMode;
187 static char gIOHibernateBootSignature[256+1];
188 static char gIOHibernateFilename[MAXPATHLEN+1];
189 static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
190 uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
191 static uint64_t gIOHibernateCompression = 0x80; // default compression 50%
192
193 static IODTNVRAM * gIOOptionsEntry;
194 static IORegistryEntry * gIOChosenEntry;
195 #if defined(__i386__) || defined(__x86_64__)
196 static const OSSymbol * gIOCreateEFIDevicePathSymbol;
197 static const OSSymbol * gIOHibernateRTCVariablesKey;
198 static const OSSymbol * gIOHibernateBoot0082Key;
199 static const OSSymbol * gIOHibernateBootNextKey;
200 static OSData * gIOHibernateBoot0082Data;
201 static OSData * gIOHibernateBootNextData;
202 static OSObject * gIOHibernateBootNextSave;
203 #endif
204
205 static IOLock * gFSLock;
206 static uint32_t gFSState;
207 static IOPolledFileIOVars gFileVars;
208 static IOHibernateVars gIOHibernateVars;
209 static struct kern_direct_file_io_ref_t * gIOHibernateFileRef;
210 static hibernate_cryptvars_t gIOHibernateCryptWakeContext;
211 static hibernate_graphics_t _hibernateGraphics;
212 static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
213 static hibernate_statistics_t _hibernateStats;
214 static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats;
215
216 enum
217 {
218 kFSIdle = 0,
219 kFSOpening = 2,
220 kFSOpened = 3,
221 kFSTimedOut = 4,
222 };
223
224 static IOReturn IOHibernateDone(IOHibernateVars * vars);
225
226 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
227
228 enum { kXPRamAudioVolume = 8 };
229 enum { kDefaultIOSize = 128 * 1024 };
230 enum { kVideoMapSize = 80 * 1024 * 1024 };
231
232 #ifndef kIOMediaPreferredBlockSizeKey
233 #define kIOMediaPreferredBlockSizeKey "Preferred Block Size"
234 #endif
235
236 #ifndef kIOBootPathKey
237 #define kIOBootPathKey "bootpath"
238 #endif
239 #ifndef kIOSelectedBootDeviceKey
240 #define kIOSelectedBootDeviceKey "boot-device"
241 #endif
242
243 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
244
245 // copy from phys addr to MD
246
247 static IOReturn
248 IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
249 IOByteCount offset, addr64_t bytes, IOByteCount length)
250 {
251 addr64_t srcAddr = bytes;
252 IOByteCount remaining;
253
254 remaining = length = min(length, md->getLength() - offset);
255 while (remaining) { // (process another target segment?)
256 addr64_t dstAddr64;
257 IOByteCount dstLen;
258
259 dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
260 if (!dstAddr64)
261 break;
262
263 // Clip segment length to remaining
264 if (dstLen > remaining)
265 dstLen = remaining;
266
267 #if 1
268 bcopy_phys(srcAddr, dstAddr64, dstLen);
269 #else
270 copypv(srcAddr, dstAddr64, dstLen,
271 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
272 #endif
273 srcAddr += dstLen;
274 offset += dstLen;
275 remaining -= dstLen;
276 }
277
278 assert(!remaining);
279
280 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
281 }
282
283 // copy from MD to phys addr
284
285 static IOReturn
286 IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
287 IOByteCount offset, addr64_t bytes, IOByteCount length)
288 {
289 addr64_t dstAddr = bytes;
290 IOByteCount remaining;
291
292 remaining = length = min(length, md->getLength() - offset);
293 while (remaining) { // (process another target segment?)
294 addr64_t srcAddr64;
295 IOByteCount dstLen;
296
297 srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
298 if (!srcAddr64)
299 break;
300
301 // Clip segment length to remaining
302 if (dstLen > remaining)
303 dstLen = remaining;
304
305 #if 1
306 bcopy_phys(srcAddr64, dstAddr, dstLen);
307 #else
308 copypv(srcAddr, dstAddr64, dstLen,
309 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
310 #endif
311 dstAddr += dstLen;
312 offset += dstLen;
313 remaining -= dstLen;
314 }
315
316 assert(!remaining);
317
318 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
319 }
320
321 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
322
323 void
324 hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
325 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
326 {
327 count += ppnum;
328 switch (kind)
329 {
330 case kIOHibernatePageStateUnwiredSave:
331 // unwired save
332 for (; ppnum < count; ppnum++)
333 {
334 hibernate_page_bitset(page_list, FALSE, ppnum);
335 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
336 }
337 break;
338 case kIOHibernatePageStateWiredSave:
339 // wired save
340 for (; ppnum < count; ppnum++)
341 {
342 hibernate_page_bitset(page_list, FALSE, ppnum);
343 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
344 }
345 break;
346 case kIOHibernatePageStateFree:
347 // free page
348 for (; ppnum < count; ppnum++)
349 {
350 hibernate_page_bitset(page_list, TRUE, ppnum);
351 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
352 }
353 break;
354 default:
355 panic("hibernate_set_page_state");
356 }
357 }
358
359 static vm_offset_t
360 hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
361 {
362 uint32_t page = *pPage;
363 uint32_t count;
364 hibernate_bitmap_t * bitmap;
365
366 while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
367 {
368 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
369 if (!count)
370 break;
371 page += count;
372 if (page <= bitmap->last_page)
373 break;
374 }
375
376 *pPage = page;
377 if (bitmap)
378 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
379 else
380 count = 0;
381
382 return (count);
383 }
384
385 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
386
387 static IOReturn
388 IOHibernatePollerProbe(IOPolledFileIOVars * vars, IOService * target)
389 {
390 IOReturn err = kIOReturnError;
391 int32_t idx;
392 IOPolledInterface * poller;
393
394 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
395 {
396 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
397 err = poller->probe(target);
398 if (err)
399 {
400 HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
401 break;
402 }
403 }
404
405 return (err);
406 }
407
408 static IOReturn
409 IOHibernatePollerOpen(IOPolledFileIOVars * vars, uint32_t state, IOMemoryDescriptor * md)
410 {
411 IOReturn err = kIOReturnError;
412 int32_t idx;
413 IOPolledInterface * poller;
414
415 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
416 {
417 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
418 err = poller->open(state, md);
419 if (err)
420 {
421 HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
422 break;
423 }
424 }
425
426 return (err);
427 }
428
429 static IOReturn
430 IOHibernatePollerClose(IOPolledFileIOVars * vars, uint32_t state)
431 {
432 IOReturn err = kIOReturnError;
433 int32_t idx;
434 IOPolledInterface * poller;
435
436 for (idx = 0;
437 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
438 idx++)
439 {
440 err = poller->close(state);
441 if (err)
442 HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
443 }
444
445 return (err);
446 }
447
448 static void
449 IOHibernatePollerIOComplete(void * target,
450 void * parameter,
451 IOReturn status,
452 UInt64 actualByteCount)
453 {
454 IOPolledFileIOVars * vars = (IOPolledFileIOVars *) parameter;
455
456 vars->ioStatus = status;
457 }
458
459 static IOReturn
460 IOHibernatePollerIO(IOPolledFileIOVars * vars,
461 uint32_t operation, uint32_t bufferOffset,
462 uint64_t deviceOffset, uint64_t length)
463 {
464
465 IOReturn err = kIOReturnError;
466 IOPolledInterface * poller;
467 IOPolledCompletion completion;
468
469 completion.target = 0;
470 completion.action = &IOHibernatePollerIOComplete;
471 completion.parameter = vars;
472
473 vars->ioStatus = -1;
474
475 poller = (IOPolledInterface *) vars->pollers->getObject(0);
476 err = poller->startIO(operation, bufferOffset, deviceOffset + vars->block0, length, completion);
477 if (err)
478 HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
479
480 return (err);
481 }
482
483 static IOReturn
484 IOHibernatePollerIODone(IOPolledFileIOVars * vars, bool abortable)
485 {
486 IOReturn err = kIOReturnSuccess;
487 int32_t idx = 0;
488 IOPolledInterface * poller;
489
490 while (-1 == vars->ioStatus)
491 {
492 for (idx = 0;
493 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
494 idx++)
495 {
496 IOReturn newErr;
497 newErr = poller->checkForWork();
498 if ((newErr == kIOReturnAborted) && !abortable)
499 newErr = kIOReturnSuccess;
500 if (kIOReturnSuccess == err)
501 err = newErr;
502 }
503 }
504
505 if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort())
506 {
507 err = kIOReturnAborted;
508 HIBLOG("IOPolledInterface::checkForWork sw abort\n");
509 }
510
511 if (err)
512 {
513 HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
514 }
515 else
516 {
517 err = vars->ioStatus;
518 if (kIOReturnSuccess != err)
519 HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
520 }
521
522 return (err);
523 }
524
525 IOReturn
526 IOPolledInterface::checkAllForWork(void)
527 {
528 IOReturn err = kIOReturnNotReady;
529 int32_t idx;
530 IOPolledInterface * poller;
531
532 IOHibernateVars * vars = &gIOHibernateVars;
533
534 if (!vars->fileVars || !vars->fileVars->pollers)
535 return (err);
536
537 for (idx = 0;
538 (poller = (IOPolledInterface *) vars->fileVars->pollers->getObject(idx));
539 idx++)
540 {
541 err = poller->checkForWork();
542 if (err)
543 HIBLOG("IOPolledInterface::checkAllForWork[%d] 0x%x\n", idx, err);
544 }
545
546 return (err);
547 }
548
549 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
550
551 struct _OpenFileContext
552 {
553 OSData * extents;
554 uint64_t size;
555 };
556
557 static void
558 file_extent_callback(void * ref, uint64_t start, uint64_t length)
559 {
560 _OpenFileContext * ctx = (_OpenFileContext *) ref;
561 IOPolledFileExtent extent;
562
563 extent.start = start;
564 extent.length = length;
565
566 HIBLOG("[0x%qx, 0x%qx]\n", start, length);
567
568 ctx->extents->appendBytes(&extent, sizeof(extent));
569 ctx->size += length;
570 }
571
572 static IOService *
573 IOCopyMediaForDev(dev_t device)
574 {
575 OSDictionary * matching;
576 OSNumber * num;
577 OSIterator * iter;
578 IOService * result = 0;
579
580 matching = IOService::serviceMatching("IOMedia");
581 if (!matching)
582 return (0);
583 do
584 {
585 num = OSNumber::withNumber(major(device), 32);
586 if (!num)
587 break;
588 matching->setObject(kIOBSDMajorKey, num);
589 num->release();
590 num = OSNumber::withNumber(minor(device), 32);
591 if (!num)
592 break;
593 matching->setObject(kIOBSDMinorKey, num);
594 num->release();
595 if (!num)
596 break;
597 iter = IOService::getMatchingServices(matching);
598 if (iter)
599 {
600 result = (IOService *) iter->getNextObject();
601 result->retain();
602 iter->release();
603 }
604 }
605 while (false);
606 matching->release();
607
608 return (result);
609 }
610
611 IOReturn
612 IOPolledFileOpen( const char * filename, uint64_t setFileSize,
613 IOBufferMemoryDescriptor * ioBuffer,
614 IOPolledFileIOVars ** fileVars, OSData ** fileExtents,
615 OSData ** imagePath, uint8_t * volumeCryptKey)
616 {
617 IOReturn err = kIOReturnSuccess;
618 IOPolledFileIOVars * vars;
619 _OpenFileContext ctx;
620 OSData * extentsData;
621 OSNumber * num;
622 IOService * part = 0;
623 OSString * keyUUID = 0;
624 OSString * keyStoreUUID = 0;
625 dev_t block_dev;
626 dev_t hibernate_image_dev;
627 uint64_t maxiobytes;
628 AbsoluteTime startTime, endTime;
629 uint64_t nsec;
630
631 vars = IONew(IOPolledFileIOVars, 1);
632 if (!vars) return (kIOReturnNoMemory);
633 bzero(vars, sizeof(*vars));
634
635 do
636 {
637 vars->io = false;
638 vars->buffer = (uint8_t *) ioBuffer->getBytesNoCopy();
639 vars->bufferHalf = 0;
640 vars->bufferOffset = 0;
641 vars->bufferSize = ioBuffer->getLength() >> 1;
642
643 extentsData = OSData::withCapacity(32);
644 ctx.extents = extentsData;
645 ctx.size = 0;
646 clock_get_uptime(&startTime);
647 vars->fileRef = kern_open_file_for_direct_io(filename,
648 &file_extent_callback, &ctx,
649 setFileSize,
650 // write file:
651 0, (caddr_t) gIOHibernateCurrentHeader,
652 sizeof(IOHibernateImageHeader),
653 // results
654 &block_dev,
655 &hibernate_image_dev,
656 &vars->block0,
657 &maxiobytes,
658 &vars->flags);
659 #if 0
660 uint32_t msDelay = (131071 & random());
661 HIBLOG("sleep %d\n", msDelay);
662 IOSleep(msDelay);
663 #endif
664 clock_get_uptime(&endTime);
665 SUB_ABSOLUTETIME(&endTime, &startTime);
666 absolutetime_to_nanoseconds(endTime, &nsec);
667
668 if (!vars->fileRef) err = kIOReturnNoSpace;
669
670 IOLockLock(gFSLock);
671 if (kFSOpening != gFSState) err = kIOReturnTimeout;
672 IOLockUnlock(gFSLock);
673
674 HIBLOG("kern_open_file_for_direct_io(%d) took %qd ms\n", err, nsec / 1000000ULL);
675 if (kIOReturnSuccess != err) break;
676
677 if (kIOHibernateModeSSDInvert & gIOHibernateMode)
678 vars->flags ^= kIOHibernateOptionSSD;
679
680 HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx ssd %d\n", filename, ctx.size,
681 vars->block0, maxiobytes, kIOHibernateOptionSSD & vars->flags);
682 if (ctx.size < 1*1024*1024) // check against image size estimate!
683 {
684 err = kIOReturnNoSpace;
685 break;
686 }
687
688 vars->fileSize = ctx.size;
689 if (maxiobytes < vars->bufferSize) vars->bufferSize = maxiobytes;
690
691 vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
692
693 part = IOCopyMediaForDev(block_dev);
694 if (!part)
695 {
696 err = kIOReturnNotFound;
697 break;
698 }
699 err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false,
700 (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
701 if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID)
702 {
703 // IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
704 uuid_t volumeKeyUUID;
705 aks_volume_key_t vek;
706 static IOService * sKeyStore;
707 static const OSSymbol * sAKSGetKey;
708
709 if (!sAKSGetKey)
710 sAKSGetKey = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY);
711 if (!sKeyStore)
712 sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
713 if (sKeyStore)
714 err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
715 else
716 err = kIOReturnNoResources;
717 if (kIOReturnSuccess == err)
718 err = sKeyStore->callPlatformFunction(sAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
719 if (kIOReturnSuccess != err)
720 IOLog("volume key err 0x%x\n", err);
721 else
722 {
723 size_t bytes = (kIOHibernateAESKeySize / 8);
724 if (vek.key.keybytecount < bytes)
725 bytes = vek.key.keybytecount;
726 bcopy(&vek.key.keybytes[0], volumeCryptKey, bytes);
727 }
728 bzero(&vek, sizeof(vek));
729 }
730 part->release();
731
732 part = IOCopyMediaForDev(hibernate_image_dev);
733 if (!part)
734 {
735 err = kIOReturnNotFound;
736 break;
737 }
738
739 IORegistryEntry * next;
740 IORegistryEntry * child;
741 IOService * service;
742 OSData * data;
743
744 vars->pollers = OSArray::withCapacity(4);
745 if (!vars->pollers)
746 {
747 err = kIOReturnNoMemory;
748 break;
749 }
750
751 vars->blockSize = 512;
752 next = part;
753 do
754 {
755 IOPolledInterface * poller;
756 OSObject * obj;
757
758 obj = next->getProperty(kIOPolledInterfaceSupportKey);
759 if (kOSBooleanFalse == obj)
760 {
761 vars->pollers->flushCollection();
762 break;
763 }
764 else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
765 vars->pollers->setObject(poller);
766
767 if ((service = OSDynamicCast(IOService, next))
768 && service->getDeviceMemory()
769 && !vars->pollers->getCount()) break;
770
771 if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey))))
772 vars->blockSize = num->unsigned32BitValue();
773 child = next;
774 }
775 while ((next = child->getParentEntry(gIOServicePlane))
776 && child->isParent(next, gIOServicePlane, true));
777
778 if (vars->blockSize < 4096) vars->blockSize = 4096;
779
780 HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
781 major(hibernate_image_dev), minor(hibernate_image_dev), (long)vars->blockSize,
782 vars->pollers->getCount());
783
784 if (!vars->pollers->getCount())
785 {
786 err = kIOReturnUnsupported;
787 continue;
788 }
789 if (vars->blockSize < sizeof(IOHibernateImageHeader))
790 {
791 err = kIOReturnError;
792 continue;
793 }
794
795 err = IOHibernatePollerProbe(vars, (IOService *) part);
796 if (kIOReturnSuccess != err) break;
797
798 err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer);
799 if (kIOReturnSuccess != err) break;
800
801 vars->media = part;
802 next = part;
803 while (next)
804 {
805 next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue);
806 next = next->getParentEntry(gIOServicePlane);
807 }
808
809 *fileVars = vars;
810 *fileExtents = extentsData;
811
812 // make imagePath
813
814 if ((extentsData->getLength() >= sizeof(IOPolledFileExtent)))
815 {
816 char str2[24 + sizeof(uuid_string_t) + 2];
817
818 #if defined(__i386__) || defined(__x86_64__)
819 if (!gIOCreateEFIDevicePathSymbol)
820 gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
821
822 if (keyUUID)
823 snprintf(str2, sizeof(str2), "%qx:%s",
824 vars->extentMap[0].start, keyUUID->getCStringNoCopy());
825 else
826 snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
827
828 err = IOService::getPlatform()->callPlatformFunction(
829 gIOCreateEFIDevicePathSymbol, false,
830 (void *) part, (void *) str2,
831 (void *) (uintptr_t) true, (void *) &data);
832 #else
833 char str1[256];
834 int len = sizeof(str1);
835
836 if (!part->getPath(str1, &len, gIODTPlane))
837 err = kIOReturnNotFound;
838 else
839 {
840 snprintf(str2, sizeof(str2), ",%qx", vars->extentMap[0].start);
841 // (strip the plane name)
842 char * tail = strchr(str1, ':');
843 if (!tail)
844 tail = str1 - 1;
845 data = OSData::withBytes(tail + 1, strlen(tail + 1));
846 data->appendBytes(str2, strlen(str2));
847 }
848 #endif
849 if (kIOReturnSuccess == err)
850 *imagePath = data;
851 else
852 HIBLOG("error 0x%x getting path\n", err);
853 }
854 }
855 while (false);
856
857 if (kIOReturnSuccess != err)
858 {
859 HIBLOG("error 0x%x opening hibernation file\n", err);
860 if (vars->fileRef)
861 {
862 kern_close_file_for_direct_io(vars->fileRef, 0, 0, 0, 0, 0);
863 vars->fileRef = NULL;
864 }
865 }
866
867 if (part)
868 part->release();
869
870 return (err);
871 }
872
873 IOReturn
874 IOPolledFileClose( IOPolledFileIOVars * vars )
875 {
876 if (vars->pollers)
877 {
878 IOHibernatePollerClose(vars, kIOPolledPostflightState);
879 vars->pollers->release();
880 }
881
882 bzero(vars, sizeof(IOPolledFileIOVars));
883
884 return (kIOReturnSuccess);
885 }
886
887 static IOReturn
888 IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
889 {
890 IOPolledFileExtent * extentMap;
891
892 extentMap = vars->extentMap;
893
894 vars->position = position;
895
896 while (position >= extentMap->length)
897 {
898 position -= extentMap->length;
899 extentMap++;
900 }
901
902 vars->currentExtent = extentMap;
903 vars->extentRemaining = extentMap->length - position;
904 vars->extentPosition = vars->position - position;
905
906 if (vars->bufferSize <= vars->extentRemaining)
907 vars->bufferLimit = vars->bufferSize;
908 else
909 vars->bufferLimit = vars->extentRemaining;
910
911 return (kIOReturnSuccess);
912 }
913
914 static IOReturn
915 IOPolledFileWrite(IOPolledFileIOVars * vars,
916 const uint8_t * bytes, IOByteCount size,
917 hibernate_cryptvars_t * cryptvars)
918 {
919 IOReturn err = kIOReturnSuccess;
920 IOByteCount copy;
921 bool flush = false;
922
923 do
924 {
925 if (!bytes && !size)
926 {
927 // seek to end of block & flush
928 size = vars->position & (vars->blockSize - 1);
929 if (size)
930 size = vars->blockSize - size;
931 flush = true;
932 // use some garbage for the fill
933 bytes = vars->buffer + vars->bufferOffset;
934 }
935
936 copy = vars->bufferLimit - vars->bufferOffset;
937 if (copy > size)
938 copy = size;
939 else
940 flush = true;
941
942 if (bytes)
943 {
944 bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
945 bytes += copy;
946 }
947 else
948 bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
949
950 size -= copy;
951 vars->bufferOffset += copy;
952 vars->position += copy;
953
954 if (flush && vars->bufferOffset)
955 {
956 uint64_t offset = (vars->position - vars->bufferOffset
957 - vars->extentPosition + vars->currentExtent->start);
958 uint32_t length = (vars->bufferOffset);
959
960 #if CRYPTO
961 if (cryptvars && vars->encryptStart
962 && (vars->position > vars->encryptStart)
963 && ((vars->position - length) < vars->encryptEnd))
964 {
965 AbsoluteTime startTime, endTime;
966
967 uint64_t encryptLen, encryptStart;
968 encryptLen = vars->position - vars->encryptStart;
969 if (encryptLen > length)
970 encryptLen = length;
971 encryptStart = length - encryptLen;
972 if (vars->position > vars->encryptEnd)
973 encryptLen -= (vars->position - vars->encryptEnd);
974
975 clock_get_uptime(&startTime);
976
977 // encrypt the buffer
978 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
979 &cryptvars->aes_iv[0],
980 encryptLen / AES_BLOCK_SIZE,
981 vars->buffer + vars->bufferHalf + encryptStart,
982 &cryptvars->ctx.encrypt);
983
984 clock_get_uptime(&endTime);
985 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
986 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
987 vars->cryptBytes += encryptLen;
988
989 // save initial vector for following encrypts
990 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
991 &cryptvars->aes_iv[0],
992 AES_BLOCK_SIZE);
993 }
994 #endif /* CRYPTO */
995
996 if (vars->io)
997 {
998 err = IOHibernatePollerIODone(vars, true);
999 if (kIOReturnSuccess != err)
1000 break;
1001 }
1002
1003 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
1004 //if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
1005
1006 err = IOHibernatePollerIO(vars, kIOPolledWrite, vars->bufferHalf, offset, length);
1007 if (kIOReturnSuccess != err)
1008 break;
1009 vars->io = true;
1010
1011 vars->extentRemaining -= vars->bufferOffset;
1012 if (!vars->extentRemaining)
1013 {
1014 vars->currentExtent++;
1015 vars->extentRemaining = vars->currentExtent->length;
1016 vars->extentPosition = vars->position;
1017 if (!vars->extentRemaining)
1018 {
1019 err = kIOReturnOverrun;
1020 break;
1021 }
1022 }
1023
1024 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1025 vars->bufferOffset = 0;
1026 if (vars->bufferSize <= vars->extentRemaining)
1027 vars->bufferLimit = vars->bufferSize;
1028 else
1029 vars->bufferLimit = vars->extentRemaining;
1030
1031 flush = false;
1032 }
1033 }
1034 while (size);
1035
1036 return (err);
1037 }
1038
1039 static IOReturn
1040 IOPolledFileRead(IOPolledFileIOVars * vars,
1041 uint8_t * bytes, IOByteCount size,
1042 hibernate_cryptvars_t * cryptvars)
1043 {
1044 IOReturn err = kIOReturnSuccess;
1045 IOByteCount copy;
1046
1047 // bytesWritten += size;
1048
1049 do
1050 {
1051 copy = vars->bufferLimit - vars->bufferOffset;
1052 if (copy > size)
1053 copy = size;
1054
1055 if (bytes)
1056 {
1057 bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
1058 bytes += copy;
1059 }
1060 size -= copy;
1061 vars->bufferOffset += copy;
1062 // vars->position += copy;
1063
1064 if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd))
1065 {
1066 if (vars->io)
1067 {
1068 err = IOHibernatePollerIODone(vars, false);
1069 if (kIOReturnSuccess != err)
1070 break;
1071 }
1072 else
1073 cryptvars = 0;
1074
1075 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
1076
1077 vars->position += vars->lastRead;
1078 vars->extentRemaining -= vars->lastRead;
1079 vars->bufferLimit = vars->lastRead;
1080
1081 if (!vars->extentRemaining)
1082 {
1083 vars->currentExtent++;
1084 vars->extentRemaining = vars->currentExtent->length;
1085 vars->extentPosition = vars->position;
1086 if (!vars->extentRemaining)
1087 {
1088 err = kIOReturnOverrun;
1089 break;
1090 }
1091 }
1092
1093 uint64_t length;
1094 uint64_t lastReadLength = vars->lastRead;
1095 uint64_t offset = (vars->position
1096 - vars->extentPosition + vars->currentExtent->start);
1097 if (vars->extentRemaining <= vars->bufferSize)
1098 length = vars->extentRemaining;
1099 else
1100 length = vars->bufferSize;
1101 if ((length + vars->position) > vars->readEnd)
1102 length = vars->readEnd - vars->position;
1103
1104 vars->lastRead = length;
1105 if (length)
1106 {
1107 //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
1108 err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
1109 if (kIOReturnSuccess != err)
1110 break;
1111 vars->io = true;
1112 }
1113
1114 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1115 vars->bufferOffset = 0;
1116
1117 #if CRYPTO
1118 if (cryptvars)
1119 {
1120 uint8_t thisVector[AES_BLOCK_SIZE];
1121 AbsoluteTime startTime, endTime;
1122
1123 // save initial vector for following decrypts
1124 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
1125 bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
1126 &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1127
1128 // decrypt the buffer
1129 clock_get_uptime(&startTime);
1130
1131 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
1132 &thisVector[0],
1133 lastReadLength / AES_BLOCK_SIZE,
1134 vars->buffer + vars->bufferHalf,
1135 &cryptvars->ctx.decrypt);
1136
1137 clock_get_uptime(&endTime);
1138 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
1139 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
1140 vars->cryptBytes += lastReadLength;
1141 }
1142 #endif /* CRYPTO */
1143 }
1144 }
1145 while (size);
1146
1147 return (err);
1148 }
1149
1150 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1151
1152 IOReturn
1153 IOHibernateSystemSleep(void)
1154 {
1155 IOReturn err;
1156 OSData * data;
1157 OSObject * obj;
1158 OSString * str;
1159 OSNumber * num;
1160 bool dsSSD, vmflush;
1161 IOHibernateVars * vars;
1162
1163 gIOHibernateState = kIOHibernateStateInactive;
1164
1165 if (!gIOChosenEntry)
1166 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1167
1168 gIOHibernateDebugFlags = 0;
1169 if (kIOLogHibernate & gIOKitDebug)
1170 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
1171
1172 if (IOService::getPMRootDomain()->getHibernateSettings(
1173 &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime))
1174 {
1175 if (kIOHibernateModeSleep & gIOHibernateMode)
1176 // default to discard clean for safe sleep
1177 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
1178 | kIOHibernateModeDiscardCleanActive);
1179 }
1180
1181 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
1182 {
1183 if ((str = OSDynamicCast(OSString, obj)))
1184 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
1185 sizeof(gIOHibernateFilename));
1186 obj->release();
1187 }
1188
1189 if (!gIOHibernateMode || !gIOHibernateFilename[0])
1190 return (kIOReturnUnsupported);
1191
1192 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
1193
1194 vars = IONew(IOHibernateVars, 1);
1195 if (!vars) return (kIOReturnNoMemory);
1196 bzero(vars, sizeof(*vars));
1197
1198 IOLockLock(gFSLock);
1199 if (kFSIdle != gFSState)
1200 {
1201 HIBLOG("hibernate file busy\n");
1202 IOLockUnlock(gFSLock);
1203 IODelete(vars, IOHibernateVars, 1);
1204 return (kIOReturnBusy);
1205 }
1206 gFSState = kFSOpening;
1207 IOLockUnlock(gFSLock);
1208
1209 do
1210 {
1211 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1212 2 * page_size + WKdm_SCRATCH_BUF_SIZE, page_size);
1213 vars->ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1214 2 * kDefaultIOSize, page_size);
1215
1216 vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1217 ptoa_64(gIOHibernateHandoffPageCount), page_size);
1218
1219 if (!vars->srcBuffer || !vars->ioBuffer || !vars->handoffBuffer)
1220 {
1221 err = kIOReturnNoMemory;
1222 break;
1223 }
1224
1225 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey)))
1226 {
1227 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMinSize = num->unsigned64BitValue();
1228 obj->release();
1229 }
1230 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey)))
1231 {
1232 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMaxSize = num->unsigned64BitValue();
1233 obj->release();
1234 }
1235
1236 boolean_t encryptedswap = true;
1237 uint32_t pageCount;
1238 AbsoluteTime startTime, endTime;
1239 uint64_t nsec;
1240
1241 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1242 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
1243 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1244
1245 vmflush = (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey));
1246 uint64_t setFileSize = 0;
1247 err = hibernate_alloc_page_lists(&vars->page_list,
1248 &vars->page_list_wired,
1249 &vars->page_list_pal);
1250 if (KERN_SUCCESS != err)
1251 break;
1252
1253 if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode))
1254 {
1255 hibernate_page_list_setall(vars->page_list,
1256 vars->page_list_wired,
1257 vars->page_list_pal,
1258 true /* preflight */,
1259 vmflush /* discard */,
1260 &pageCount);
1261 PE_Video consoleInfo;
1262 bzero(&consoleInfo, sizeof(consoleInfo));
1263 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1264
1265 // estimate: 5% increase in pages compressed
1266 // screen preview 2 images compressed 50%
1267 setFileSize = ((ptoa_64((105 * pageCount) / 100) * gIOHibernateCompression) >> 8)
1268 + vars->page_list->list_size
1269 + (consoleInfo.v_width * consoleInfo.v_height * 4);
1270
1271 HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
1272 pageCount, (100ULL * gIOHibernateCompression) >> 8,
1273 setFileSize, vars->fileMinSize);
1274
1275 if (!(kIOHibernateModeFileResize & gIOHibernateMode)
1276 && (setFileSize < vars->fileMinSize))
1277 {
1278 setFileSize = vars->fileMinSize;
1279 }
1280 }
1281
1282 // open & invalidate the image file
1283
1284 err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, vars->ioBuffer,
1285 &vars->fileVars, &vars->fileExtents, &data,
1286 &vars->volumeCryptKey[0]);
1287 if (KERN_SUCCESS != err)
1288 {
1289 HIBLOG("IOPolledFileOpen(%x)\n", err);
1290 break;
1291 }
1292
1293 clock_get_uptime(&startTime);
1294 err = hibernate_setup(gIOHibernateCurrentHeader,
1295 gIOHibernateFreeRatio, gIOHibernateFreeTime,
1296 vmflush,
1297 vars->page_list, vars->page_list_wired, vars->page_list_pal);
1298 clock_get_uptime(&endTime);
1299 SUB_ABSOLUTETIME(&endTime, &startTime);
1300 absolutetime_to_nanoseconds(endTime, &nsec);
1301 HIBLOG("hibernate_setup(%d) took %qd ms\n", err, nsec / 1000000ULL);
1302
1303 dsSSD = ((0 != (kIOHibernateOptionSSD & vars->fileVars->flags))
1304 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
1305 if (dsSSD)
1306 {
1307 gIOHibernateCurrentHeader->options |=
1308 kIOHibernateOptionSSD
1309 | kIOHibernateOptionColor;
1310
1311 #if defined(__i386__) || defined(__x86_64__)
1312 if (!uuid_is_null(vars->volumeCryptKey) &&
1313 (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey)))
1314 {
1315 uintptr_t smcVars[2];
1316 smcVars[0] = sizeof(vars->volumeCryptKey);
1317 smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
1318
1319 IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
1320 bzero(smcVars, sizeof(smcVars));
1321 }
1322 #endif
1323 }
1324 else
1325 {
1326 gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
1327 }
1328
1329
1330 if (KERN_SUCCESS != err)
1331 break;
1332
1333 if (encryptedswap || !uuid_is_null(vars->volumeCryptKey))
1334 gIOHibernateMode ^= kIOHibernateModeEncrypt;
1335
1336 if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1337 {
1338 vars->videoAllocSize = kVideoMapSize;
1339 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
1340 vars->videoMapping = 0;
1341 }
1342
1343 // generate crypt keys
1344 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
1345 vars->wiredCryptKey[i] = random();
1346 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
1347 vars->cryptKey[i] = random();
1348
1349 // set nvram
1350
1351 IORegistryEntry * regEntry;
1352 if (!gIOOptionsEntry)
1353 {
1354 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
1355 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
1356 if (regEntry && !gIOOptionsEntry)
1357 regEntry->release();
1358 }
1359
1360 if (gIOOptionsEntry)
1361 {
1362 const OSSymbol * sym;
1363
1364 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1365 if (sym)
1366 {
1367 gIOOptionsEntry->setProperty(sym, data);
1368 sym->release();
1369 }
1370 data->release();
1371
1372 #if defined(__i386__) || defined(__x86_64__)
1373 struct AppleRTCHibernateVars
1374 {
1375 uint8_t signature[4];
1376 uint32_t revision;
1377 uint8_t booterSignature[20];
1378 uint8_t wiredCryptKey[16];
1379 };
1380 AppleRTCHibernateVars rtcVars;
1381
1382 rtcVars.signature[0] = 'A';
1383 rtcVars.signature[1] = 'A';
1384 rtcVars.signature[2] = 'P';
1385 rtcVars.signature[3] = 'L';
1386 rtcVars.revision = 1;
1387 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
1388 if (gIOHibernateBootSignature[0])
1389 {
1390 char c;
1391 uint8_t value = 0;
1392 for (uint32_t i = 0;
1393 (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
1394 i++)
1395 {
1396 if (c >= 'a')
1397 c -= 'a' - 10;
1398 else if (c >= 'A')
1399 c -= 'A' - 10;
1400 else if (c >= '0')
1401 c -= '0';
1402 else
1403 continue;
1404 value = (value << 4) | c;
1405 if (i & 1)
1406 rtcVars.booterSignature[i >> 1] = value;
1407 }
1408 }
1409 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
1410 if (data)
1411 {
1412 if (!gIOHibernateRTCVariablesKey)
1413 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1414 if (gIOHibernateRTCVariablesKey)
1415 IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
1416
1417 if( gIOOptionsEntry )
1418 {
1419 if( gIOHibernateMode & kIOHibernateModeSwitch )
1420 {
1421 const OSSymbol *sym;
1422 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSwitchVarsKey);
1423 if( sym )
1424 {
1425 gIOOptionsEntry->setProperty(sym, data); /* intentional insecure backup of rtc boot vars */
1426 sym->release();
1427 }
1428 }
1429 }
1430
1431 data->release();
1432 }
1433 if (gIOChosenEntry)
1434 {
1435 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
1436 if (data)
1437 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
1438 {
1439 // set BootNext
1440
1441 if (!gIOHibernateBoot0082Data)
1442 {
1443 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
1444 if (data)
1445 {
1446 // AppleNVRAM_EFI_LOAD_OPTION
1447 struct {
1448 uint32_t Attributes;
1449 uint16_t FilePathLength;
1450 uint16_t Desc;
1451 } loadOptionHeader;
1452 loadOptionHeader.Attributes = 1;
1453 loadOptionHeader.FilePathLength = data->getLength();
1454 loadOptionHeader.Desc = 0;
1455 gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
1456 if (gIOHibernateBoot0082Data)
1457 {
1458 gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
1459 gIOHibernateBoot0082Data->appendBytes(data);
1460 }
1461 }
1462 }
1463 if (!gIOHibernateBoot0082Key)
1464 gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
1465 if (!gIOHibernateBootNextKey)
1466 gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
1467 if (!gIOHibernateBootNextData)
1468 {
1469 uint16_t bits = 0x0082;
1470 gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
1471 }
1472 if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData)
1473 {
1474 gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
1475 gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
1476 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
1477 }
1478 }
1479 }
1480 #else /* !i386 && !x86_64 */
1481 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1482 {
1483 data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1484 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
1485 if (sym && data)
1486 gIOOptionsEntry->setProperty(sym, data);
1487 if (sym)
1488 sym->release();
1489 if (data)
1490 data->release();
1491 if (false && gIOHibernateBootSignature[0])
1492 {
1493 data = OSData::withCapacity(16);
1494 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
1495 if (sym && data)
1496 {
1497 char c;
1498 uint8_t value = 0;
1499 for (uint32_t i = 0; (c = gIOHibernateBootSignature[i]); i++)
1500 {
1501 if (c >= 'a')
1502 c -= 'a' - 10;
1503 else if (c >= 'A')
1504 c -= 'A' - 10;
1505 else if (c >= '0')
1506 c -= '0';
1507 else
1508 continue;
1509 value = (value << 4) | c;
1510 if (i & 1)
1511 data->appendBytes(&value, sizeof(value));
1512 }
1513 gIOOptionsEntry->setProperty(sym, data);
1514 }
1515 if (sym)
1516 sym->release();
1517 if (data)
1518 data->release();
1519 }
1520 }
1521 if (!vars->haveFastBoot)
1522 {
1523 // set boot volume to zero
1524 IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
1525 if (platform && (kIOReturnSuccess == platform->readXPRAM(kXPRamAudioVolume,
1526 &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume))))
1527 {
1528 uint8_t newVolume;
1529 newVolume = vars->saveBootAudioVolume & 0xf8;
1530 platform->writeXPRAM(kXPRamAudioVolume,
1531 &newVolume, sizeof(newVolume));
1532 }
1533 }
1534 #endif /* !i386 && !x86_64 */
1535 }
1536 // --
1537
1538 }
1539 while (false);
1540
1541 IOLockLock(gFSLock);
1542 if ((kIOReturnSuccess == err) && (kFSOpening == gFSState))
1543 {
1544 gFSState = kFSOpened;
1545 gIOHibernateVars = *vars;
1546 gFileVars = *vars->fileVars;
1547 gIOHibernateVars.fileVars = &gFileVars;
1548 gIOHibernateFileRef = gFileVars.fileRef;
1549 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
1550 gIOHibernateState = kIOHibernateStateHibernating;
1551 }
1552 else
1553 {
1554 HIBLOG("hibernate file close due timeout\n");
1555 if (vars->fileVars && vars->fileVars->fileRef) kern_close_file_for_direct_io(vars->fileVars->fileRef, 0, 0, 0, 0, 0);
1556 IOHibernateDone(vars);
1557 gFSState = kFSIdle;
1558 }
1559 IOLockUnlock(gFSLock);
1560
1561 if (vars->fileVars) IODelete(vars->fileVars, IOPolledFileIOVars, 1);
1562 IODelete(vars, IOHibernateVars, 1);
1563
1564 return (err);
1565 }
1566
1567 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1568
1569 DECLARE_IOHIBERNATEPROGRESSALPHA
1570
1571 static void
1572 ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
1573 {
1574 uint32_t rowBytes, pixelShift;
1575 uint32_t x, y;
1576 int32_t blob;
1577 uint32_t alpha, in, color, result;
1578 uint8_t * out;
1579 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1580
1581 rowBytes = display->rowBytes;
1582 pixelShift = display->depth >> 4;
1583 if (pixelShift < 1) return;
1584
1585 screen += ((display->width
1586 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1587 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1588
1589 for (y = 0; y < kIOHibernateProgressHeight; y++)
1590 {
1591 out = screen + y * rowBytes;
1592 for (blob = 0; blob < kIOHibernateProgressCount; blob++)
1593 {
1594 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
1595 for (x = 0; x < kIOHibernateProgressWidth; x++)
1596 {
1597 alpha = gIOHibernateProgressAlpha[y][x];
1598 result = color;
1599 if (alpha)
1600 {
1601 if (0xff != alpha)
1602 {
1603 if (1 == pixelShift)
1604 {
1605 in = *((uint16_t *)out) & 0x1f; // 16
1606 in = (in << 3) | (in >> 2);
1607 }
1608 else
1609 in = *((uint32_t *)out) & 0xff; // 32
1610 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
1611 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
1612 }
1613 if (1 == pixelShift)
1614 {
1615 result >>= 3;
1616 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1617 }
1618 else
1619 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1620 }
1621 out += (1 << pixelShift);
1622 }
1623 out += (kIOHibernateProgressSpacing << pixelShift);
1624 }
1625 }
1626 }
1627
1628
1629 static void
1630 ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
1631 {
1632 uint32_t rowBytes, pixelShift;
1633 uint32_t x, y;
1634 int32_t blob, lastBlob;
1635 uint32_t alpha, in, color, result;
1636 uint8_t * out;
1637 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1638
1639 pixelShift = display->depth >> 4;
1640 if (pixelShift < 1)
1641 return;
1642
1643 rowBytes = display->rowBytes;
1644
1645 screen += ((display->width
1646 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1647 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1648
1649 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
1650
1651 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
1652
1653 for (y = 0; y < kIOHibernateProgressHeight; y++)
1654 {
1655 out = screen + y * rowBytes;
1656 for (blob = firstBlob; blob <= lastBlob; blob++)
1657 {
1658 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
1659 for (x = 0; x < kIOHibernateProgressWidth; x++)
1660 {
1661 alpha = gIOHibernateProgressAlpha[y][x];
1662 result = color;
1663 if (alpha)
1664 {
1665 if (0xff != alpha)
1666 {
1667 in = display->progressSaveUnder[blob][saveindex[blob]++];
1668 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
1669 }
1670 if (1 == pixelShift)
1671 {
1672 result >>= 3;
1673 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1674 }
1675 else
1676 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1677 }
1678 out += (1 << pixelShift);
1679 }
1680 out += (kIOHibernateProgressSpacing << pixelShift);
1681 }
1682 }
1683 }
1684
1685 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1686
1687 IOReturn
1688 IOHibernateIOKitSleep(void)
1689 {
1690 IOReturn ret = kIOReturnSuccess;
1691 IOLockLock(gFSLock);
1692 if (kFSOpening == gFSState)
1693 {
1694 gFSState = kFSTimedOut;
1695 HIBLOG("hibernate file open timed out\n");
1696 ret = kIOReturnTimeout;
1697 }
1698 IOLockUnlock(gFSLock);
1699 return (ret);
1700 }
1701
1702 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1703
1704 IOReturn
1705 IOHibernateSystemHasSlept(void)
1706 {
1707 IOReturn ret = kIOReturnSuccess;
1708 IOHibernateVars * vars = &gIOHibernateVars;
1709 OSObject * obj = 0;
1710 OSData * data;
1711
1712 IOLockLock(gFSLock);
1713 if ((kFSOpened != gFSState) && gIOHibernateMode)
1714 {
1715 ret = kIOReturnTimeout;
1716 }
1717 IOLockUnlock(gFSLock);
1718 if (kIOReturnSuccess != ret) return (ret);
1719
1720 if (gIOHibernateMode) obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
1721 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1722 if (obj && !vars->previewBuffer)
1723 obj->release();
1724
1725 vars->consoleMapping = NULL;
1726 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
1727 {
1728 vars->previewBuffer->release();
1729 vars->previewBuffer = 0;
1730 }
1731
1732 if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1733 && vars->previewBuffer
1734 && (data = OSDynamicCast(OSData,
1735 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
1736 {
1737 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
1738 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
1739
1740 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1741
1742 if (kIOHibernatePreviewUpdates & flags)
1743 {
1744 PE_Video consoleInfo;
1745 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1746
1747 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1748
1749 graphicsInfo->width = consoleInfo.v_width;
1750 graphicsInfo->height = consoleInfo.v_height;
1751 graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
1752 graphicsInfo->depth = consoleInfo.v_depth;
1753 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1754
1755 HIBPRINT("video %p %d %d %d\n",
1756 vars->consoleMapping, graphicsInfo->depth,
1757 graphicsInfo->width, graphicsInfo->height);
1758 if (vars->consoleMapping)
1759 ProgressInit(graphicsInfo, vars->consoleMapping,
1760 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1761 }
1762 }
1763
1764 if (gIOOptionsEntry)
1765 gIOOptionsEntry->sync();
1766
1767 return (ret);
1768 }
1769
1770 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1771
1772 static DeviceTreeNode *
1773 MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry)
1774 {
1775 DeviceTreeNodeProperty * prop;
1776 DeviceTreeNode * child;
1777 IORegistryEntry * childRegEntry;
1778 const char * nameProp;
1779 unsigned int propLen, idx;
1780
1781 prop = (DeviceTreeNodeProperty *) (entry + 1);
1782 for (idx = 0; idx < entry->nProperties; idx++)
1783 {
1784 if (regEntry && (0 != strcmp("name", prop->name)))
1785 {
1786 regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
1787 // HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
1788 }
1789 prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
1790 }
1791
1792 child = (DeviceTreeNode *) prop;
1793 for (idx = 0; idx < entry->nChildren; idx++)
1794 {
1795 if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen))
1796 panic("no name");
1797 childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
1798 // HIBPRINT("%s == %p\n", nameProp, childRegEntry);
1799 child = MergeDeviceTree(child, childRegEntry);
1800 }
1801 return (child);
1802 }
1803
1804 IOReturn
1805 IOHibernateSystemWake(void)
1806 {
1807 if (kFSOpened == gFSState)
1808 {
1809 IOHibernateDone(&gIOHibernateVars);
1810 }
1811 else
1812 {
1813 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1814 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1815 }
1816 return (kIOReturnSuccess);
1817 }
1818
1819 static IOReturn
1820 IOHibernateDone(IOHibernateVars * vars)
1821 {
1822 IORegistryEntry * next;
1823
1824 hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
1825
1826 if (vars->videoMapping)
1827 {
1828 if (vars->videoMapSize)
1829 // remove mappings
1830 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1831 if (vars->videoAllocSize)
1832 // dealloc range
1833 kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
1834 }
1835
1836 if (vars->previewBuffer)
1837 {
1838 vars->previewBuffer->release();
1839 vars->previewBuffer = 0;
1840 }
1841
1842 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1843 {
1844 IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
1845 gIOHibernateCurrentHeader->options, 32);
1846 }
1847 else
1848 {
1849 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1850 }
1851
1852 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1853 && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus))
1854 {
1855 IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
1856 &gIOHibernateGraphicsInfo->gfxStatus,
1857 sizeof(gIOHibernateGraphicsInfo->gfxStatus));
1858 }
1859 else
1860 {
1861 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1862 }
1863
1864 if (vars->fileVars)
1865 {
1866 if ((next = vars->fileVars->media)) do
1867 {
1868 next->removeProperty(kIOPolledInterfaceActiveKey);
1869 next = next->getParentEntry(gIOServicePlane);
1870 }
1871 while (next);
1872 IOPolledFileClose(vars->fileVars);
1873 }
1874
1875 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1876
1877 #if defined(__i386__) || defined(__x86_64__)
1878 IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
1879 IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
1880
1881 /*
1882 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1883 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1884 */
1885 if (gIOOptionsEntry) {
1886
1887 if (gIOHibernateRTCVariablesKey) {
1888 if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
1889 gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
1890 }
1891 }
1892
1893 if (gIOHibernateBootNextKey)
1894 {
1895 if (gIOHibernateBootNextSave)
1896 {
1897 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
1898 gIOHibernateBootNextSave->release();
1899 gIOHibernateBootNextSave = NULL;
1900 }
1901 else
1902 gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
1903 }
1904 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) gIOOptionsEntry->sync();
1905 }
1906 #endif
1907
1908 if (vars->srcBuffer)
1909 vars->srcBuffer->release();
1910 if (vars->ioBuffer)
1911 vars->ioBuffer->release();
1912 bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
1913 if (vars->handoffBuffer)
1914 {
1915 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1916 {
1917 IOHibernateHandoff * handoff;
1918 bool done = false;
1919 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
1920 !done;
1921 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
1922 {
1923 HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
1924 uint8_t * data = &handoff->data[0];
1925 switch (handoff->type)
1926 {
1927 case kIOHibernateHandoffTypeEnd:
1928 done = true;
1929 break;
1930
1931 case kIOHibernateHandoffTypeDeviceTree:
1932 MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
1933 break;
1934
1935 case kIOHibernateHandoffTypeKeyStore:
1936 #if defined(__i386__) || defined(__x86_64__)
1937 {
1938 IOBufferMemoryDescriptor *
1939 md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
1940 if (md)
1941 {
1942 IOSetKeyStoreData(md);
1943 }
1944 }
1945 #endif
1946 break;
1947
1948 default:
1949 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
1950 break;
1951 }
1952 }
1953 }
1954 vars->handoffBuffer->release();
1955 }
1956 if (vars->fileExtents)
1957 vars->fileExtents->release();
1958
1959 bzero(vars, sizeof(*vars));
1960
1961 // gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1962
1963 return (kIOReturnSuccess);
1964 }
1965
1966 IOReturn
1967 IOHibernateSystemPostWake(void)
1968 {
1969 struct kern_direct_file_io_ref_t * fileRef;
1970
1971 if (kFSOpened == gFSState)
1972 {
1973 // invalidate & close the image file
1974 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1975 if ((fileRef = gIOHibernateFileRef))
1976 {
1977 gIOHibernateFileRef = 0;
1978 IOSleep(TRIM_DELAY);
1979 kern_close_file_for_direct_io(fileRef,
1980 #if DISABLE_TRIM
1981 0, 0, 0, 0, 0);
1982 #else
1983 0, (caddr_t) gIOHibernateCurrentHeader,
1984 sizeof(IOHibernateImageHeader),
1985 0,
1986 gIOHibernateCurrentHeader->imageSize);
1987 #endif
1988 }
1989 gFSState = kFSIdle;
1990 }
1991 return (kIOReturnSuccess);
1992 }
1993
1994 bool IOHibernateWasScreenLocked(void)
1995 {
1996 bool ret = false;
1997 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry)
1998 {
1999 OSData *
2000 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
2001 if (data) switch (*((uint32_t *)data->getBytesNoCopy()))
2002 {
2003 case kIOScreenLockLocked:
2004 case kIOScreenLockFileVaultDialog:
2005 ret = true;
2006 break;
2007 case kIOScreenLockNoLock:
2008 case kIOScreenLockUnlocked:
2009 default:
2010 ret = false;
2011 break;
2012 }
2013 }
2014 return (ret);
2015 }
2016
2017 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2018
2019 SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
2020 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2021 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
2022 SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
2023 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2024 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
2025 SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
2026 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2027 &gIOHibernateMode, 0, "");
2028 SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
2029 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2030 gIOHibernateStats, hibernate_statistics_t, "");
2031
2032 SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
2033 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
2034 &gIOHibernateStats->graphicsReadyTime, 0, "");
2035 SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
2036 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
2037 &gIOHibernateStats->wakeNotificationTime, 0, "");
2038 SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
2039 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
2040 &gIOHibernateStats->lockScreenReadyTime, 0, "");
2041 SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
2042 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
2043 &gIOHibernateStats->hidReadyTime, 0, "");
2044
2045
2046 void
2047 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
2048 {
2049 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
2050 if (data)
2051 {
2052 rootDomain->setProperty(kIOHibernateStateKey, data);
2053 data->release();
2054 }
2055
2056 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))
2057 gIOHibernateMode = kIOHibernateModeOn;
2058 else
2059 gIOHibernateFilename[0] = 0;
2060
2061 sysctl_register_oid(&sysctl__kern_hibernatefile);
2062 sysctl_register_oid(&sysctl__kern_bootsignature);
2063 sysctl_register_oid(&sysctl__kern_hibernatemode);
2064 sysctl_register_oid(&sysctl__kern_hibernatestatistics);
2065 sysctl_register_oid(&sysctl__kern_hibernategraphicsready);
2066 sysctl_register_oid(&sysctl__kern_hibernatewakenotification);
2067 sysctl_register_oid(&sysctl__kern_hibernatelockscreenready);
2068 sysctl_register_oid(&sysctl__kern_hibernatehidready);
2069
2070 gFSLock = IOLockAlloc();
2071 }
2072
2073
2074 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2075
2076 static void
2077 hibernate_setup_for_wake(void)
2078 {
2079 }
2080
2081 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2082
2083 #define C_ASSERT(e) typedef char __C_ASSERT__[(e) ? 1 : -1]
2084
2085 static bool
2086 no_encrypt_page(vm_offset_t ppnum)
2087 {
2088 if (pmap_is_noencrypt((ppnum_t)ppnum) == TRUE)
2089 {
2090 return true;
2091 }
2092 return false;
2093 }
2094
2095 static void
2096 hibernate_pal_callback(void *vars_arg, vm_offset_t addr)
2097 {
2098 IOHibernateVars *vars = (IOHibernateVars *)vars_arg;
2099 /* Make sure it's not in either of the save lists */
2100 hibernate_set_page_state(vars->page_list, vars->page_list_wired, atop_64(addr), 1, kIOHibernatePageStateFree);
2101
2102 /* Set it in the bitmap of pages owned by the PAL */
2103 hibernate_page_bitset(vars->page_list_pal, TRUE, atop_64(addr));
2104 }
2105
2106 static struct hibernate_cryptvars_t *local_cryptvars;
2107
2108 extern "C" int
2109 hibernate_pal_write(void *buffer, size_t size)
2110 {
2111 IOHibernateVars * vars = &gIOHibernateVars;
2112
2113 IOReturn err = IOPolledFileWrite(vars->fileVars, (const uint8_t *)buffer, size, local_cryptvars);
2114 if (kIOReturnSuccess != err) {
2115 kprintf("epic hibernate fail! %d\n", err);
2116 return err;
2117 }
2118
2119 return 0;
2120 }
2121
2122
2123 extern "C" uint32_t
2124 hibernate_write_image(void)
2125 {
2126 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
2127 IOHibernateVars * vars = &gIOHibernateVars;
2128 IOPolledFileExtent * fileExtents;
2129
2130 C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
2131
2132 uint32_t pageCount, pagesDone;
2133 IOReturn err;
2134 vm_offset_t ppnum, page;
2135 IOItemCount count;
2136 uint8_t * src;
2137 uint8_t * data;
2138 uint8_t * compressed;
2139 uint8_t * scratch;
2140 void * zerosCompressed;
2141 IOByteCount pageCompressedSize, zerosCompressedLen;
2142 uint64_t compressedSize, uncompressedSize;
2143 uint64_t image1Size = 0;
2144 uint32_t bitmap_size;
2145 bool iterDone, pollerOpen, needEncrypt;
2146 uint32_t restore1Sum, sum, sum1, sum2;
2147 int wkresult;
2148 uint32_t tag;
2149 uint32_t pageType;
2150 uint32_t pageAndCount[2];
2151 addr64_t phys64;
2152 IOByteCount segLen;
2153
2154 AbsoluteTime startTime, endTime;
2155 AbsoluteTime allTime, compTime;
2156 uint64_t compBytes;
2157 uint64_t nsec;
2158 uint32_t lastProgressStamp = 0;
2159 uint32_t progressStamp;
2160 uint32_t blob, lastBlob = (uint32_t) -1L;
2161
2162 uint32_t wiredPagesEncrypted;
2163 uint32_t dirtyPagesEncrypted;
2164 uint32_t wiredPagesClear;
2165 uint32_t zeroPageCount;
2166
2167 hibernate_cryptvars_t _cryptvars;
2168 hibernate_cryptvars_t * cryptvars = 0;
2169
2170 wiredPagesEncrypted = 0;
2171 dirtyPagesEncrypted = 0;
2172 wiredPagesClear = 0;
2173 zeroPageCount = 0;
2174
2175 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
2176 return (false /* sleep */ );
2177
2178 if (kIOHibernateModeSleep & gIOHibernateMode)
2179 kdebug_enable = save_kdebug_enable;
2180
2181 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START, 0, 0, 0, 0, 0);
2182 IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
2183
2184 restore1Sum = sum1 = sum2 = 0;
2185
2186 hibernate_pal_prepare();
2187
2188 #if CRYPTO
2189 // encryption data. "iv" is the "initial vector".
2190 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2191 {
2192 static const unsigned char first_iv[AES_BLOCK_SIZE]
2193 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
2194 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
2195
2196 cryptvars = &gIOHibernateCryptWakeContext;
2197 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
2198 aes_encrypt_key(vars->cryptKey,
2199 kIOHibernateAESKeySize,
2200 &cryptvars->ctx.encrypt);
2201 aes_decrypt_key(vars->cryptKey,
2202 kIOHibernateAESKeySize,
2203 &cryptvars->ctx.decrypt);
2204
2205 cryptvars = &_cryptvars;
2206 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
2207 for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++)
2208 vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
2209 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
2210 aes_encrypt_key(vars->wiredCryptKey,
2211 kIOHibernateAESKeySize,
2212 &cryptvars->ctx.encrypt);
2213
2214 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
2215 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
2216 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
2217
2218 local_cryptvars = cryptvars;
2219 }
2220 #endif /* CRYPTO */
2221
2222 hibernate_setup_for_wake();
2223
2224 hibernate_page_list_setall(vars->page_list,
2225 vars->page_list_wired,
2226 vars->page_list_pal,
2227 false /* !preflight */,
2228 /* discard_all */
2229 ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
2230 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
2231 &pageCount);
2232
2233 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
2234
2235 fileExtents = (IOPolledFileExtent *) vars->fileExtents->getBytesNoCopy();
2236
2237 #if 0
2238 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
2239 for (page = 0; page < count; page++)
2240 {
2241 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
2242 fileExtents[page].start, fileExtents[page].length,
2243 fileExtents[page].start + fileExtents[page].length);
2244 }
2245 #endif
2246
2247 needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
2248 AbsoluteTime_to_scalar(&compTime) = 0;
2249 compBytes = 0;
2250
2251 clock_get_uptime(&allTime);
2252 IOService::getPMRootDomain()->pmStatsRecordEvent(
2253 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
2254 do
2255 {
2256 compressedSize = 0;
2257 uncompressedSize = 0;
2258 zeroPageCount = 0;
2259
2260 IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
2261
2262 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
2263 ml_get_interrupts_enabled());
2264 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledBeforeSleepState, vars->ioBuffer);
2265 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
2266 pollerOpen = (kIOReturnSuccess == err);
2267 if (!pollerOpen)
2268 break;
2269
2270 // copy file block extent list if larger than header
2271
2272 count = vars->fileExtents->getLength();
2273 if (count > sizeof(header->fileExtentMap))
2274 {
2275 count -= sizeof(header->fileExtentMap);
2276 err = IOPolledFileWrite(vars->fileVars,
2277 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
2278 if (kIOReturnSuccess != err)
2279 break;
2280 }
2281
2282 uintptr_t hibernateBase;
2283 uintptr_t hibernateEnd;
2284
2285 hibernateBase = HIB_BASE; /* Defined in PAL headers */
2286
2287 hibernateEnd = (segHIBB + segSizeHIB);
2288
2289 // copy out restore1 code
2290
2291 for (count = 0;
2292 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2293 count += segLen)
2294 {
2295 for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++)
2296 {
2297 gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
2298 }
2299 }
2300
2301 page = atop_32(kvtophys(hibernateBase));
2302 count = atop_32(round_page(hibernateEnd) - hibernateBase);
2303 header->restore1CodePhysPage = page;
2304 header->restore1CodeVirt = hibernateBase;
2305 header->restore1PageCount = count;
2306 header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
2307 header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
2308
2309 // sum __HIB seg, with zeros for the stack
2310 src = (uint8_t *) trunc_page(hibernateBase);
2311 for (page = 0; page < count; page++)
2312 {
2313 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
2314 restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
2315 else
2316 restore1Sum += 0x00000000;
2317 src += page_size;
2318 }
2319 sum1 = restore1Sum;
2320
2321 // write the __HIB seg, with zeros for the stack
2322
2323 src = (uint8_t *) trunc_page(hibernateBase);
2324 count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
2325 if (count)
2326 {
2327 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
2328 if (kIOReturnSuccess != err)
2329 break;
2330 }
2331 err = IOPolledFileWrite(vars->fileVars,
2332 (uint8_t *) 0,
2333 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
2334 cryptvars);
2335 if (kIOReturnSuccess != err)
2336 break;
2337 src = &gIOHibernateRestoreStackEnd[0];
2338 count = round_page(hibernateEnd) - ((uintptr_t) src);
2339 if (count)
2340 {
2341 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
2342 if (kIOReturnSuccess != err)
2343 break;
2344 }
2345
2346 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2347 {
2348 vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
2349 vars->fileVars->encryptEnd = UINT64_MAX;
2350 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
2351 }
2352
2353 // write the preview buffer
2354
2355 if (vars->previewBuffer)
2356 {
2357 ppnum = 0;
2358 count = 0;
2359 do
2360 {
2361 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
2362 pageAndCount[0] = atop_64(phys64);
2363 pageAndCount[1] = atop_32(segLen);
2364 err = IOPolledFileWrite(vars->fileVars,
2365 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
2366 cryptvars);
2367 if (kIOReturnSuccess != err)
2368 break;
2369 count += segLen;
2370 ppnum += sizeof(pageAndCount);
2371 }
2372 while (phys64);
2373 if (kIOReturnSuccess != err)
2374 break;
2375
2376 src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
2377
2378 ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
2379
2380 count = vars->previewBuffer->getLength();
2381
2382 header->previewPageListSize = ppnum;
2383 header->previewSize = count + ppnum;
2384
2385 for (page = 0; page < count; page += page_size)
2386 {
2387 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
2388 sum1 += hibernate_sum_page(src + page, atop_64(phys64));
2389 }
2390 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
2391 if (kIOReturnSuccess != err)
2392 break;
2393 }
2394
2395 // mark areas for no save
2396
2397 for (count = 0;
2398 (phys64 = vars->ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2399 count += segLen)
2400 {
2401 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2402 atop_64(phys64), atop_32(segLen),
2403 kIOHibernatePageStateFree);
2404 pageCount -= atop_32(segLen);
2405 }
2406
2407 for (count = 0;
2408 (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2409 count += segLen)
2410 {
2411 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2412 atop_64(phys64), atop_32(segLen),
2413 kIOHibernatePageStateFree);
2414 pageCount -= atop_32(segLen);
2415 }
2416
2417 // copy out bitmap of pages available for trashing during restore
2418
2419 bitmap_size = vars->page_list_wired->list_size;
2420 src = (uint8_t *) vars->page_list_wired;
2421 err = IOPolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
2422 if (kIOReturnSuccess != err)
2423 break;
2424
2425 // mark more areas for no save, but these are not available
2426 // for trashing during restore
2427
2428 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
2429
2430
2431 page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
2432 count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
2433 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2434 page, count,
2435 kIOHibernatePageStateFree);
2436 pageCount -= count;
2437
2438 if (vars->previewBuffer) for (count = 0;
2439 (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2440 count += segLen)
2441 {
2442 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2443 atop_64(phys64), atop_32(segLen),
2444 kIOHibernatePageStateFree);
2445 pageCount -= atop_32(segLen);
2446 }
2447
2448 for (count = 0;
2449 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2450 count += segLen)
2451 {
2452 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2453 atop_64(phys64), atop_32(segLen),
2454 kIOHibernatePageStateFree);
2455 pageCount -= atop_32(segLen);
2456 }
2457
2458 (void)hibernate_pal_callback;
2459
2460 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2461 compressed = src + page_size;
2462 scratch = compressed + page_size;
2463
2464 // compress a zero page
2465 bzero(src, page_size);
2466 zerosCompressed = vars->handoffBuffer->getBytesNoCopy();
2467 zerosCompressedLen = WKdm_compress_new((WK_word*) src,
2468 (WK_word*) zerosCompressed,
2469 (WK_word*) scratch,
2470 page_size - 4);
2471
2472 pagesDone = 0;
2473 lastBlob = 0;
2474
2475 HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
2476 bitmap_size, header->previewSize,
2477 pageCount, vars->fileVars->position);
2478
2479 enum
2480 // pageType
2481 {
2482 kWired = 0x02,
2483 kEncrypt = 0x01,
2484 kWiredEncrypt = kWired | kEncrypt,
2485 kWiredClear = kWired,
2486 kUnwiredEncrypt = kEncrypt
2487 };
2488
2489 for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)
2490 {
2491 if (kUnwiredEncrypt == pageType)
2492 {
2493 // start unwired image
2494 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2495 {
2496 vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
2497 vars->fileVars->encryptEnd = UINT64_MAX;
2498 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
2499 }
2500 bcopy(&cryptvars->aes_iv[0],
2501 &gIOHibernateCryptWakeContext.aes_iv[0],
2502 sizeof(cryptvars->aes_iv));
2503 cryptvars = &gIOHibernateCryptWakeContext;
2504 }
2505 for (iterDone = false, ppnum = 0; !iterDone; )
2506 {
2507 count = hibernate_page_list_iterate((kWired & pageType)
2508 ? vars->page_list_wired : vars->page_list,
2509 &ppnum);
2510 // kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
2511 iterDone = !count;
2512
2513 if (count && (kWired & pageType) && needEncrypt)
2514 {
2515 uint32_t checkIndex;
2516 for (checkIndex = 0;
2517 (checkIndex < count)
2518 && (((kEncrypt & pageType) == 0) == no_encrypt_page(ppnum + checkIndex));
2519 checkIndex++)
2520 {}
2521 if (!checkIndex)
2522 {
2523 ppnum++;
2524 continue;
2525 }
2526 count = checkIndex;
2527 }
2528
2529 switch (pageType)
2530 {
2531 case kWiredEncrypt: wiredPagesEncrypted += count; break;
2532 case kWiredClear: wiredPagesClear += count; break;
2533 case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
2534 }
2535
2536 if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */}
2537 else
2538 {
2539 pageAndCount[0] = ppnum;
2540 pageAndCount[1] = count;
2541 err = IOPolledFileWrite(vars->fileVars,
2542 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
2543 cryptvars);
2544 if (kIOReturnSuccess != err)
2545 break;
2546 }
2547
2548 for (page = ppnum; page < (ppnum + count); page++)
2549 {
2550 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
2551 if (err)
2552 {
2553 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
2554 break;
2555 }
2556
2557 sum = hibernate_sum_page(src, page);
2558 if (kWired & pageType)
2559 sum1 += sum;
2560 else
2561 sum2 += sum;
2562
2563 clock_get_uptime(&startTime);
2564 wkresult = WKdm_compress_new((WK_word*) src,
2565 (WK_word*) compressed,
2566 (WK_word*) scratch,
2567 page_size - 4);
2568
2569 clock_get_uptime(&endTime);
2570 ADD_ABSOLUTETIME(&compTime, &endTime);
2571 SUB_ABSOLUTETIME(&compTime, &startTime);
2572
2573 compBytes += page_size;
2574 pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
2575
2576 if ((pageCompressedSize == zerosCompressedLen)
2577 && !bcmp(compressed, zerosCompressed, zerosCompressedLen))
2578 {
2579 pageCompressedSize = 0;
2580 zeroPageCount++;
2581 }
2582
2583 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2584 pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
2585
2586 if (pageCompressedSize != page_size)
2587 data = compressed;
2588 else
2589 data = src;
2590
2591 tag = pageCompressedSize | kIOHibernateTagSignature;
2592 err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
2593 if (kIOReturnSuccess != err)
2594 break;
2595
2596 err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
2597 if (kIOReturnSuccess != err)
2598 break;
2599
2600 compressedSize += pageCompressedSize;
2601 uncompressedSize += page_size;
2602 pagesDone++;
2603
2604 if (vars->consoleMapping && (0 == (1023 & pagesDone)))
2605 {
2606 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
2607 if (blob != lastBlob)
2608 {
2609 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
2610 lastBlob = blob;
2611 }
2612 }
2613 if (0 == (8191 & pagesDone))
2614 {
2615 clock_get_uptime(&endTime);
2616 SUB_ABSOLUTETIME(&endTime, &allTime);
2617 absolutetime_to_nanoseconds(endTime, &nsec);
2618 progressStamp = nsec / 750000000ULL;
2619 if (progressStamp != lastProgressStamp)
2620 {
2621 lastProgressStamp = progressStamp;
2622 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
2623 }
2624 }
2625 }
2626 if (kIOReturnSuccess != err)
2627 break;
2628 ppnum = page;
2629 }
2630
2631 if (kIOReturnSuccess != err)
2632 break;
2633
2634 if ((kEncrypt & pageType) && vars->fileVars->encryptStart)
2635 {
2636 vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
2637 HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
2638 }
2639
2640 if (kWiredEncrypt != pageType)
2641 {
2642 // end of image1/2 - fill to next block
2643 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2644 if (kIOReturnSuccess != err)
2645 break;
2646 }
2647 if (kWiredClear == pageType)
2648 {
2649 // enlarge wired image for test
2650 // err = IOPolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
2651
2652 // end wired image
2653 header->encryptStart = vars->fileVars->encryptStart;
2654 header->encryptEnd = vars->fileVars->encryptEnd;
2655 image1Size = vars->fileVars->position;
2656 HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
2657 image1Size, header->encryptStart, header->encryptEnd);
2658 }
2659 }
2660 if (kIOReturnSuccess != err)
2661 break;
2662
2663 // Header:
2664
2665 header->imageSize = vars->fileVars->position;
2666 header->image1Size = image1Size;
2667 header->bitmapSize = bitmap_size;
2668 header->pageCount = pageCount;
2669
2670 header->restore1Sum = restore1Sum;
2671 header->image1Sum = sum1;
2672 header->image2Sum = sum2;
2673 header->sleepTime = gIOLastSleepTime.tv_sec;
2674
2675 header->compression = (compressedSize << 8) / uncompressedSize;
2676 gIOHibernateCompression = header->compression;
2677
2678 count = vars->fileExtents->getLength();
2679 if (count > sizeof(header->fileExtentMap))
2680 {
2681 header->fileExtentMapSize = count;
2682 count = sizeof(header->fileExtentMap);
2683 }
2684 else
2685 header->fileExtentMapSize = sizeof(header->fileExtentMap);
2686 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
2687
2688 header->deviceBase = vars->fileVars->block0;
2689 header->deviceBlockSize = vars->fileVars->blockSize;
2690
2691 IOPolledFileSeek(vars->fileVars, 0);
2692 err = IOPolledFileWrite(vars->fileVars,
2693 (uint8_t *) header, sizeof(IOHibernateImageHeader),
2694 cryptvars);
2695 if (kIOReturnSuccess != err)
2696 break;
2697 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2698 if (kIOReturnSuccess != err)
2699 break;
2700 err = IOHibernatePollerIODone(vars->fileVars, true);
2701 if (kIOReturnSuccess != err)
2702 break;
2703 }
2704 while (false);
2705
2706 clock_get_uptime(&endTime);
2707
2708 IOService::getPMRootDomain()->pmStatsRecordEvent(
2709 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
2710
2711 SUB_ABSOLUTETIME(&endTime, &allTime);
2712 absolutetime_to_nanoseconds(endTime, &nsec);
2713 HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
2714
2715 absolutetime_to_nanoseconds(compTime, &nsec);
2716 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2717 compBytes,
2718 nsec / 1000000ULL,
2719 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2720
2721 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2722 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
2723 vars->fileVars->cryptBytes,
2724 nsec / 1000000ULL,
2725 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2726
2727 HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
2728 header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
2729 uncompressedSize, atop_32(uncompressedSize), compressedSize,
2730 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
2731 sum1, sum2);
2732
2733 HIBLOG("zeroPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
2734 zeroPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
2735
2736 if (vars->fileVars->io)
2737 (void) IOHibernatePollerIODone(vars->fileVars, false);
2738
2739 if (pollerOpen)
2740 IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
2741
2742 if (vars->consoleMapping)
2743 ProgressUpdate(gIOHibernateGraphicsInfo,
2744 vars->consoleMapping, 0, kIOHibernateProgressCount);
2745
2746 HIBLOG("hibernate_write_image done(%x)\n", err);
2747
2748 // should we come back via regular wake, set the state in memory.
2749 gIOHibernateState = kIOHibernateStateInactive;
2750
2751 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END,
2752 wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted, 0, 0);
2753
2754 if (kIOReturnSuccess == err)
2755 {
2756 if (kIOHibernateModeSleep & gIOHibernateMode)
2757 {
2758 return (kIOHibernatePostWriteSleep);
2759 }
2760 else if(kIOHibernateModeRestart & gIOHibernateMode)
2761 {
2762 return (kIOHibernatePostWriteRestart);
2763 }
2764 else
2765 {
2766 /* by default, power down */
2767 return (kIOHibernatePostWriteHalt);
2768 }
2769 }
2770 else if (kIOReturnAborted == err)
2771 {
2772 return (kIOHibernatePostWriteWake);
2773 }
2774 else
2775 {
2776 /* on error, sleep */
2777 return (kIOHibernatePostWriteSleep);
2778 }
2779 }
2780
2781 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2782
2783 extern "C" void
2784 hibernate_machine_init(void)
2785 {
2786 IOReturn err;
2787 uint32_t sum;
2788 uint32_t pagesDone;
2789 uint32_t pagesRead = 0;
2790 AbsoluteTime startTime, compTime;
2791 AbsoluteTime allTime, endTime;
2792 AbsoluteTime startIOTime, endIOTime;
2793 uint64_t nsec, nsecIO;
2794 uint64_t compBytes;
2795 uint32_t lastProgressStamp = 0;
2796 uint32_t progressStamp;
2797 hibernate_cryptvars_t * cryptvars = 0;
2798
2799 IOHibernateVars * vars = &gIOHibernateVars;
2800 bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
2801
2802 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
2803 return;
2804
2805 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2806 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2807
2808 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
2809 {
2810 HIBLOG("regular wake\n");
2811 return;
2812 }
2813
2814 HIBPRINT("diag %x %x %x %x\n",
2815 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
2816 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
2817
2818 #define t40ms(x) (tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)
2819 #define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
2820 tStat(booterStart, booterStart);
2821 gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart,
2822 tStat(booterDuration0, booterTime0);
2823 tStat(booterDuration1, booterTime1);
2824 tStat(booterDuration2, booterTime2);
2825 tStat(booterDuration, booterTime);
2826 tStat(booterConnectDisplayDuration, connectDisplayTime);
2827 tStat(booterSplashDuration, splashTime);
2828 tStat(trampolineDuration, trampolineTime);
2829
2830 gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size;
2831 gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize;
2832 gIOHibernateStats->image1Pages = pagesDone;
2833
2834 HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
2835 gIOHibernateStats->booterStart,
2836 gIOHibernateStats->smcStart,
2837 gIOHibernateStats->booterDuration0,
2838 gIOHibernateStats->booterDuration1,
2839 gIOHibernateStats->booterDuration2,
2840 gIOHibernateStats->booterDuration,
2841 gIOHibernateStats->booterConnectDisplayDuration,
2842 gIOHibernateStats->booterSplashDuration,
2843 gIOHibernateStats->trampolineDuration);
2844
2845 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
2846 gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
2847 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
2848
2849 if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
2850 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)))
2851 {
2852 hibernate_page_list_discard(vars->page_list);
2853 }
2854
2855 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
2856
2857 if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount)
2858 panic("handoff overflow");
2859
2860 IOHibernateHandoff * handoff;
2861 bool done = false;
2862 bool foundCryptData = false;
2863
2864 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
2865 !done;
2866 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
2867 {
2868 // HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
2869 uint8_t * data = &handoff->data[0];
2870 switch (handoff->type)
2871 {
2872 case kIOHibernateHandoffTypeEnd:
2873 done = true;
2874 break;
2875
2876 case kIOHibernateHandoffTypeGraphicsInfo:
2877 bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
2878 break;
2879
2880 case kIOHibernateHandoffTypeCryptVars:
2881 if (cryptvars)
2882 {
2883 hibernate_cryptwakevars_t *
2884 wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
2885 bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
2886 }
2887 foundCryptData = true;
2888 bzero(data, handoff->bytecount);
2889 break;
2890
2891 case kIOHibernateHandoffTypeMemoryMap:
2892
2893 clock_get_uptime(&allTime);
2894
2895 hibernate_newruntime_map(data, handoff->bytecount,
2896 gIOHibernateCurrentHeader->systemTableOffset);
2897
2898 clock_get_uptime(&endTime);
2899
2900 SUB_ABSOLUTETIME(&endTime, &allTime);
2901 absolutetime_to_nanoseconds(endTime, &nsec);
2902
2903 HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
2904
2905 break;
2906
2907 case kIOHibernateHandoffTypeDeviceTree:
2908 {
2909 // DTEntry chosen = NULL;
2910 // HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen));
2911 }
2912 break;
2913
2914 default:
2915 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
2916 break;
2917 }
2918 }
2919 if (cryptvars && !foundCryptData)
2920 panic("hibernate handoff");
2921
2922 HIBPRINT("video %x %d %d %d status %x\n",
2923 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2924 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
2925
2926 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)
2927 {
2928 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2929 * gIOHibernateGraphicsInfo->rowBytes);
2930 if (vars->videoMapSize > vars->videoAllocSize) vars->videoMapSize = 0;
2931 else
2932 {
2933 IOMapPages(kernel_map,
2934 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2935 vars->videoMapSize, kIOMapInhibitCache );
2936 }
2937 }
2938
2939 if (vars->videoMapSize)
2940 ProgressUpdate(gIOHibernateGraphicsInfo,
2941 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2942
2943 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2944 uint8_t * compressed = src + page_size;
2945 uint8_t * scratch = compressed + page_size;
2946 uint32_t decoOffset;
2947
2948 clock_get_uptime(&allTime);
2949 AbsoluteTime_to_scalar(&compTime) = 0;
2950 compBytes = 0;
2951
2952 HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2953 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0);
2954 clock_get_uptime(&startIOTime);
2955 endTime = startIOTime;
2956 SUB_ABSOLUTETIME(&endTime, &allTime);
2957 absolutetime_to_nanoseconds(endTime, &nsec);
2958 HIBLOG("IOHibernatePollerOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
2959
2960 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2961
2962 // kick off the read ahead
2963 vars->fileVars->io = false;
2964 vars->fileVars->bufferHalf = 0;
2965 vars->fileVars->bufferLimit = 0;
2966 vars->fileVars->lastRead = 0;
2967 vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
2968 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2969 vars->fileVars->cryptBytes = 0;
2970 AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
2971
2972 err = IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
2973 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2974 // --
2975
2976 HIBLOG("hibernate_machine_init reading\n");
2977
2978 uint32_t * header = (uint32_t *) src;
2979 sum = 0;
2980
2981 while (kIOReturnSuccess == err)
2982 {
2983 unsigned int count;
2984 unsigned int page;
2985 uint32_t tag;
2986 vm_offset_t ppnum, compressedSize;
2987
2988 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2989 if (kIOReturnSuccess != err)
2990 break;
2991
2992 ppnum = header[0];
2993 count = header[1];
2994
2995 // HIBPRINT("(%x, %x)\n", ppnum, count);
2996
2997 if (!count)
2998 break;
2999
3000 for (page = 0; page < count; page++)
3001 {
3002 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
3003 if (kIOReturnSuccess != err)
3004 break;
3005
3006 compressedSize = kIOHibernateTagLength & tag;
3007 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength))
3008 {
3009 err = kIOReturnIPCError;
3010 break;
3011 }
3012
3013 if (!compressedSize) bzero_phys(ptoa_64(ppnum), page_size);
3014 else
3015 {
3016 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
3017 if (kIOReturnSuccess != err) break;
3018 if (compressedSize < page_size)
3019 {
3020 decoOffset = page_size;
3021 clock_get_uptime(&startTime);
3022 WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, page_size);
3023 clock_get_uptime(&endTime);
3024 ADD_ABSOLUTETIME(&compTime, &endTime);
3025 SUB_ABSOLUTETIME(&compTime, &startTime);
3026 compBytes += page_size;
3027 }
3028 else decoOffset = 0;
3029
3030 sum += hibernate_sum_page((src + decoOffset), ppnum);
3031 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
3032 if (err)
3033 {
3034 HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
3035 break;
3036 }
3037 }
3038
3039 ppnum++;
3040 pagesDone++;
3041 pagesRead++;
3042
3043 if (0 == (8191 & pagesDone))
3044 {
3045 clock_get_uptime(&endTime);
3046 SUB_ABSOLUTETIME(&endTime, &allTime);
3047 absolutetime_to_nanoseconds(endTime, &nsec);
3048 progressStamp = nsec / 750000000ULL;
3049 if (progressStamp != lastProgressStamp)
3050 {
3051 lastProgressStamp = progressStamp;
3052 HIBPRINT("pages %d (%d%%)\n", pagesDone,
3053 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
3054 }
3055 }
3056 }
3057 }
3058 if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages))
3059 err = kIOReturnLockedRead;
3060
3061 if (kIOReturnSuccess != err)
3062 panic("Hibernate restore error %x", err);
3063
3064 gIOHibernateCurrentHeader->actualImage2Sum = sum;
3065 gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
3066
3067 if (vars->fileVars->io)
3068 (void) IOHibernatePollerIODone(vars->fileVars, false);
3069
3070 clock_get_uptime(&endIOTime);
3071
3072 err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
3073
3074 clock_get_uptime(&endTime);
3075
3076 IOService::getPMRootDomain()->pmStatsRecordEvent(
3077 kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
3078 IOService::getPMRootDomain()->pmStatsRecordEvent(
3079 kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
3080
3081 SUB_ABSOLUTETIME(&endTime, &allTime);
3082 absolutetime_to_nanoseconds(endTime, &nsec);
3083
3084 SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
3085 absolutetime_to_nanoseconds(endIOTime, &nsecIO);
3086
3087 gIOHibernateStats->kernelImageReadDuration = nsec / 1000000ULL;
3088 gIOHibernateStats->imagePages = pagesDone;
3089
3090 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
3091 pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
3092 nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
3093
3094 absolutetime_to_nanoseconds(compTime, &nsec);
3095 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
3096 compBytes,
3097 nsec / 1000000ULL,
3098 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
3099
3100 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
3101 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
3102 vars->fileVars->cryptBytes,
3103 nsec / 1000000ULL,
3104 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
3105
3106 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_NONE, pagesRead, pagesDone, 0, 0, 0);
3107 }
3108
3109 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
3110
3111 void IOHibernateSetWakeCapabilities(uint32_t capability)
3112 {
3113 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
3114 {
3115 gIOHibernateStats->wakeCapability = capability;
3116 }
3117 }
3118
3119 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
3120
3121 void IOHibernateSystemRestart(void)
3122 {
3123 static uint8_t noteStore[32] __attribute__((aligned(32)));
3124 IORegistryEntry * regEntry;
3125 const OSSymbol * sym;
3126 OSData * noteProp;
3127 OSData * data;
3128 uintptr_t * smcVars;
3129 uint8_t * smcBytes;
3130 size_t len;
3131 addr64_t element;
3132
3133 data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
3134 if (!data) return;
3135
3136 smcVars = (typeof(smcVars)) data->getBytesNoCopy();
3137 smcBytes = (typeof(smcBytes)) smcVars[1];
3138 len = smcVars[0];
3139 if (len > sizeof(noteStore)) len = sizeof(noteStore);
3140 noteProp = OSData::withCapacity(3 * sizeof(element));
3141 if (!noteProp) return;
3142 element = len;
3143 noteProp->appendBytes(&element, sizeof(element));
3144 element = crc32(0, smcBytes, len);
3145 noteProp->appendBytes(&element, sizeof(element));
3146
3147 bcopy(smcBytes, noteStore, len);
3148 element = (addr64_t) &noteStore[0];
3149 element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
3150 noteProp->appendBytes(&element, sizeof(element));
3151
3152 if (!gIOOptionsEntry)
3153 {
3154 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
3155 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
3156 if (regEntry && !gIOOptionsEntry)
3157 regEntry->release();
3158 }
3159
3160 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
3161 if (gIOOptionsEntry && sym) gIOOptionsEntry->setProperty(sym, noteProp);
3162 if (noteProp) noteProp->release();
3163 if (sym) sym->release();
3164 }
3165
3166
3167
mirror of XNU