]> git.saurik.com Git - apple/xnu.git/blob - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blob
? search:


83ab5e70320179cb38d0dfcf3c571b07debedeb5
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
1 /*
2 * Copyright (c) 2004-2008 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29
30 /*
31
32 Sleep:
33
34 - PMRootDomain calls IOHibernateSystemSleep() before system sleep
35 (devices awake, normal execution context)
36 - IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
37 grabs its extents and searches for a polling driver willing to work with that IOMedia.
38 The BSD code makes an ioctl to the storage driver to get the partition base offset to
39 the disk, and other ioctls to get the transfer constraints
40 If successful, the file is written to make sure its initially not bootable (in case of
41 later failure) and nvram set to point to the first block of the file. (Has to be done
42 here so blocking is possible in nvram support).
43 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
44 page out any pages it wants to (currently zero, but probably some percentage of memory).
45 Its assumed just allocating pages will cause the VM system to naturally select the best
46 pages for eviction. It also copies processor flags needed for the restore path and sets
47 a flag in the boot processor proc info.
48 gIOHibernateState = kIOHibernateStateHibernating.
49 - Regular sleep progresses - some drivers may inspect the root domain property
50 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
51 as usual but leaves motherboard I/O on.
52 - Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
53 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
54 all ppc RC bits out of the hash table and caches into the mapping structures.
55 - hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
56 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
57 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
58 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
59 The image header and block list are written. The header includes the second file extent so
60 only the header block is needed to read the file, regardless of filesystem.
61 The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
62 (only) is used to decompress the image during wake/boot.
63 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
64 The bitmaps are written to the image.
65 More areas are removed from the bitmaps (after they have been written to the image) - the
66 segment "__HIB" pages and interrupt stack.
67 Each wired page is compressed and written and then each non-wired page. Compression and
68 disk writes are in parallel.
69 The image header is written to the start of the file and the polling driver closed.
70 The machine powers down (or sleeps).
71
72 Boot/Wake:
73
74 - BootX sees the boot-image nvram variable containing the device and block number of the image,
75 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
76 - BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
77 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
78 that is in the kernel's __HIB section.
79 - BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
80 hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
81 only code & data in that section is safe to call since all the other wired pages are still
82 compressed in the image.
83 - hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
84 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
85 location directly, and copies those that can't to interim free pages. When the image has been
86 completed, the copies are uncompressed, overwriting the wired image pages.
87 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
88 is used to get pages into place for 64bit.
89 - the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
90 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
91 are removed from the software strutures, and the hash table is reinitialized.
92 - After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
93 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
94 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
95 for the remaining non wired pages.
96 - The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
97 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
98 is closed via bsd.
99
100 Polled Mode I/O:
101
102 IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
103 registry, specifying an object of calls IOPolledInterface.
104
105 Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
106 partition) that the image is going to live, looking for polled interface properties. If it finds
107 one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
108 interfaces found are kept in an ordered list.
109
110 There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
111 few different contexts things happen in:
112
113 - there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
114 up and running) and after wake - this is safe to allocate memory and do anything. The device
115 ignores sleep requests from that point since its a waste of time if it goes to sleep and
116 immediately wakes back up for the image write.
117
118 - there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
119 immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
120 from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
121 used to flush and set the disk to sleep.
122
123 - there is an Open/Close (AfterSleep) pair made around the image read operations that happen
124 immediately after sleep. These can't block or allocate memory. This is happening after the platform
125 expert has woken the low level bits of the system, but most of the I/O system has not. There is only
126 one thread running.
127
128 For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
129 (prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
130 the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
131 Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
132 that is called for the hardware to check for events, and complete the I/O via the callback.
133 The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
134 to restrict I/O ops.
135 */
136
137 #include <sys/systm.h>
138
139 #include <IOKit/IOWorkLoop.h>
140 #include <IOKit/IOCommandGate.h>
141 #include <IOKit/IOTimerEventSource.h>
142 #include <IOKit/IOPlatformExpert.h>
143 #include <IOKit/IOKitDebug.h>
144 #include <IOKit/IOTimeStamp.h>
145 #include <IOKit/pwr_mgt/RootDomain.h>
146 #include <IOKit/pwr_mgt/IOPMPrivate.h>
147 #include <IOKit/IOMessage.h>
148 #include <IOKit/IODeviceTreeSupport.h>
149 #include <IOKit/IOBSD.h>
150 #include "RootDomainUserClient.h"
151 #include <IOKit/pwr_mgt/IOPowerConnection.h>
152 #include "IOPMPowerStateQueue.h"
153 #include <IOKit/IOBufferMemoryDescriptor.h>
154 #include <IOKit/AppleKeyStoreInterface.h>
155 #include <libkern/crypto/aes.h>
156
157 #include <sys/uio.h>
158 #include <sys/conf.h>
159 #include <sys/stat.h>
160 #include <sys/fcntl.h> // (FWRITE, ...)
161 #include <sys/sysctl.h>
162 #include <sys/kdebug.h>
163
164 #include <IOKit/IOHibernatePrivate.h>
165 #include <IOKit/IOPolledInterface.h>
166 #include <IOKit/IONVRAM.h>
167 #include "IOHibernateInternal.h"
168 #include <vm/WKdm_new.h>
169 #include "IOKitKernelInternal.h"
170 #include <pexpert/device_tree.h>
171
172 #include <machine/pal_routines.h>
173 #include <machine/pal_hibernate.h>
174 #include <i386/tsc.h>
175
176 extern "C" addr64_t kvtophys(vm_offset_t va);
177 extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
178
179 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
180
181 #define DISABLE_TRIM 0
182 #define TRIM_DELAY 5000
183
184 extern unsigned int save_kdebug_enable;
185 extern uint32_t gIOHibernateState;
186 uint32_t gIOHibernateMode;
187 static char gIOHibernateBootSignature[256+1];
188 static char gIOHibernateFilename[MAXPATHLEN+1];
189 static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
190 uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
191 static uint64_t gIOHibernateCompression = 0x80; // default compression 50%
192
193 static IODTNVRAM * gIOOptionsEntry;
194 static IORegistryEntry * gIOChosenEntry;
195 #if defined(__i386__) || defined(__x86_64__)
196 static const OSSymbol * gIOCreateEFIDevicePathSymbol;
197 static const OSSymbol * gIOHibernateRTCVariablesKey;
198 static const OSSymbol * gIOHibernateBoot0082Key;
199 static const OSSymbol * gIOHibernateBootNextKey;
200 static OSData * gIOHibernateBoot0082Data;
201 static OSData * gIOHibernateBootNextData;
202 static OSObject * gIOHibernateBootNextSave;
203 #endif
204
205 static IOLock * gFSLock;
206 static uint32_t gFSState;
207 static IOPolledFileIOVars gFileVars;
208 static IOHibernateVars gIOHibernateVars;
209 static struct kern_direct_file_io_ref_t * gIOHibernateFileRef;
210 static hibernate_cryptvars_t gIOHibernateCryptWakeContext;
211 static hibernate_graphics_t _hibernateGraphics;
212 static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
213 static hibernate_statistics_t _hibernateStats;
214 static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats;
215
216 enum
217 {
218 kFSIdle = 0,
219 kFSOpening = 2,
220 kFSOpened = 3,
221 kFSTimedOut = 4,
222 };
223
224 static IOReturn IOHibernateDone(IOHibernateVars * vars);
225
226 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
227
228 enum { kXPRamAudioVolume = 8 };
229 enum { kDefaultIOSize = 128 * 1024 };
230 enum { kVideoMapSize = 32 * 1024 * 1024 };
231
232 #ifndef kIOMediaPreferredBlockSizeKey
233 #define kIOMediaPreferredBlockSizeKey "Preferred Block Size"
234 #endif
235
236 #ifndef kIOBootPathKey
237 #define kIOBootPathKey "bootpath"
238 #endif
239 #ifndef kIOSelectedBootDeviceKey
240 #define kIOSelectedBootDeviceKey "boot-device"
241 #endif
242
243
244 enum { kIOHibernateMinPollersNeeded = 2 };
245
246 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
247
248 // copy from phys addr to MD
249
250 static IOReturn
251 IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
252 IOByteCount offset, addr64_t bytes, IOByteCount length)
253 {
254 addr64_t srcAddr = bytes;
255 IOByteCount remaining;
256
257 remaining = length = min(length, md->getLength() - offset);
258 while (remaining) { // (process another target segment?)
259 addr64_t dstAddr64;
260 IOByteCount dstLen;
261
262 dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
263 if (!dstAddr64)
264 break;
265
266 // Clip segment length to remaining
267 if (dstLen > remaining)
268 dstLen = remaining;
269
270 #if 1
271 bcopy_phys(srcAddr, dstAddr64, dstLen);
272 #else
273 copypv(srcAddr, dstAddr64, dstLen,
274 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
275 #endif
276 srcAddr += dstLen;
277 offset += dstLen;
278 remaining -= dstLen;
279 }
280
281 assert(!remaining);
282
283 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
284 }
285
286 // copy from MD to phys addr
287
288 static IOReturn
289 IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
290 IOByteCount offset, addr64_t bytes, IOByteCount length)
291 {
292 addr64_t dstAddr = bytes;
293 IOByteCount remaining;
294
295 remaining = length = min(length, md->getLength() - offset);
296 while (remaining) { // (process another target segment?)
297 addr64_t srcAddr64;
298 IOByteCount dstLen;
299
300 srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
301 if (!srcAddr64)
302 break;
303
304 // Clip segment length to remaining
305 if (dstLen > remaining)
306 dstLen = remaining;
307
308 #if 1
309 bcopy_phys(srcAddr64, dstAddr, dstLen);
310 #else
311 copypv(srcAddr, dstAddr64, dstLen,
312 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
313 #endif
314 dstAddr += dstLen;
315 offset += dstLen;
316 remaining -= dstLen;
317 }
318
319 assert(!remaining);
320
321 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
322 }
323
324 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
325
326 void
327 hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
328 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
329 {
330 count += ppnum;
331 switch (kind)
332 {
333 case kIOHibernatePageStateUnwiredSave:
334 // unwired save
335 for (; ppnum < count; ppnum++)
336 {
337 hibernate_page_bitset(page_list, FALSE, ppnum);
338 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
339 }
340 break;
341 case kIOHibernatePageStateWiredSave:
342 // wired save
343 for (; ppnum < count; ppnum++)
344 {
345 hibernate_page_bitset(page_list, FALSE, ppnum);
346 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
347 }
348 break;
349 case kIOHibernatePageStateFree:
350 // free page
351 for (; ppnum < count; ppnum++)
352 {
353 hibernate_page_bitset(page_list, TRUE, ppnum);
354 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
355 }
356 break;
357 default:
358 panic("hibernate_set_page_state");
359 }
360 }
361
362 static vm_offset_t
363 hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
364 {
365 uint32_t page = *pPage;
366 uint32_t count;
367 hibernate_bitmap_t * bitmap;
368
369 while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
370 {
371 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
372 if (!count)
373 break;
374 page += count;
375 if (page <= bitmap->last_page)
376 break;
377 }
378
379 *pPage = page;
380 if (bitmap)
381 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
382 else
383 count = 0;
384
385 return (count);
386 }
387
388 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
389
390 static IOReturn
391 IOHibernatePollerProbe(IOPolledFileIOVars * vars, IOService * target)
392 {
393 IOReturn err = kIOReturnError;
394 int32_t idx;
395 IOPolledInterface * poller;
396
397 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
398 {
399 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
400 err = poller->probe(target);
401 if (err)
402 {
403 HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
404 break;
405 }
406 }
407
408 return (err);
409 }
410
411 static IOReturn
412 IOHibernatePollerOpen(IOPolledFileIOVars * vars, uint32_t state, IOMemoryDescriptor * md)
413 {
414 IOReturn err = kIOReturnError;
415 int32_t idx;
416 IOPolledInterface * poller;
417
418 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
419 {
420 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
421 err = poller->open(state, md);
422 if (err)
423 {
424 HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
425 break;
426 }
427 }
428
429 return (err);
430 }
431
432 static IOReturn
433 IOHibernatePollerClose(IOPolledFileIOVars * vars, uint32_t state)
434 {
435 IOReturn err = kIOReturnError;
436 int32_t idx;
437 IOPolledInterface * poller;
438
439 for (idx = 0;
440 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
441 idx++)
442 {
443 err = poller->close(state);
444 if (err)
445 HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
446 }
447
448 return (err);
449 }
450
451 static void
452 IOHibernatePollerIOComplete(void * target,
453 void * parameter,
454 IOReturn status,
455 UInt64 actualByteCount)
456 {
457 IOPolledFileIOVars * vars = (IOPolledFileIOVars *) parameter;
458
459 vars->ioStatus = status;
460 }
461
462 static IOReturn
463 IOHibernatePollerIO(IOPolledFileIOVars * vars,
464 uint32_t operation, uint32_t bufferOffset,
465 uint64_t deviceOffset, uint64_t length)
466 {
467
468 IOReturn err = kIOReturnError;
469 IOPolledInterface * poller;
470 IOPolledCompletion completion;
471
472 completion.target = 0;
473 completion.action = &IOHibernatePollerIOComplete;
474 completion.parameter = vars;
475
476 vars->ioStatus = -1;
477
478 poller = (IOPolledInterface *) vars->pollers->getObject(0);
479 err = poller->startIO(operation, bufferOffset, deviceOffset + vars->block0, length, completion);
480 if (err)
481 HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
482
483 return (err);
484 }
485
486 static IOReturn
487 IOHibernatePollerIODone(IOPolledFileIOVars * vars, bool abortable)
488 {
489 IOReturn err = kIOReturnSuccess;
490 int32_t idx = 0;
491 IOPolledInterface * poller;
492
493 while (-1 == vars->ioStatus)
494 {
495 for (idx = 0;
496 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
497 idx++)
498 {
499 IOReturn newErr;
500 newErr = poller->checkForWork();
501 if ((newErr == kIOReturnAborted) && !abortable)
502 newErr = kIOReturnSuccess;
503 if (kIOReturnSuccess == err)
504 err = newErr;
505 }
506 }
507
508 if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort())
509 {
510 err = kIOReturnAborted;
511 HIBLOG("IOPolledInterface::checkForWork sw abort\n");
512 }
513
514 if (err)
515 {
516 HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
517 }
518 else
519 {
520 err = vars->ioStatus;
521 if (kIOReturnSuccess != err)
522 HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
523 }
524
525 return (err);
526 }
527
528 IOReturn
529 IOPolledInterface::checkAllForWork(void)
530 {
531 IOReturn err = kIOReturnNotReady;
532 int32_t idx;
533 IOPolledInterface * poller;
534
535 IOHibernateVars * vars = &gIOHibernateVars;
536
537 if (!vars->fileVars || !vars->fileVars->pollers)
538 return (err);
539
540 for (idx = 0;
541 (poller = (IOPolledInterface *) vars->fileVars->pollers->getObject(idx));
542 idx++)
543 {
544 err = poller->checkForWork();
545 if (err)
546 HIBLOG("IOPolledInterface::checkAllForWork[%d] 0x%x\n", idx, err);
547 }
548
549 return (err);
550 }
551
552 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
553
554 struct _OpenFileContext
555 {
556 OSData * extents;
557 uint64_t size;
558 };
559
560 static void
561 file_extent_callback(void * ref, uint64_t start, uint64_t length)
562 {
563 _OpenFileContext * ctx = (_OpenFileContext *) ref;
564 IOPolledFileExtent extent;
565
566 extent.start = start;
567 extent.length = length;
568
569 HIBLOG("[0x%qx, 0x%qx]\n", start, length);
570
571 ctx->extents->appendBytes(&extent, sizeof(extent));
572 ctx->size += length;
573 }
574
575 static IOService *
576 IOCopyMediaForDev(dev_t device)
577 {
578 OSDictionary * matching;
579 OSNumber * num;
580 OSIterator * iter;
581 IOService * result = 0;
582
583 matching = IOService::serviceMatching("IOMedia");
584 if (!matching)
585 return (0);
586 do
587 {
588 num = OSNumber::withNumber(major(device), 32);
589 if (!num)
590 break;
591 matching->setObject(kIOBSDMajorKey, num);
592 num->release();
593 num = OSNumber::withNumber(minor(device), 32);
594 if (!num)
595 break;
596 matching->setObject(kIOBSDMinorKey, num);
597 num->release();
598 if (!num)
599 break;
600 iter = IOService::getMatchingServices(matching);
601 if (iter)
602 {
603 result = (IOService *) iter->getNextObject();
604 result->retain();
605 iter->release();
606 }
607 }
608 while (false);
609 matching->release();
610
611 return (result);
612 }
613
614 IOReturn
615 IOPolledFileOpen( const char * filename, uint64_t setFileSize,
616 IOBufferMemoryDescriptor * ioBuffer,
617 IOPolledFileIOVars ** fileVars, OSData ** fileExtents,
618 OSData ** imagePath, uint8_t * volumeCryptKey)
619 {
620 IOReturn err = kIOReturnSuccess;
621 IOPolledFileIOVars * vars;
622 _OpenFileContext ctx;
623 OSData * extentsData;
624 OSNumber * num;
625 IOService * part = 0;
626 OSString * keyUUID = 0;
627 OSString * keyStoreUUID = 0;
628 dev_t block_dev;
629 dev_t hibernate_image_dev;
630 uint64_t maxiobytes;
631 AbsoluteTime startTime, endTime;
632 uint64_t nsec;
633
634 vars = IONew(IOPolledFileIOVars, 1);
635 if (!vars) return (kIOReturnNoMemory);
636 bzero(vars, sizeof(*vars));
637
638 do
639 {
640 vars->io = false;
641 vars->buffer = (uint8_t *) ioBuffer->getBytesNoCopy();
642 vars->bufferHalf = 0;
643 vars->bufferOffset = 0;
644 vars->bufferSize = ioBuffer->getLength() >> 1;
645
646 extentsData = OSData::withCapacity(32);
647 ctx.extents = extentsData;
648 ctx.size = 0;
649 clock_get_uptime(&startTime);
650 vars->fileRef = kern_open_file_for_direct_io(filename,
651 &file_extent_callback, &ctx,
652 setFileSize,
653 // write file:
654 0, (caddr_t) gIOHibernateCurrentHeader,
655 sizeof(IOHibernateImageHeader),
656 // results
657 &block_dev,
658 &hibernate_image_dev,
659 &vars->block0,
660 &maxiobytes,
661 &vars->flags);
662 #if 0
663 uint32_t msDelay = (131071 & random());
664 HIBLOG("sleep %d\n", msDelay);
665 IOSleep(msDelay);
666 #endif
667 clock_get_uptime(&endTime);
668 SUB_ABSOLUTETIME(&endTime, &startTime);
669 absolutetime_to_nanoseconds(endTime, &nsec);
670
671 if (!vars->fileRef) err = kIOReturnNoSpace;
672
673 IOLockLock(gFSLock);
674 if (kFSOpening != gFSState) err = kIOReturnTimeout;
675 IOLockUnlock(gFSLock);
676
677 HIBLOG("kern_open_file_for_direct_io(%d) took %qd ms\n", err, nsec / 1000000ULL);
678 if (kIOReturnSuccess != err) break;
679
680 if (kIOHibernateModeSSDInvert & gIOHibernateMode)
681 vars->flags ^= kIOHibernateOptionSSD;
682
683 HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx ssd %d\n", filename, ctx.size,
684 vars->block0, maxiobytes, kIOHibernateOptionSSD & vars->flags);
685 if (ctx.size < 1*1024*1024) // check against image size estimate!
686 {
687 err = kIOReturnNoSpace;
688 break;
689 }
690
691 vars->fileSize = ctx.size;
692 if (maxiobytes < vars->bufferSize) vars->bufferSize = maxiobytes;
693
694 vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
695
696 part = IOCopyMediaForDev(block_dev);
697 if (!part)
698 {
699 err = kIOReturnNotFound;
700 break;
701 }
702 err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false,
703 (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
704 if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID)
705 {
706 // IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
707 uuid_t volumeKeyUUID;
708 aks_volume_key_t vek;
709 static IOService * sKeyStore;
710 static const OSSymbol * sAKSGetKey;
711
712 if (!sAKSGetKey)
713 sAKSGetKey = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY);
714 if (!sKeyStore)
715 sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
716 if (sKeyStore)
717 err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
718 else
719 err = kIOReturnNoResources;
720 if (kIOReturnSuccess == err)
721 err = sKeyStore->callPlatformFunction(sAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
722 if (kIOReturnSuccess != err)
723 IOLog("volume key err 0x%x\n", err);
724 else
725 {
726 size_t bytes = (kIOHibernateAESKeySize / 8);
727 if (vek.key.keybytecount < bytes)
728 bytes = vek.key.keybytecount;
729 bcopy(&vek.key.keybytes[0], volumeCryptKey, bytes);
730 }
731 bzero(&vek, sizeof(vek));
732 }
733 part->release();
734
735 part = IOCopyMediaForDev(hibernate_image_dev);
736 if (!part)
737 {
738 err = kIOReturnNotFound;
739 break;
740 }
741
742 IORegistryEntry * next;
743 IORegistryEntry * child;
744 OSData * data;
745
746 vars->pollers = OSArray::withCapacity(4);
747 if (!vars->pollers)
748 {
749 err = kIOReturnNoMemory;
750 break;
751 }
752
753 vars->blockSize = 512;
754 next = part;
755 do
756 {
757 IOPolledInterface * poller;
758 OSObject * obj;
759
760 obj = next->getProperty(kIOPolledInterfaceSupportKey);
761 if (kOSBooleanFalse == obj)
762 {
763 vars->pollers->flushCollection();
764 break;
765 }
766 else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
767 vars->pollers->setObject(poller);
768 if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey))))
769 vars->blockSize = num->unsigned32BitValue();
770 child = next;
771 }
772 while ((next = child->getParentEntry(gIOServicePlane))
773 && child->isParent(next, gIOServicePlane, true));
774
775 if (vars->blockSize < 4096) vars->blockSize = 4096;
776
777 HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
778 major(hibernate_image_dev), minor(hibernate_image_dev), (long)vars->blockSize, vars->pollers->getCount());
779
780 if (vars->pollers->getCount() < kIOHibernateMinPollersNeeded)
781 {
782 err = kIOReturnUnsupported;
783 continue;
784 }
785 if (vars->blockSize < sizeof(IOHibernateImageHeader))
786 {
787 err = kIOReturnError;
788 continue;
789 }
790
791 err = IOHibernatePollerProbe(vars, (IOService *) part);
792 if (kIOReturnSuccess != err) break;
793
794 err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer);
795 if (kIOReturnSuccess != err) break;
796
797 vars->media = part;
798 next = part;
799 while (next)
800 {
801 next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue);
802 next = next->getParentEntry(gIOServicePlane);
803 }
804
805 *fileVars = vars;
806 *fileExtents = extentsData;
807
808 // make imagePath
809
810 if ((extentsData->getLength() >= sizeof(IOPolledFileExtent)))
811 {
812 char str2[24 + sizeof(uuid_string_t) + 2];
813
814 #if defined(__i386__) || defined(__x86_64__)
815 if (!gIOCreateEFIDevicePathSymbol)
816 gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
817
818 if (keyUUID)
819 snprintf(str2, sizeof(str2), "%qx:%s",
820 vars->extentMap[0].start, keyUUID->getCStringNoCopy());
821 else
822 snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
823
824 err = IOService::getPlatform()->callPlatformFunction(
825 gIOCreateEFIDevicePathSymbol, false,
826 (void *) part, (void *) str2,
827 (void *) (uintptr_t) true, (void *) &data);
828 #else
829 char str1[256];
830 int len = sizeof(str1);
831
832 if (!part->getPath(str1, &len, gIODTPlane))
833 err = kIOReturnNotFound;
834 else
835 {
836 snprintf(str2, sizeof(str2), ",%qx", vars->extentMap[0].start);
837 // (strip the plane name)
838 char * tail = strchr(str1, ':');
839 if (!tail)
840 tail = str1 - 1;
841 data = OSData::withBytes(tail + 1, strlen(tail + 1));
842 data->appendBytes(str2, strlen(str2));
843 }
844 #endif
845 if (kIOReturnSuccess == err)
846 *imagePath = data;
847 else
848 HIBLOG("error 0x%x getting path\n", err);
849 }
850 }
851 while (false);
852
853 if (kIOReturnSuccess != err)
854 {
855 HIBLOG("error 0x%x opening hibernation file\n", err);
856 if (vars->fileRef)
857 {
858 kern_close_file_for_direct_io(vars->fileRef, 0, 0, 0, 0, 0);
859 vars->fileRef = NULL;
860 }
861 }
862
863 if (part)
864 part->release();
865
866 return (err);
867 }
868
869 IOReturn
870 IOPolledFileClose( IOPolledFileIOVars * vars )
871 {
872 if (vars->pollers)
873 {
874 IOHibernatePollerClose(vars, kIOPolledPostflightState);
875 vars->pollers->release();
876 }
877
878 bzero(vars, sizeof(IOPolledFileIOVars));
879
880 return (kIOReturnSuccess);
881 }
882
883 static IOReturn
884 IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
885 {
886 IOPolledFileExtent * extentMap;
887
888 extentMap = vars->extentMap;
889
890 vars->position = position;
891
892 while (position >= extentMap->length)
893 {
894 position -= extentMap->length;
895 extentMap++;
896 }
897
898 vars->currentExtent = extentMap;
899 vars->extentRemaining = extentMap->length - position;
900 vars->extentPosition = vars->position - position;
901
902 if (vars->bufferSize <= vars->extentRemaining)
903 vars->bufferLimit = vars->bufferSize;
904 else
905 vars->bufferLimit = vars->extentRemaining;
906
907 return (kIOReturnSuccess);
908 }
909
910 static IOReturn
911 IOPolledFileWrite(IOPolledFileIOVars * vars,
912 const uint8_t * bytes, IOByteCount size,
913 hibernate_cryptvars_t * cryptvars)
914 {
915 IOReturn err = kIOReturnSuccess;
916 IOByteCount copy;
917 bool flush = false;
918
919 do
920 {
921 if (!bytes && !size)
922 {
923 // seek to end of block & flush
924 size = vars->position & (vars->blockSize - 1);
925 if (size)
926 size = vars->blockSize - size;
927 flush = true;
928 // use some garbage for the fill
929 bytes = vars->buffer + vars->bufferOffset;
930 }
931
932 copy = vars->bufferLimit - vars->bufferOffset;
933 if (copy > size)
934 copy = size;
935 else
936 flush = true;
937
938 if (bytes)
939 {
940 bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
941 bytes += copy;
942 }
943 else
944 bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
945
946 size -= copy;
947 vars->bufferOffset += copy;
948 vars->position += copy;
949
950 if (flush && vars->bufferOffset)
951 {
952 uint64_t offset = (vars->position - vars->bufferOffset
953 - vars->extentPosition + vars->currentExtent->start);
954 uint32_t length = (vars->bufferOffset);
955
956 #if CRYPTO
957 if (cryptvars && vars->encryptStart
958 && (vars->position > vars->encryptStart)
959 && ((vars->position - length) < vars->encryptEnd))
960 {
961 AbsoluteTime startTime, endTime;
962
963 uint64_t encryptLen, encryptStart;
964 encryptLen = vars->position - vars->encryptStart;
965 if (encryptLen > length)
966 encryptLen = length;
967 encryptStart = length - encryptLen;
968 if (vars->position > vars->encryptEnd)
969 encryptLen -= (vars->position - vars->encryptEnd);
970
971 clock_get_uptime(&startTime);
972
973 // encrypt the buffer
974 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
975 &cryptvars->aes_iv[0],
976 encryptLen / AES_BLOCK_SIZE,
977 vars->buffer + vars->bufferHalf + encryptStart,
978 &cryptvars->ctx.encrypt);
979
980 clock_get_uptime(&endTime);
981 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
982 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
983 vars->cryptBytes += encryptLen;
984
985 // save initial vector for following encrypts
986 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
987 &cryptvars->aes_iv[0],
988 AES_BLOCK_SIZE);
989 }
990 #endif /* CRYPTO */
991
992 if (vars->io)
993 {
994 err = IOHibernatePollerIODone(vars, true);
995 if (kIOReturnSuccess != err)
996 break;
997 }
998
999 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
1000 //if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
1001
1002 err = IOHibernatePollerIO(vars, kIOPolledWrite, vars->bufferHalf, offset, length);
1003 if (kIOReturnSuccess != err)
1004 break;
1005 vars->io = true;
1006
1007 vars->extentRemaining -= vars->bufferOffset;
1008 if (!vars->extentRemaining)
1009 {
1010 vars->currentExtent++;
1011 vars->extentRemaining = vars->currentExtent->length;
1012 vars->extentPosition = vars->position;
1013 if (!vars->extentRemaining)
1014 {
1015 err = kIOReturnOverrun;
1016 break;
1017 }
1018 }
1019
1020 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1021 vars->bufferOffset = 0;
1022 if (vars->bufferSize <= vars->extentRemaining)
1023 vars->bufferLimit = vars->bufferSize;
1024 else
1025 vars->bufferLimit = vars->extentRemaining;
1026
1027 flush = false;
1028 }
1029 }
1030 while (size);
1031
1032 return (err);
1033 }
1034
1035 static IOReturn
1036 IOPolledFileRead(IOPolledFileIOVars * vars,
1037 uint8_t * bytes, IOByteCount size,
1038 hibernate_cryptvars_t * cryptvars)
1039 {
1040 IOReturn err = kIOReturnSuccess;
1041 IOByteCount copy;
1042
1043 // bytesWritten += size;
1044
1045 do
1046 {
1047 copy = vars->bufferLimit - vars->bufferOffset;
1048 if (copy > size)
1049 copy = size;
1050
1051 if (bytes)
1052 {
1053 bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
1054 bytes += copy;
1055 }
1056 size -= copy;
1057 vars->bufferOffset += copy;
1058 // vars->position += copy;
1059
1060 if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd))
1061 {
1062 if (vars->io)
1063 {
1064 err = IOHibernatePollerIODone(vars, false);
1065 if (kIOReturnSuccess != err)
1066 break;
1067 }
1068 else
1069 cryptvars = 0;
1070
1071 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
1072
1073 vars->position += vars->lastRead;
1074 vars->extentRemaining -= vars->lastRead;
1075 vars->bufferLimit = vars->lastRead;
1076
1077 if (!vars->extentRemaining)
1078 {
1079 vars->currentExtent++;
1080 vars->extentRemaining = vars->currentExtent->length;
1081 vars->extentPosition = vars->position;
1082 if (!vars->extentRemaining)
1083 {
1084 err = kIOReturnOverrun;
1085 break;
1086 }
1087 }
1088
1089 uint64_t length;
1090 uint64_t lastReadLength = vars->lastRead;
1091 uint64_t offset = (vars->position
1092 - vars->extentPosition + vars->currentExtent->start);
1093 if (vars->extentRemaining <= vars->bufferSize)
1094 length = vars->extentRemaining;
1095 else
1096 length = vars->bufferSize;
1097 if ((length + vars->position) > vars->readEnd)
1098 length = vars->readEnd - vars->position;
1099
1100 vars->lastRead = length;
1101 if (length)
1102 {
1103 //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
1104 err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
1105 if (kIOReturnSuccess != err)
1106 break;
1107 vars->io = true;
1108 }
1109
1110 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1111 vars->bufferOffset = 0;
1112
1113 #if CRYPTO
1114 if (cryptvars)
1115 {
1116 uint8_t thisVector[AES_BLOCK_SIZE];
1117 AbsoluteTime startTime, endTime;
1118
1119 // save initial vector for following decrypts
1120 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
1121 bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
1122 &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1123
1124 // decrypt the buffer
1125 clock_get_uptime(&startTime);
1126
1127 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
1128 &thisVector[0],
1129 lastReadLength / AES_BLOCK_SIZE,
1130 vars->buffer + vars->bufferHalf,
1131 &cryptvars->ctx.decrypt);
1132
1133 clock_get_uptime(&endTime);
1134 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
1135 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
1136 vars->cryptBytes += lastReadLength;
1137 }
1138 #endif /* CRYPTO */
1139 }
1140 }
1141 while (size);
1142
1143 return (err);
1144 }
1145
1146 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1147
1148 IOReturn
1149 IOHibernateSystemSleep(void)
1150 {
1151 IOReturn err;
1152 OSData * data;
1153 OSObject * obj;
1154 OSString * str;
1155 OSNumber * num;
1156 bool dsSSD, vmflush;
1157 IOHibernateVars * vars;
1158
1159 gIOHibernateState = kIOHibernateStateInactive;
1160
1161 if (!gIOChosenEntry)
1162 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1163
1164 gIOHibernateDebugFlags = 0;
1165 if (kIOLogHibernate & gIOKitDebug)
1166 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
1167
1168 if (IOService::getPMRootDomain()->getHibernateSettings(
1169 &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime))
1170 {
1171 if (kIOHibernateModeSleep & gIOHibernateMode)
1172 // default to discard clean for safe sleep
1173 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
1174 | kIOHibernateModeDiscardCleanActive);
1175 }
1176
1177 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
1178 {
1179 if ((str = OSDynamicCast(OSString, obj)))
1180 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
1181 sizeof(gIOHibernateFilename));
1182 obj->release();
1183 }
1184
1185 if (!gIOHibernateMode || !gIOHibernateFilename[0])
1186 return (kIOReturnUnsupported);
1187
1188 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
1189
1190 vars = IONew(IOHibernateVars, 1);
1191 if (!vars) return (kIOReturnNoMemory);
1192 bzero(vars, sizeof(*vars));
1193
1194 IOLockLock(gFSLock);
1195 if (kFSIdle != gFSState)
1196 {
1197 HIBLOG("hibernate file busy\n");
1198 IOLockUnlock(gFSLock);
1199 IODelete(vars, IOHibernateVars, 1);
1200 return (kIOReturnBusy);
1201 }
1202 gFSState = kFSOpening;
1203 IOLockUnlock(gFSLock);
1204
1205 do
1206 {
1207 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1208 2 * page_size + WKdm_SCRATCH_BUF_SIZE, page_size);
1209 vars->ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1210 2 * kDefaultIOSize, page_size);
1211
1212 vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1213 ptoa_64(gIOHibernateHandoffPageCount), page_size);
1214
1215 if (!vars->srcBuffer || !vars->ioBuffer || !vars->handoffBuffer)
1216 {
1217 err = kIOReturnNoMemory;
1218 break;
1219 }
1220
1221 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey)))
1222 {
1223 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMinSize = num->unsigned64BitValue();
1224 obj->release();
1225 }
1226 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey)))
1227 {
1228 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMaxSize = num->unsigned64BitValue();
1229 obj->release();
1230 }
1231
1232 boolean_t encryptedswap = true;
1233 uint32_t pageCount;
1234 AbsoluteTime startTime, endTime;
1235 uint64_t nsec;
1236
1237 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1238 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
1239 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1240
1241 vmflush = (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey));
1242 uint64_t setFileSize = 0;
1243 err = hibernate_alloc_page_lists(&vars->page_list,
1244 &vars->page_list_wired,
1245 &vars->page_list_pal);
1246 if (KERN_SUCCESS != err)
1247 break;
1248
1249 if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode))
1250 {
1251 hibernate_page_list_setall(vars->page_list,
1252 vars->page_list_wired,
1253 vars->page_list_pal,
1254 true /* preflight */,
1255 vmflush /* discard */,
1256 &pageCount);
1257 PE_Video consoleInfo;
1258 bzero(&consoleInfo, sizeof(consoleInfo));
1259 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1260
1261 // estimate: 5% increase in pages compressed
1262 // screen preview 2 images compressed 50%
1263 setFileSize = ((ptoa_64((105 * pageCount) / 100) * gIOHibernateCompression) >> 8)
1264 + vars->page_list->list_size
1265 + (consoleInfo.v_width * consoleInfo.v_height * 4);
1266
1267 HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
1268 pageCount, (100ULL * gIOHibernateCompression) >> 8,
1269 setFileSize, vars->fileMinSize);
1270
1271 if (!(kIOHibernateModeFileResize & gIOHibernateMode)
1272 && (setFileSize < vars->fileMinSize))
1273 {
1274 setFileSize = vars->fileMinSize;
1275 }
1276 }
1277
1278 // open & invalidate the image file
1279
1280 err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, vars->ioBuffer,
1281 &vars->fileVars, &vars->fileExtents, &data,
1282 &vars->volumeCryptKey[0]);
1283 if (KERN_SUCCESS != err)
1284 {
1285 HIBLOG("IOPolledFileOpen(%x)\n", err);
1286 break;
1287 }
1288
1289 clock_get_uptime(&startTime);
1290 err = hibernate_setup(gIOHibernateCurrentHeader,
1291 gIOHibernateFreeRatio, gIOHibernateFreeTime,
1292 vmflush,
1293 vars->page_list, vars->page_list_wired, vars->page_list_pal);
1294 clock_get_uptime(&endTime);
1295 SUB_ABSOLUTETIME(&endTime, &startTime);
1296 absolutetime_to_nanoseconds(endTime, &nsec);
1297 HIBLOG("hibernate_setup(%d) took %qd ms\n", err, nsec / 1000000ULL);
1298
1299 dsSSD = ((0 != (kIOHibernateOptionSSD & vars->fileVars->flags))
1300 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
1301 if (dsSSD)
1302 {
1303 gIOHibernateCurrentHeader->options |=
1304 kIOHibernateOptionSSD
1305 | kIOHibernateOptionColor;
1306
1307 #if defined(__i386__) || defined(__x86_64__)
1308 if (!uuid_is_null(vars->volumeCryptKey) &&
1309 (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey)))
1310 {
1311 uintptr_t smcVars[2];
1312 smcVars[0] = sizeof(vars->volumeCryptKey);
1313 smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
1314
1315 IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
1316 bzero(smcVars, sizeof(smcVars));
1317 }
1318 #endif
1319 }
1320 else
1321 {
1322 gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
1323 }
1324
1325
1326 if (KERN_SUCCESS != err)
1327 break;
1328
1329 if (encryptedswap || !uuid_is_null(vars->volumeCryptKey))
1330 gIOHibernateMode ^= kIOHibernateModeEncrypt;
1331
1332 if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1333 {
1334 vars->videoAllocSize = kVideoMapSize;
1335 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
1336 vars->videoMapping = 0;
1337 }
1338
1339 // generate crypt keys
1340 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
1341 vars->wiredCryptKey[i] = random();
1342 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
1343 vars->cryptKey[i] = random();
1344
1345 // set nvram
1346
1347 IORegistryEntry * regEntry;
1348 if (!gIOOptionsEntry)
1349 {
1350 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
1351 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
1352 if (regEntry && !gIOOptionsEntry)
1353 regEntry->release();
1354 }
1355
1356 if (gIOOptionsEntry)
1357 {
1358 const OSSymbol * sym;
1359
1360 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1361 if (sym)
1362 {
1363 gIOOptionsEntry->setProperty(sym, data);
1364 sym->release();
1365 }
1366 data->release();
1367
1368 #if defined(__i386__) || defined(__x86_64__)
1369 struct AppleRTCHibernateVars
1370 {
1371 uint8_t signature[4];
1372 uint32_t revision;
1373 uint8_t booterSignature[20];
1374 uint8_t wiredCryptKey[16];
1375 };
1376 AppleRTCHibernateVars rtcVars;
1377
1378 rtcVars.signature[0] = 'A';
1379 rtcVars.signature[1] = 'A';
1380 rtcVars.signature[2] = 'P';
1381 rtcVars.signature[3] = 'L';
1382 rtcVars.revision = 1;
1383 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
1384 if (gIOHibernateBootSignature[0])
1385 {
1386 char c;
1387 uint8_t value = 0;
1388 for (uint32_t i = 0;
1389 (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
1390 i++)
1391 {
1392 if (c >= 'a')
1393 c -= 'a' - 10;
1394 else if (c >= 'A')
1395 c -= 'A' - 10;
1396 else if (c >= '0')
1397 c -= '0';
1398 else
1399 continue;
1400 value = (value << 4) | c;
1401 if (i & 1)
1402 rtcVars.booterSignature[i >> 1] = value;
1403 }
1404 }
1405 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
1406 if (data)
1407 {
1408 if (!gIOHibernateRTCVariablesKey)
1409 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1410 if (gIOHibernateRTCVariablesKey)
1411 IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
1412
1413 if( gIOOptionsEntry )
1414 {
1415 if( gIOHibernateMode & kIOHibernateModeSwitch )
1416 {
1417 const OSSymbol *sym;
1418 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSwitchVarsKey);
1419 if( sym )
1420 {
1421 gIOOptionsEntry->setProperty(sym, data); /* intentional insecure backup of rtc boot vars */
1422 sym->release();
1423 }
1424 }
1425 }
1426
1427 data->release();
1428 }
1429 if (gIOChosenEntry)
1430 {
1431 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
1432 if (data)
1433 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
1434 {
1435 // set BootNext
1436
1437 if (!gIOHibernateBoot0082Data)
1438 {
1439 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
1440 if (data)
1441 {
1442 // AppleNVRAM_EFI_LOAD_OPTION
1443 struct {
1444 uint32_t Attributes;
1445 uint16_t FilePathLength;
1446 uint16_t Desc;
1447 } loadOptionHeader;
1448 loadOptionHeader.Attributes = 1;
1449 loadOptionHeader.FilePathLength = data->getLength();
1450 loadOptionHeader.Desc = 0;
1451 gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
1452 if (gIOHibernateBoot0082Data)
1453 {
1454 gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
1455 gIOHibernateBoot0082Data->appendBytes(data);
1456 }
1457 }
1458 }
1459 if (!gIOHibernateBoot0082Key)
1460 gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
1461 if (!gIOHibernateBootNextKey)
1462 gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
1463 if (!gIOHibernateBootNextData)
1464 {
1465 uint16_t bits = 0x0082;
1466 gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
1467 }
1468 if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData)
1469 {
1470 gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
1471 gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
1472 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
1473 }
1474 }
1475 }
1476 #else /* !i386 && !x86_64 */
1477 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1478 {
1479 data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1480 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
1481 if (sym && data)
1482 gIOOptionsEntry->setProperty(sym, data);
1483 if (sym)
1484 sym->release();
1485 if (data)
1486 data->release();
1487 if (false && gIOHibernateBootSignature[0])
1488 {
1489 data = OSData::withCapacity(16);
1490 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
1491 if (sym && data)
1492 {
1493 char c;
1494 uint8_t value = 0;
1495 for (uint32_t i = 0; (c = gIOHibernateBootSignature[i]); i++)
1496 {
1497 if (c >= 'a')
1498 c -= 'a' - 10;
1499 else if (c >= 'A')
1500 c -= 'A' - 10;
1501 else if (c >= '0')
1502 c -= '0';
1503 else
1504 continue;
1505 value = (value << 4) | c;
1506 if (i & 1)
1507 data->appendBytes(&value, sizeof(value));
1508 }
1509 gIOOptionsEntry->setProperty(sym, data);
1510 }
1511 if (sym)
1512 sym->release();
1513 if (data)
1514 data->release();
1515 }
1516 }
1517 if (!vars->haveFastBoot)
1518 {
1519 // set boot volume to zero
1520 IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
1521 if (platform && (kIOReturnSuccess == platform->readXPRAM(kXPRamAudioVolume,
1522 &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume))))
1523 {
1524 uint8_t newVolume;
1525 newVolume = vars->saveBootAudioVolume & 0xf8;
1526 platform->writeXPRAM(kXPRamAudioVolume,
1527 &newVolume, sizeof(newVolume));
1528 }
1529 }
1530 #endif /* !i386 && !x86_64 */
1531 }
1532 // --
1533
1534 }
1535 while (false);
1536
1537 IOLockLock(gFSLock);
1538 if ((kIOReturnSuccess == err) && (kFSOpening == gFSState))
1539 {
1540 gFSState = kFSOpened;
1541 gIOHibernateVars = *vars;
1542 gFileVars = *vars->fileVars;
1543 gIOHibernateVars.fileVars = &gFileVars;
1544 gIOHibernateFileRef = gFileVars.fileRef;
1545 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
1546 gIOHibernateState = kIOHibernateStateHibernating;
1547 }
1548 else
1549 {
1550 HIBLOG("hibernate file close due timeout\n");
1551 if (vars->fileVars && vars->fileVars->fileRef) kern_close_file_for_direct_io(vars->fileVars->fileRef, 0, 0, 0, 0, 0);
1552 IOHibernateDone(vars);
1553 gFSState = kFSIdle;
1554 }
1555 IOLockUnlock(gFSLock);
1556
1557 if (vars->fileVars) IODelete(vars->fileVars, IOPolledFileIOVars, 1);
1558 IODelete(vars, IOHibernateVars, 1);
1559
1560 return (err);
1561 }
1562
1563 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1564
1565 DECLARE_IOHIBERNATEPROGRESSALPHA
1566
1567 static void
1568 ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
1569 {
1570 uint32_t rowBytes, pixelShift;
1571 uint32_t x, y;
1572 int32_t blob;
1573 uint32_t alpha, in, color, result;
1574 uint8_t * out;
1575 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1576
1577 rowBytes = display->rowBytes;
1578 pixelShift = display->depth >> 4;
1579 if (pixelShift < 1) return;
1580
1581 screen += ((display->width
1582 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1583 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1584
1585 for (y = 0; y < kIOHibernateProgressHeight; y++)
1586 {
1587 out = screen + y * rowBytes;
1588 for (blob = 0; blob < kIOHibernateProgressCount; blob++)
1589 {
1590 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
1591 for (x = 0; x < kIOHibernateProgressWidth; x++)
1592 {
1593 alpha = gIOHibernateProgressAlpha[y][x];
1594 result = color;
1595 if (alpha)
1596 {
1597 if (0xff != alpha)
1598 {
1599 if (1 == pixelShift)
1600 {
1601 in = *((uint16_t *)out) & 0x1f; // 16
1602 in = (in << 3) | (in >> 2);
1603 }
1604 else
1605 in = *((uint32_t *)out) & 0xff; // 32
1606 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
1607 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
1608 }
1609 if (1 == pixelShift)
1610 {
1611 result >>= 3;
1612 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1613 }
1614 else
1615 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1616 }
1617 out += (1 << pixelShift);
1618 }
1619 out += (kIOHibernateProgressSpacing << pixelShift);
1620 }
1621 }
1622 }
1623
1624
1625 static void
1626 ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
1627 {
1628 uint32_t rowBytes, pixelShift;
1629 uint32_t x, y;
1630 int32_t blob, lastBlob;
1631 uint32_t alpha, in, color, result;
1632 uint8_t * out;
1633 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1634
1635 pixelShift = display->depth >> 4;
1636 if (pixelShift < 1)
1637 return;
1638
1639 rowBytes = display->rowBytes;
1640
1641 screen += ((display->width
1642 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1643 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1644
1645 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
1646
1647 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
1648
1649 for (y = 0; y < kIOHibernateProgressHeight; y++)
1650 {
1651 out = screen + y * rowBytes;
1652 for (blob = firstBlob; blob <= lastBlob; blob++)
1653 {
1654 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
1655 for (x = 0; x < kIOHibernateProgressWidth; x++)
1656 {
1657 alpha = gIOHibernateProgressAlpha[y][x];
1658 result = color;
1659 if (alpha)
1660 {
1661 if (0xff != alpha)
1662 {
1663 in = display->progressSaveUnder[blob][saveindex[blob]++];
1664 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
1665 }
1666 if (1 == pixelShift)
1667 {
1668 result >>= 3;
1669 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1670 }
1671 else
1672 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1673 }
1674 out += (1 << pixelShift);
1675 }
1676 out += (kIOHibernateProgressSpacing << pixelShift);
1677 }
1678 }
1679 }
1680
1681 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1682
1683 IOReturn
1684 IOHibernateIOKitSleep(void)
1685 {
1686 IOReturn ret = kIOReturnSuccess;
1687 IOLockLock(gFSLock);
1688 if (kFSOpening == gFSState)
1689 {
1690 gFSState = kFSTimedOut;
1691 HIBLOG("hibernate file open timed out\n");
1692 ret = kIOReturnTimeout;
1693 }
1694 IOLockUnlock(gFSLock);
1695 return (ret);
1696 }
1697
1698 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1699
1700 IOReturn
1701 IOHibernateSystemHasSlept(void)
1702 {
1703 IOReturn ret = kIOReturnSuccess;
1704 IOHibernateVars * vars = &gIOHibernateVars;
1705 OSObject * obj = 0;
1706 OSData * data;
1707
1708 IOLockLock(gFSLock);
1709 if ((kFSOpened != gFSState) && gIOHibernateMode)
1710 {
1711 ret = kIOReturnTimeout;
1712 }
1713 IOLockUnlock(gFSLock);
1714 if (kIOReturnSuccess != ret) return (ret);
1715
1716 if (gIOHibernateMode) obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
1717 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1718 if (obj && !vars->previewBuffer)
1719 obj->release();
1720
1721 vars->consoleMapping = NULL;
1722 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
1723 {
1724 vars->previewBuffer->release();
1725 vars->previewBuffer = 0;
1726 }
1727
1728 if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1729 && vars->previewBuffer
1730 && (data = OSDynamicCast(OSData,
1731 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
1732 {
1733 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
1734 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
1735
1736 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1737
1738 if (kIOHibernatePreviewUpdates & flags)
1739 {
1740 PE_Video consoleInfo;
1741 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1742
1743 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1744
1745 graphicsInfo->width = consoleInfo.v_width;
1746 graphicsInfo->height = consoleInfo.v_height;
1747 graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
1748 graphicsInfo->depth = consoleInfo.v_depth;
1749 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1750
1751 HIBPRINT("video %p %d %d %d\n",
1752 vars->consoleMapping, graphicsInfo->depth,
1753 graphicsInfo->width, graphicsInfo->height);
1754 if (vars->consoleMapping)
1755 ProgressInit(graphicsInfo, vars->consoleMapping,
1756 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1757 }
1758 }
1759
1760 if (gIOOptionsEntry)
1761 gIOOptionsEntry->sync();
1762
1763 return (ret);
1764 }
1765
1766 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1767
1768 static DeviceTreeNode *
1769 MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry)
1770 {
1771 DeviceTreeNodeProperty * prop;
1772 DeviceTreeNode * child;
1773 IORegistryEntry * childRegEntry;
1774 const char * nameProp;
1775 unsigned int propLen, idx;
1776
1777 prop = (DeviceTreeNodeProperty *) (entry + 1);
1778 for (idx = 0; idx < entry->nProperties; idx++)
1779 {
1780 if (regEntry && (0 != strcmp("name", prop->name)))
1781 {
1782 regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
1783 // HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
1784 }
1785 prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
1786 }
1787
1788 child = (DeviceTreeNode *) prop;
1789 for (idx = 0; idx < entry->nChildren; idx++)
1790 {
1791 if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen))
1792 panic("no name");
1793 childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
1794 // HIBPRINT("%s == %p\n", nameProp, childRegEntry);
1795 child = MergeDeviceTree(child, childRegEntry);
1796 }
1797 return (child);
1798 }
1799
1800 IOReturn
1801 IOHibernateSystemWake(void)
1802 {
1803 if (kFSOpened == gFSState)
1804 {
1805 IOHibernateDone(&gIOHibernateVars);
1806 }
1807 else
1808 {
1809 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1810 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1811 }
1812 return (kIOReturnSuccess);
1813 }
1814
1815 static IOReturn
1816 IOHibernateDone(IOHibernateVars * vars)
1817 {
1818 IORegistryEntry * next;
1819
1820 hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
1821
1822 if (vars->videoMapping)
1823 {
1824 if (vars->videoMapSize)
1825 // remove mappings
1826 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1827 if (vars->videoAllocSize)
1828 // dealloc range
1829 kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
1830 }
1831
1832 if (vars->previewBuffer)
1833 {
1834 vars->previewBuffer->release();
1835 vars->previewBuffer = 0;
1836 }
1837
1838 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1839 {
1840 IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
1841 gIOHibernateCurrentHeader->options, 32);
1842 }
1843 else
1844 {
1845 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1846 }
1847
1848 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1849 && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus))
1850 {
1851 IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
1852 &gIOHibernateGraphicsInfo->gfxStatus,
1853 sizeof(gIOHibernateGraphicsInfo->gfxStatus));
1854 }
1855 else
1856 {
1857 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1858 }
1859
1860 if (vars->fileVars)
1861 {
1862 if ((next = vars->fileVars->media)) do
1863 {
1864 next->removeProperty(kIOPolledInterfaceActiveKey);
1865 next = next->getParentEntry(gIOServicePlane);
1866 }
1867 while (next);
1868 IOPolledFileClose(vars->fileVars);
1869 }
1870
1871 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1872
1873 #if defined(__i386__) || defined(__x86_64__)
1874 IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
1875 IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
1876
1877 /*
1878 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1879 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1880 */
1881 if (gIOOptionsEntry) {
1882
1883 if (gIOHibernateRTCVariablesKey) {
1884 if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
1885 gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
1886 }
1887 }
1888
1889 if (gIOHibernateBootNextKey)
1890 {
1891 if (gIOHibernateBootNextSave)
1892 {
1893 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
1894 gIOHibernateBootNextSave->release();
1895 gIOHibernateBootNextSave = NULL;
1896 }
1897 else
1898 gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
1899 }
1900 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) gIOOptionsEntry->sync();
1901 }
1902 #endif
1903
1904 if (vars->srcBuffer)
1905 vars->srcBuffer->release();
1906 if (vars->ioBuffer)
1907 vars->ioBuffer->release();
1908 bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
1909 if (vars->handoffBuffer)
1910 {
1911 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1912 {
1913 IOHibernateHandoff * handoff;
1914 bool done = false;
1915 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
1916 !done;
1917 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
1918 {
1919 HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
1920 uint8_t * data = &handoff->data[0];
1921 switch (handoff->type)
1922 {
1923 case kIOHibernateHandoffTypeEnd:
1924 done = true;
1925 break;
1926
1927 case kIOHibernateHandoffTypeDeviceTree:
1928 MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
1929 break;
1930
1931 case kIOHibernateHandoffTypeKeyStore:
1932 #if defined(__i386__) || defined(__x86_64__)
1933 {
1934 IOBufferMemoryDescriptor *
1935 md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
1936 if (md)
1937 {
1938 IOSetKeyStoreData(md);
1939 }
1940 }
1941 #endif
1942 break;
1943
1944 default:
1945 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
1946 break;
1947 }
1948 }
1949 }
1950 vars->handoffBuffer->release();
1951 }
1952 if (vars->fileExtents)
1953 vars->fileExtents->release();
1954
1955 bzero(vars, sizeof(*vars));
1956
1957 // gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1958
1959 return (kIOReturnSuccess);
1960 }
1961
1962 IOReturn
1963 IOHibernateSystemPostWake(void)
1964 {
1965 struct kern_direct_file_io_ref_t * fileRef;
1966
1967 if (kFSOpened == gFSState)
1968 {
1969 // invalidate & close the image file
1970 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1971 if ((fileRef = gIOHibernateFileRef))
1972 {
1973 gIOHibernateFileRef = 0;
1974 IOSleep(TRIM_DELAY);
1975 kern_close_file_for_direct_io(fileRef,
1976 #if DISABLE_TRIM
1977 0, 0, 0, 0, 0);
1978 #else
1979 0, (caddr_t) gIOHibernateCurrentHeader,
1980 sizeof(IOHibernateImageHeader),
1981 0,
1982 gIOHibernateCurrentHeader->imageSize);
1983 #endif
1984 }
1985 gFSState = kFSIdle;
1986 }
1987 return (kIOReturnSuccess);
1988 }
1989
1990 bool IOHibernateWasScreenLocked(void)
1991 {
1992 bool ret = false;
1993 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry)
1994 {
1995 OSData *
1996 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
1997 if (data) switch (*((uint32_t *)data->getBytesNoCopy()))
1998 {
1999 case kIOScreenLockLocked:
2000 case kIOScreenLockFileVaultDialog:
2001 ret = true;
2002 break;
2003 case kIOScreenLockNoLock:
2004 case kIOScreenLockUnlocked:
2005 default:
2006 ret = false;
2007 break;
2008 }
2009 }
2010 return (ret);
2011 }
2012
2013 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2014
2015 SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
2016 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2017 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
2018 SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
2019 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2020 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
2021 SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
2022 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2023 &gIOHibernateMode, 0, "");
2024 SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
2025 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2026 gIOHibernateStats, hibernate_statistics_t, "");
2027
2028 SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
2029 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
2030 &gIOHibernateStats->graphicsReadyTime, 0, "");
2031 SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
2032 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
2033 &gIOHibernateStats->wakeNotificationTime, 0, "");
2034 SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
2035 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
2036 &gIOHibernateStats->lockScreenReadyTime, 0, "");
2037 SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
2038 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
2039 &gIOHibernateStats->hidReadyTime, 0, "");
2040
2041
2042 void
2043 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
2044 {
2045 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
2046 if (data)
2047 {
2048 rootDomain->setProperty(kIOHibernateStateKey, data);
2049 data->release();
2050 }
2051
2052 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))
2053 gIOHibernateMode = kIOHibernateModeOn;
2054 else
2055 gIOHibernateFilename[0] = 0;
2056
2057 sysctl_register_oid(&sysctl__kern_hibernatefile);
2058 sysctl_register_oid(&sysctl__kern_bootsignature);
2059 sysctl_register_oid(&sysctl__kern_hibernatemode);
2060 sysctl_register_oid(&sysctl__kern_hibernatestatistics);
2061 sysctl_register_oid(&sysctl__kern_hibernategraphicsready);
2062 sysctl_register_oid(&sysctl__kern_hibernatewakenotification);
2063 sysctl_register_oid(&sysctl__kern_hibernatelockscreenready);
2064 sysctl_register_oid(&sysctl__kern_hibernatehidready);
2065
2066 gFSLock = IOLockAlloc();
2067 }
2068
2069
2070 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2071
2072 static void
2073 hibernate_setup_for_wake(void)
2074 {
2075 }
2076
2077 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2078
2079 #define C_ASSERT(e) typedef char __C_ASSERT__[(e) ? 1 : -1]
2080
2081 static bool
2082 no_encrypt_page(vm_offset_t ppnum)
2083 {
2084 if (pmap_is_noencrypt((ppnum_t)ppnum) == TRUE)
2085 {
2086 return true;
2087 }
2088 return false;
2089 }
2090
2091 static void
2092 hibernate_pal_callback(void *vars_arg, vm_offset_t addr)
2093 {
2094 IOHibernateVars *vars = (IOHibernateVars *)vars_arg;
2095 /* Make sure it's not in either of the save lists */
2096 hibernate_set_page_state(vars->page_list, vars->page_list_wired, atop_64(addr), 1, kIOHibernatePageStateFree);
2097
2098 /* Set it in the bitmap of pages owned by the PAL */
2099 hibernate_page_bitset(vars->page_list_pal, TRUE, atop_64(addr));
2100 }
2101
2102 static struct hibernate_cryptvars_t *local_cryptvars;
2103
2104 extern "C" int
2105 hibernate_pal_write(void *buffer, size_t size)
2106 {
2107 IOHibernateVars * vars = &gIOHibernateVars;
2108
2109 IOReturn err = IOPolledFileWrite(vars->fileVars, (const uint8_t *)buffer, size, local_cryptvars);
2110 if (kIOReturnSuccess != err) {
2111 kprintf("epic hibernate fail! %d\n", err);
2112 return err;
2113 }
2114
2115 return 0;
2116 }
2117
2118
2119 extern "C" uint32_t
2120 hibernate_write_image(void)
2121 {
2122 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
2123 IOHibernateVars * vars = &gIOHibernateVars;
2124 IOPolledFileExtent * fileExtents;
2125
2126 C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
2127
2128 uint32_t pageCount, pagesDone;
2129 IOReturn err;
2130 vm_offset_t ppnum, page;
2131 IOItemCount count;
2132 uint8_t * src;
2133 uint8_t * data;
2134 uint8_t * compressed;
2135 uint8_t * scratch;
2136 void * zerosCompressed;
2137 IOByteCount pageCompressedSize, zerosCompressedLen;
2138 uint64_t compressedSize, uncompressedSize;
2139 uint64_t image1Size = 0;
2140 uint32_t bitmap_size;
2141 bool iterDone, pollerOpen, needEncrypt;
2142 uint32_t restore1Sum, sum, sum1, sum2;
2143 int wkresult;
2144 uint32_t tag;
2145 uint32_t pageType;
2146 uint32_t pageAndCount[2];
2147 addr64_t phys64;
2148 IOByteCount segLen;
2149
2150 AbsoluteTime startTime, endTime;
2151 AbsoluteTime allTime, compTime;
2152 uint64_t compBytes;
2153 uint64_t nsec;
2154 uint32_t lastProgressStamp = 0;
2155 uint32_t progressStamp;
2156 uint32_t blob, lastBlob = (uint32_t) -1L;
2157
2158 uint32_t wiredPagesEncrypted;
2159 uint32_t dirtyPagesEncrypted;
2160 uint32_t wiredPagesClear;
2161 uint32_t zeroPageCount;
2162
2163 hibernate_cryptvars_t _cryptvars;
2164 hibernate_cryptvars_t * cryptvars = 0;
2165
2166 wiredPagesEncrypted = 0;
2167 dirtyPagesEncrypted = 0;
2168 wiredPagesClear = 0;
2169 zeroPageCount = 0;
2170
2171 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
2172 return (false /* sleep */ );
2173
2174 if (kIOHibernateModeSleep & gIOHibernateMode)
2175 kdebug_enable = save_kdebug_enable;
2176
2177 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START, 0, 0, 0, 0, 0);
2178 IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
2179
2180 restore1Sum = sum1 = sum2 = 0;
2181
2182 hibernate_pal_prepare();
2183
2184 #if CRYPTO
2185 // encryption data. "iv" is the "initial vector".
2186 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2187 {
2188 static const unsigned char first_iv[AES_BLOCK_SIZE]
2189 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
2190 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
2191
2192 cryptvars = &gIOHibernateCryptWakeContext;
2193 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
2194 aes_encrypt_key(vars->cryptKey,
2195 kIOHibernateAESKeySize,
2196 &cryptvars->ctx.encrypt);
2197 aes_decrypt_key(vars->cryptKey,
2198 kIOHibernateAESKeySize,
2199 &cryptvars->ctx.decrypt);
2200
2201 cryptvars = &_cryptvars;
2202 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
2203 for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++)
2204 vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
2205 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
2206 aes_encrypt_key(vars->wiredCryptKey,
2207 kIOHibernateAESKeySize,
2208 &cryptvars->ctx.encrypt);
2209
2210 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
2211 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
2212 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
2213
2214 local_cryptvars = cryptvars;
2215 }
2216 #endif /* CRYPTO */
2217
2218 hibernate_setup_for_wake();
2219
2220 hibernate_page_list_setall(vars->page_list,
2221 vars->page_list_wired,
2222 vars->page_list_pal,
2223 false /* !preflight */,
2224 /* discard_all */
2225 ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
2226 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
2227 &pageCount);
2228
2229 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
2230
2231 fileExtents = (IOPolledFileExtent *) vars->fileExtents->getBytesNoCopy();
2232
2233 #if 0
2234 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
2235 for (page = 0; page < count; page++)
2236 {
2237 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
2238 fileExtents[page].start, fileExtents[page].length,
2239 fileExtents[page].start + fileExtents[page].length);
2240 }
2241 #endif
2242
2243 needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
2244 AbsoluteTime_to_scalar(&compTime) = 0;
2245 compBytes = 0;
2246
2247 clock_get_uptime(&allTime);
2248 IOService::getPMRootDomain()->pmStatsRecordEvent(
2249 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
2250 do
2251 {
2252 compressedSize = 0;
2253 uncompressedSize = 0;
2254 zeroPageCount = 0;
2255
2256 IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
2257
2258 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
2259 ml_get_interrupts_enabled());
2260 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledBeforeSleepState, vars->ioBuffer);
2261 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
2262 pollerOpen = (kIOReturnSuccess == err);
2263 if (!pollerOpen)
2264 break;
2265
2266 // copy file block extent list if larger than header
2267
2268 count = vars->fileExtents->getLength();
2269 if (count > sizeof(header->fileExtentMap))
2270 {
2271 count -= sizeof(header->fileExtentMap);
2272 err = IOPolledFileWrite(vars->fileVars,
2273 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
2274 if (kIOReturnSuccess != err)
2275 break;
2276 }
2277
2278 uintptr_t hibernateBase;
2279 uintptr_t hibernateEnd;
2280
2281 hibernateBase = HIB_BASE; /* Defined in PAL headers */
2282
2283 hibernateEnd = (segHIBB + segSizeHIB);
2284
2285 // copy out restore1 code
2286
2287 for (count = 0;
2288 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2289 count += segLen)
2290 {
2291 for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++)
2292 {
2293 gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
2294 }
2295 }
2296
2297 page = atop_32(kvtophys(hibernateBase));
2298 count = atop_32(round_page(hibernateEnd) - hibernateBase);
2299 header->restore1CodePhysPage = page;
2300 header->restore1CodeVirt = hibernateBase;
2301 header->restore1PageCount = count;
2302 header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
2303 header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
2304
2305 // sum __HIB seg, with zeros for the stack
2306 src = (uint8_t *) trunc_page(hibernateBase);
2307 for (page = 0; page < count; page++)
2308 {
2309 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
2310 restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
2311 else
2312 restore1Sum += 0x00000000;
2313 src += page_size;
2314 }
2315 sum1 = restore1Sum;
2316
2317 // write the __HIB seg, with zeros for the stack
2318
2319 src = (uint8_t *) trunc_page(hibernateBase);
2320 count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
2321 if (count)
2322 {
2323 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
2324 if (kIOReturnSuccess != err)
2325 break;
2326 }
2327 err = IOPolledFileWrite(vars->fileVars,
2328 (uint8_t *) 0,
2329 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
2330 cryptvars);
2331 if (kIOReturnSuccess != err)
2332 break;
2333 src = &gIOHibernateRestoreStackEnd[0];
2334 count = round_page(hibernateEnd) - ((uintptr_t) src);
2335 if (count)
2336 {
2337 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
2338 if (kIOReturnSuccess != err)
2339 break;
2340 }
2341
2342 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2343 {
2344 vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
2345 vars->fileVars->encryptEnd = UINT64_MAX;
2346 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
2347 }
2348
2349 // write the preview buffer
2350
2351 if (vars->previewBuffer)
2352 {
2353 ppnum = 0;
2354 count = 0;
2355 do
2356 {
2357 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
2358 pageAndCount[0] = atop_64(phys64);
2359 pageAndCount[1] = atop_32(segLen);
2360 err = IOPolledFileWrite(vars->fileVars,
2361 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
2362 cryptvars);
2363 if (kIOReturnSuccess != err)
2364 break;
2365 count += segLen;
2366 ppnum += sizeof(pageAndCount);
2367 }
2368 while (phys64);
2369 if (kIOReturnSuccess != err)
2370 break;
2371
2372 src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
2373
2374 ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
2375
2376 count = vars->previewBuffer->getLength();
2377
2378 header->previewPageListSize = ppnum;
2379 header->previewSize = count + ppnum;
2380
2381 for (page = 0; page < count; page += page_size)
2382 {
2383 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
2384 sum1 += hibernate_sum_page(src + page, atop_64(phys64));
2385 }
2386 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
2387 if (kIOReturnSuccess != err)
2388 break;
2389 }
2390
2391 // mark areas for no save
2392
2393 for (count = 0;
2394 (phys64 = vars->ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2395 count += segLen)
2396 {
2397 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2398 atop_64(phys64), atop_32(segLen),
2399 kIOHibernatePageStateFree);
2400 pageCount -= atop_32(segLen);
2401 }
2402
2403 for (count = 0;
2404 (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2405 count += segLen)
2406 {
2407 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2408 atop_64(phys64), atop_32(segLen),
2409 kIOHibernatePageStateFree);
2410 pageCount -= atop_32(segLen);
2411 }
2412
2413 // copy out bitmap of pages available for trashing during restore
2414
2415 bitmap_size = vars->page_list_wired->list_size;
2416 src = (uint8_t *) vars->page_list_wired;
2417 err = IOPolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
2418 if (kIOReturnSuccess != err)
2419 break;
2420
2421 // mark more areas for no save, but these are not available
2422 // for trashing during restore
2423
2424 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
2425
2426
2427 page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
2428 count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
2429 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2430 page, count,
2431 kIOHibernatePageStateFree);
2432 pageCount -= count;
2433
2434 if (vars->previewBuffer) for (count = 0;
2435 (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2436 count += segLen)
2437 {
2438 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2439 atop_64(phys64), atop_32(segLen),
2440 kIOHibernatePageStateFree);
2441 pageCount -= atop_32(segLen);
2442 }
2443
2444 for (count = 0;
2445 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2446 count += segLen)
2447 {
2448 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2449 atop_64(phys64), atop_32(segLen),
2450 kIOHibernatePageStateFree);
2451 pageCount -= atop_32(segLen);
2452 }
2453
2454 (void)hibernate_pal_callback;
2455
2456 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2457 compressed = src + page_size;
2458 scratch = compressed + page_size;
2459
2460 // compress a zero page
2461 bzero(src, page_size);
2462 zerosCompressed = vars->handoffBuffer->getBytesNoCopy();
2463 zerosCompressedLen = WKdm_compress_new((WK_word*) src,
2464 (WK_word*) zerosCompressed,
2465 (WK_word*) scratch,
2466 page_size - 4);
2467
2468 pagesDone = 0;
2469 lastBlob = 0;
2470
2471 HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
2472 bitmap_size, header->previewSize,
2473 pageCount, vars->fileVars->position);
2474
2475 enum
2476 // pageType
2477 {
2478 kWired = 0x02,
2479 kEncrypt = 0x01,
2480 kWiredEncrypt = kWired | kEncrypt,
2481 kWiredClear = kWired,
2482 kUnwiredEncrypt = kEncrypt
2483 };
2484
2485 for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)
2486 {
2487 if (kUnwiredEncrypt == pageType)
2488 {
2489 // start unwired image
2490 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2491 {
2492 vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
2493 vars->fileVars->encryptEnd = UINT64_MAX;
2494 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
2495 }
2496 bcopy(&cryptvars->aes_iv[0],
2497 &gIOHibernateCryptWakeContext.aes_iv[0],
2498 sizeof(cryptvars->aes_iv));
2499 cryptvars = &gIOHibernateCryptWakeContext;
2500 }
2501 for (iterDone = false, ppnum = 0; !iterDone; )
2502 {
2503 count = hibernate_page_list_iterate((kWired & pageType)
2504 ? vars->page_list_wired : vars->page_list,
2505 &ppnum);
2506 // kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
2507 iterDone = !count;
2508
2509 if (count && (kWired & pageType) && needEncrypt)
2510 {
2511 uint32_t checkIndex;
2512 for (checkIndex = 0;
2513 (checkIndex < count)
2514 && (((kEncrypt & pageType) == 0) == no_encrypt_page(ppnum + checkIndex));
2515 checkIndex++)
2516 {}
2517 if (!checkIndex)
2518 {
2519 ppnum++;
2520 continue;
2521 }
2522 count = checkIndex;
2523 }
2524
2525 switch (pageType)
2526 {
2527 case kWiredEncrypt: wiredPagesEncrypted += count; break;
2528 case kWiredClear: wiredPagesClear += count; break;
2529 case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
2530 }
2531
2532 if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */}
2533 else
2534 {
2535 pageAndCount[0] = ppnum;
2536 pageAndCount[1] = count;
2537 err = IOPolledFileWrite(vars->fileVars,
2538 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
2539 cryptvars);
2540 if (kIOReturnSuccess != err)
2541 break;
2542 }
2543
2544 for (page = ppnum; page < (ppnum + count); page++)
2545 {
2546 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
2547 if (err)
2548 {
2549 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
2550 break;
2551 }
2552
2553 sum = hibernate_sum_page(src, page);
2554 if (kWired & pageType)
2555 sum1 += sum;
2556 else
2557 sum2 += sum;
2558
2559 clock_get_uptime(&startTime);
2560 wkresult = WKdm_compress_new((WK_word*) src,
2561 (WK_word*) compressed,
2562 (WK_word*) scratch,
2563 page_size - 4);
2564
2565 clock_get_uptime(&endTime);
2566 ADD_ABSOLUTETIME(&compTime, &endTime);
2567 SUB_ABSOLUTETIME(&compTime, &startTime);
2568
2569 compBytes += page_size;
2570 pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
2571
2572 if ((pageCompressedSize == zerosCompressedLen)
2573 && !bcmp(compressed, zerosCompressed, zerosCompressedLen))
2574 {
2575 pageCompressedSize = 0;
2576 zeroPageCount++;
2577 }
2578
2579 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2580 pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
2581
2582 if (pageCompressedSize != page_size)
2583 data = compressed;
2584 else
2585 data = src;
2586
2587 tag = pageCompressedSize | kIOHibernateTagSignature;
2588 err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
2589 if (kIOReturnSuccess != err)
2590 break;
2591
2592 err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
2593 if (kIOReturnSuccess != err)
2594 break;
2595
2596 compressedSize += pageCompressedSize;
2597 uncompressedSize += page_size;
2598 pagesDone++;
2599
2600 if (vars->consoleMapping && (0 == (1023 & pagesDone)))
2601 {
2602 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
2603 if (blob != lastBlob)
2604 {
2605 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
2606 lastBlob = blob;
2607 }
2608 }
2609 if (0 == (8191 & pagesDone))
2610 {
2611 clock_get_uptime(&endTime);
2612 SUB_ABSOLUTETIME(&endTime, &allTime);
2613 absolutetime_to_nanoseconds(endTime, &nsec);
2614 progressStamp = nsec / 750000000ULL;
2615 if (progressStamp != lastProgressStamp)
2616 {
2617 lastProgressStamp = progressStamp;
2618 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
2619 }
2620 }
2621 }
2622 if (kIOReturnSuccess != err)
2623 break;
2624 ppnum = page;
2625 }
2626
2627 if (kIOReturnSuccess != err)
2628 break;
2629
2630 if ((kEncrypt & pageType) && vars->fileVars->encryptStart)
2631 {
2632 vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
2633 HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
2634 }
2635
2636 if (kWiredEncrypt != pageType)
2637 {
2638 // end of image1/2 - fill to next block
2639 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2640 if (kIOReturnSuccess != err)
2641 break;
2642 }
2643 if (kWiredClear == pageType)
2644 {
2645 // enlarge wired image for test
2646 // err = IOPolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
2647
2648 // end wired image
2649 header->encryptStart = vars->fileVars->encryptStart;
2650 header->encryptEnd = vars->fileVars->encryptEnd;
2651 image1Size = vars->fileVars->position;
2652 HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
2653 image1Size, header->encryptStart, header->encryptEnd);
2654 }
2655 }
2656 if (kIOReturnSuccess != err)
2657 break;
2658
2659 // Header:
2660
2661 header->imageSize = vars->fileVars->position;
2662 header->image1Size = image1Size;
2663 header->bitmapSize = bitmap_size;
2664 header->pageCount = pageCount;
2665
2666 header->restore1Sum = restore1Sum;
2667 header->image1Sum = sum1;
2668 header->image2Sum = sum2;
2669 header->sleepTime = gIOLastSleepTime.tv_sec;
2670
2671 header->compression = (compressedSize << 8) / uncompressedSize;
2672 gIOHibernateCompression = header->compression;
2673
2674 count = vars->fileExtents->getLength();
2675 if (count > sizeof(header->fileExtentMap))
2676 {
2677 header->fileExtentMapSize = count;
2678 count = sizeof(header->fileExtentMap);
2679 }
2680 else
2681 header->fileExtentMapSize = sizeof(header->fileExtentMap);
2682 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
2683
2684 header->deviceBase = vars->fileVars->block0;
2685 header->deviceBlockSize = vars->fileVars->blockSize;
2686
2687 IOPolledFileSeek(vars->fileVars, 0);
2688 err = IOPolledFileWrite(vars->fileVars,
2689 (uint8_t *) header, sizeof(IOHibernateImageHeader),
2690 cryptvars);
2691 if (kIOReturnSuccess != err)
2692 break;
2693 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2694 if (kIOReturnSuccess != err)
2695 break;
2696 err = IOHibernatePollerIODone(vars->fileVars, true);
2697 if (kIOReturnSuccess != err)
2698 break;
2699 }
2700 while (false);
2701
2702 clock_get_uptime(&endTime);
2703
2704 IOService::getPMRootDomain()->pmStatsRecordEvent(
2705 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
2706
2707 SUB_ABSOLUTETIME(&endTime, &allTime);
2708 absolutetime_to_nanoseconds(endTime, &nsec);
2709 HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
2710
2711 absolutetime_to_nanoseconds(compTime, &nsec);
2712 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2713 compBytes,
2714 nsec / 1000000ULL,
2715 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2716
2717 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2718 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
2719 vars->fileVars->cryptBytes,
2720 nsec / 1000000ULL,
2721 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2722
2723 HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
2724 header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
2725 uncompressedSize, atop_32(uncompressedSize), compressedSize,
2726 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
2727 sum1, sum2);
2728
2729 HIBLOG("zeroPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
2730 zeroPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
2731
2732 if (vars->fileVars->io)
2733 (void) IOHibernatePollerIODone(vars->fileVars, false);
2734
2735 if (pollerOpen)
2736 IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
2737
2738 if (vars->consoleMapping)
2739 ProgressUpdate(gIOHibernateGraphicsInfo,
2740 vars->consoleMapping, 0, kIOHibernateProgressCount);
2741
2742 HIBLOG("hibernate_write_image done(%x)\n", err);
2743
2744 // should we come back via regular wake, set the state in memory.
2745 gIOHibernateState = kIOHibernateStateInactive;
2746
2747 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END,
2748 wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted, 0, 0);
2749
2750 if (kIOReturnSuccess == err)
2751 {
2752 if (kIOHibernateModeSleep & gIOHibernateMode)
2753 {
2754 return (kIOHibernatePostWriteSleep);
2755 }
2756 else if(kIOHibernateModeRestart & gIOHibernateMode)
2757 {
2758 return (kIOHibernatePostWriteRestart);
2759 }
2760 else
2761 {
2762 /* by default, power down */
2763 return (kIOHibernatePostWriteHalt);
2764 }
2765 }
2766 else if (kIOReturnAborted == err)
2767 {
2768 return (kIOHibernatePostWriteWake);
2769 }
2770 else
2771 {
2772 /* on error, sleep */
2773 return (kIOHibernatePostWriteSleep);
2774 }
2775 }
2776
2777 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2778
2779 extern "C" void
2780 hibernate_machine_init(void)
2781 {
2782 IOReturn err;
2783 uint32_t sum;
2784 uint32_t pagesDone;
2785 uint32_t pagesRead = 0;
2786 AbsoluteTime startTime, compTime;
2787 AbsoluteTime allTime, endTime;
2788 AbsoluteTime startIOTime, endIOTime;
2789 uint64_t nsec, nsecIO;
2790 uint64_t compBytes;
2791 uint32_t lastProgressStamp = 0;
2792 uint32_t progressStamp;
2793 hibernate_cryptvars_t * cryptvars = 0;
2794
2795 IOHibernateVars * vars = &gIOHibernateVars;
2796 bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
2797
2798 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
2799 return;
2800
2801 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2802 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2803
2804 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
2805 {
2806 HIBLOG("regular wake\n");
2807 return;
2808 }
2809
2810 HIBPRINT("diag %x %x %x %x\n",
2811 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
2812 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
2813
2814 #define t40ms(x) (tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)
2815 #define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
2816 tStat(booterStart, booterStart);
2817 gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart,
2818 tStat(booterDuration0, booterTime0);
2819 tStat(booterDuration1, booterTime1);
2820 tStat(booterDuration2, booterTime2);
2821 tStat(booterDuration, booterTime);
2822 tStat(booterConnectDisplayDuration, connectDisplayTime);
2823 tStat(booterSplashDuration, splashTime);
2824 tStat(trampolineDuration, trampolineTime);
2825
2826 gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size;
2827 gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize;
2828 gIOHibernateStats->image1Pages = pagesDone;
2829
2830 HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
2831 gIOHibernateStats->booterStart,
2832 gIOHibernateStats->smcStart,
2833 gIOHibernateStats->booterDuration0,
2834 gIOHibernateStats->booterDuration1,
2835 gIOHibernateStats->booterDuration2,
2836 gIOHibernateStats->booterDuration,
2837 gIOHibernateStats->booterConnectDisplayDuration,
2838 gIOHibernateStats->booterSplashDuration,
2839 gIOHibernateStats->trampolineDuration);
2840
2841 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
2842 gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
2843 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
2844
2845 if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
2846 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)))
2847 {
2848 hibernate_page_list_discard(vars->page_list);
2849 }
2850
2851 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
2852
2853 if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount)
2854 panic("handoff overflow");
2855
2856 IOHibernateHandoff * handoff;
2857 bool done = false;
2858 bool foundCryptData = false;
2859
2860 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
2861 !done;
2862 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
2863 {
2864 // HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
2865 uint8_t * data = &handoff->data[0];
2866 switch (handoff->type)
2867 {
2868 case kIOHibernateHandoffTypeEnd:
2869 done = true;
2870 break;
2871
2872 case kIOHibernateHandoffTypeGraphicsInfo:
2873 bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
2874 break;
2875
2876 case kIOHibernateHandoffTypeCryptVars:
2877 if (cryptvars)
2878 {
2879 hibernate_cryptwakevars_t *
2880 wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
2881 bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
2882 }
2883 foundCryptData = true;
2884 bzero(data, handoff->bytecount);
2885 break;
2886
2887 case kIOHibernateHandoffTypeMemoryMap:
2888
2889 clock_get_uptime(&allTime);
2890
2891 hibernate_newruntime_map(data, handoff->bytecount,
2892 gIOHibernateCurrentHeader->systemTableOffset);
2893
2894 clock_get_uptime(&endTime);
2895
2896 SUB_ABSOLUTETIME(&endTime, &allTime);
2897 absolutetime_to_nanoseconds(endTime, &nsec);
2898
2899 HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
2900
2901 break;
2902
2903 case kIOHibernateHandoffTypeDeviceTree:
2904 {
2905 // DTEntry chosen = NULL;
2906 // HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen));
2907 }
2908 break;
2909
2910 default:
2911 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
2912 break;
2913 }
2914 }
2915 if (cryptvars && !foundCryptData)
2916 panic("hibernate handoff");
2917
2918 HIBPRINT("video %x %d %d %d status %x\n",
2919 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2920 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
2921
2922 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)
2923 {
2924 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2925 * gIOHibernateGraphicsInfo->rowBytes);
2926 IOMapPages(kernel_map,
2927 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2928 vars->videoMapSize, kIOMapInhibitCache );
2929 }
2930
2931 if (vars->videoMapSize)
2932 ProgressUpdate(gIOHibernateGraphicsInfo,
2933 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2934
2935 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2936 uint8_t * compressed = src + page_size;
2937 uint8_t * scratch = compressed + page_size;
2938 uint32_t decoOffset;
2939
2940 clock_get_uptime(&allTime);
2941 AbsoluteTime_to_scalar(&compTime) = 0;
2942 compBytes = 0;
2943
2944 HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2945 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0);
2946 clock_get_uptime(&startIOTime);
2947 endTime = startIOTime;
2948 SUB_ABSOLUTETIME(&endTime, &allTime);
2949 absolutetime_to_nanoseconds(endTime, &nsec);
2950 HIBLOG("IOHibernatePollerOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
2951
2952 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2953
2954 // kick off the read ahead
2955 vars->fileVars->io = false;
2956 vars->fileVars->bufferHalf = 0;
2957 vars->fileVars->bufferLimit = 0;
2958 vars->fileVars->lastRead = 0;
2959 vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
2960 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2961 vars->fileVars->cryptBytes = 0;
2962 AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
2963
2964 err = IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
2965 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2966 // --
2967
2968 HIBLOG("hibernate_machine_init reading\n");
2969
2970 uint32_t * header = (uint32_t *) src;
2971 sum = 0;
2972
2973 while (kIOReturnSuccess == err)
2974 {
2975 unsigned int count;
2976 unsigned int page;
2977 uint32_t tag;
2978 vm_offset_t ppnum, compressedSize;
2979
2980 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2981 if (kIOReturnSuccess != err)
2982 break;
2983
2984 ppnum = header[0];
2985 count = header[1];
2986
2987 // HIBPRINT("(%x, %x)\n", ppnum, count);
2988
2989 if (!count)
2990 break;
2991
2992 for (page = 0; page < count; page++)
2993 {
2994 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2995 if (kIOReturnSuccess != err)
2996 break;
2997
2998 compressedSize = kIOHibernateTagLength & tag;
2999 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength))
3000 {
3001 err = kIOReturnIPCError;
3002 break;
3003 }
3004
3005 if (!compressedSize) bzero_phys(ptoa_64(ppnum), page_size);
3006 else
3007 {
3008 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
3009 if (kIOReturnSuccess != err) break;
3010 if (compressedSize < page_size)
3011 {
3012 decoOffset = page_size;
3013 clock_get_uptime(&startTime);
3014 WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, page_size);
3015 clock_get_uptime(&endTime);
3016 ADD_ABSOLUTETIME(&compTime, &endTime);
3017 SUB_ABSOLUTETIME(&compTime, &startTime);
3018 compBytes += page_size;
3019 }
3020 else decoOffset = 0;
3021
3022 sum += hibernate_sum_page((src + decoOffset), ppnum);
3023 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
3024 if (err)
3025 {
3026 HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
3027 break;
3028 }
3029 }
3030
3031 ppnum++;
3032 pagesDone++;
3033 pagesRead++;
3034
3035 if (0 == (8191 & pagesDone))
3036 {
3037 clock_get_uptime(&endTime);
3038 SUB_ABSOLUTETIME(&endTime, &allTime);
3039 absolutetime_to_nanoseconds(endTime, &nsec);
3040 progressStamp = nsec / 750000000ULL;
3041 if (progressStamp != lastProgressStamp)
3042 {
3043 lastProgressStamp = progressStamp;
3044 HIBPRINT("pages %d (%d%%)\n", pagesDone,
3045 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
3046 }
3047 }
3048 }
3049 }
3050 if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages))
3051 err = kIOReturnLockedRead;
3052
3053 if (kIOReturnSuccess != err)
3054 panic("Hibernate restore error %x", err);
3055
3056 gIOHibernateCurrentHeader->actualImage2Sum = sum;
3057 gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
3058
3059 if (vars->fileVars->io)
3060 (void) IOHibernatePollerIODone(vars->fileVars, false);
3061
3062 clock_get_uptime(&endIOTime);
3063
3064 err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
3065
3066 clock_get_uptime(&endTime);
3067
3068 IOService::getPMRootDomain()->pmStatsRecordEvent(
3069 kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
3070 IOService::getPMRootDomain()->pmStatsRecordEvent(
3071 kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
3072
3073 SUB_ABSOLUTETIME(&endTime, &allTime);
3074 absolutetime_to_nanoseconds(endTime, &nsec);
3075
3076 SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
3077 absolutetime_to_nanoseconds(endIOTime, &nsecIO);
3078
3079 gIOHibernateStats->kernelImageReadDuration = nsec / 1000000ULL;
3080 gIOHibernateStats->imagePages = pagesDone;
3081
3082 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
3083 pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
3084 nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
3085
3086 absolutetime_to_nanoseconds(compTime, &nsec);
3087 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
3088 compBytes,
3089 nsec / 1000000ULL,
3090 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
3091
3092 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
3093 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
3094 vars->fileVars->cryptBytes,
3095 nsec / 1000000ULL,
3096 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
3097
3098 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_NONE, pagesRead, pagesDone, 0, 0, 0);
3099 }
3100
3101 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
3102
3103 void IOHibernateSetWakeCapabilities(uint32_t capability)
3104 {
3105 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
3106 {
3107 gIOHibernateStats->wakeCapability = capability;
3108 }
3109 }
3110
3111 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
3112
3113 void IOHibernateSystemRestart(void)
3114 {
3115 static uint8_t noteStore[32] __attribute__((aligned(32)));
3116 IORegistryEntry * regEntry;
3117 const OSSymbol * sym;
3118 OSData * noteProp;
3119 OSData * data;
3120 uintptr_t * smcVars;
3121 uint8_t * smcBytes;
3122 size_t len;
3123 addr64_t element;
3124
3125 data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
3126 if (!data) return;
3127
3128 smcVars = (typeof(smcVars)) data->getBytesNoCopy();
3129 smcBytes = (typeof(smcBytes)) smcVars[1];
3130 len = smcVars[0];
3131 if (len > sizeof(noteStore)) len = sizeof(noteStore);
3132 noteProp = OSData::withCapacity(3 * sizeof(element));
3133 if (!noteProp) return;
3134 element = len;
3135 noteProp->appendBytes(&element, sizeof(element));
3136 element = crc32(0, smcBytes, len);
3137 noteProp->appendBytes(&element, sizeof(element));
3138
3139 bcopy(smcBytes, noteStore, len);
3140 element = (addr64_t) &noteStore[0];
3141 element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
3142 noteProp->appendBytes(&element, sizeof(element));
3143
3144 if (!gIOOptionsEntry)
3145 {
3146 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
3147 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
3148 if (regEntry && !gIOOptionsEntry)
3149 regEntry->release();
3150 }
3151
3152 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
3153 if (gIOOptionsEntry && sym) gIOOptionsEntry->setProperty(sym, noteProp);
3154 if (noteProp) noteProp->release();
3155 if (sym) sym->release();
3156 }
3157
3158
3159
mirror of XNU