]> git.saurik.com Git - apple/xnu.git/blob - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blob
? search:


74b18022eda9786f3d8cb1c76a4c8c3140d9dd58
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
1 /*
2 * Copyright (c) 2004-2008 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29
30 /*
31
32 Sleep:
33
34 - PMRootDomain calls IOHibernateSystemSleep() before system sleep
35 (devices awake, normal execution context)
36 - IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
37 grabs its extents and searches for a polling driver willing to work with that IOMedia.
38 The BSD code makes an ioctl to the storage driver to get the partition base offset to
39 the disk, and other ioctls to get the transfer constraints
40 If successful, the file is written to make sure its initially not bootable (in case of
41 later failure) and nvram set to point to the first block of the file. (Has to be done
42 here so blocking is possible in nvram support).
43 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
44 page out any pages it wants to (currently zero, but probably some percentage of memory).
45 Its assumed just allocating pages will cause the VM system to naturally select the best
46 pages for eviction. It also copies processor flags needed for the restore path and sets
47 a flag in the boot processor proc info.
48 gIOHibernateState = kIOHibernateStateHibernating.
49 - Regular sleep progresses - some drivers may inspect the root domain property
50 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
51 as usual but leaves motherboard I/O on.
52 - Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
53 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
54 all ppc RC bits out of the hash table and caches into the mapping structures.
55 - hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
56 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
57 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
58 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
59 The image header and block list are written. The header includes the second file extent so
60 only the header block is needed to read the file, regardless of filesystem.
61 The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
62 (only) is used to decompress the image during wake/boot.
63 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
64 The bitmaps are written to the image.
65 More areas are removed from the bitmaps (after they have been written to the image) - the
66 segment "__HIB" pages and interrupt stack.
67 Each wired page is compressed and written and then each non-wired page. Compression and
68 disk writes are in parallel.
69 The image header is written to the start of the file and the polling driver closed.
70 The machine powers down (or sleeps).
71
72 Boot/Wake:
73
74 - BootX sees the boot-image nvram variable containing the device and block number of the image,
75 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
76 - BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
77 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
78 that is in the kernel's __HIB section.
79 - BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
80 hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
81 only code & data in that section is safe to call since all the other wired pages are still
82 compressed in the image.
83 - hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
84 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
85 location directly, and copies those that can't to interim free pages. When the image has been
86 completed, the copies are uncompressed, overwriting the wired image pages.
87 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
88 is used to get pages into place for 64bit.
89 - the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
90 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
91 are removed from the software strutures, and the hash table is reinitialized.
92 - After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
93 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
94 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
95 for the remaining non wired pages.
96 - The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
97 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
98 is closed via bsd.
99
100 Polled Mode I/O:
101
102 IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
103 registry, specifying an object of calls IOPolledInterface.
104
105 Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
106 partition) that the image is going to live, looking for polled interface properties. If it finds
107 one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
108 interfaces found are kept in an ordered list.
109
110 There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
111 few different contexts things happen in:
112
113 - there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
114 up and running) and after wake - this is safe to allocate memory and do anything. The device
115 ignores sleep requests from that point since its a waste of time if it goes to sleep and
116 immediately wakes back up for the image write.
117
118 - there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
119 immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
120 from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
121 used to flush and set the disk to sleep.
122
123 - there is an Open/Close (AfterSleep) pair made around the image read operations that happen
124 immediately after sleep. These can't block or allocate memory. This is happening after the platform
125 expert has woken the low level bits of the system, but most of the I/O system has not. There is only
126 one thread running.
127
128 For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
129 (prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
130 the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
131 Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
132 that is called for the hardware to check for events, and complete the I/O via the callback.
133 The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
134 to restrict I/O ops.
135 */
136
137 #include <sys/systm.h>
138
139 #include <IOKit/IOWorkLoop.h>
140 #include <IOKit/IOCommandGate.h>
141 #include <IOKit/IOTimerEventSource.h>
142 #include <IOKit/IOPlatformExpert.h>
143 #include <IOKit/IOKitDebug.h>
144 #include <IOKit/IOTimeStamp.h>
145 #include <IOKit/pwr_mgt/RootDomain.h>
146 #include <IOKit/pwr_mgt/IOPMPrivate.h>
147 #include <IOKit/IOMessage.h>
148 #include <IOKit/IODeviceTreeSupport.h>
149 #include <IOKit/IOBSD.h>
150 #include "RootDomainUserClient.h"
151 #include <IOKit/pwr_mgt/IOPowerConnection.h>
152 #include "IOPMPowerStateQueue.h"
153 #include <IOKit/IOBufferMemoryDescriptor.h>
154 #include <IOKit/AppleKeyStoreInterface.h>
155 #include <libkern/crypto/aes.h>
156
157 #include <sys/uio.h>
158 #include <sys/conf.h>
159 #include <sys/stat.h>
160 #include <sys/fcntl.h> // (FWRITE, ...)
161 #include <sys/sysctl.h>
162 #include <sys/kdebug.h>
163
164 #include <IOKit/IOHibernatePrivate.h>
165 #include <IOKit/IOPolledInterface.h>
166 #include <IOKit/IONVRAM.h>
167 #include "IOHibernateInternal.h"
168 #include <libkern/WKdm.h>
169 #include "IOKitKernelInternal.h"
170 #include <pexpert/device_tree.h>
171
172 #include <machine/pal_routines.h>
173 #include <machine/pal_hibernate.h>
174
175 extern "C" addr64_t kvtophys(vm_offset_t va);
176 extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
177
178 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
179
180 extern unsigned int save_kdebug_enable;
181 extern uint32_t gIOHibernateState;
182 uint32_t gIOHibernateMode;
183 static char gIOHibernateBootSignature[256+1];
184 static char gIOHibernateFilename[MAXPATHLEN+1];
185 static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
186 uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
187 static uint64_t gIOHibernateCompression = 0x80; // default compression 50%
188
189 static IODTNVRAM * gIOOptionsEntry;
190 static IORegistryEntry * gIOChosenEntry;
191 #if defined(__i386__) || defined(__x86_64__)
192 static const OSSymbol * gIOCreateEFIDevicePathSymbol;
193 static const OSSymbol * gIOHibernateRTCVariablesKey;
194 static const OSSymbol * gIOHibernateBoot0082Key;
195 static const OSSymbol * gIOHibernateBootNextKey;
196 static OSData * gIOHibernateBoot0082Data;
197 static OSData * gIOHibernateBootNextData;
198 static OSObject * gIOHibernateBootNextSave;
199 #endif
200
201 static IOLock * gFSLock;
202 static uint32_t gFSState;
203 static IOPolledFileIOVars gFileVars;
204 static IOHibernateVars gIOHibernateVars;
205 static struct kern_direct_file_io_ref_t * gIOHibernateFileRef;
206 static hibernate_cryptvars_t gIOHibernateCryptWakeContext;
207 static hibernate_graphics_t _hibernateGraphics;
208 static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
209
210 enum
211 {
212 kFSIdle = 0,
213 kFSOpening = 2,
214 kFSOpened = 3,
215 kFSTimedOut = 4,
216 };
217
218 static IOReturn IOHibernateDone(IOHibernateVars * vars);
219
220 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
221
222 enum { kXPRamAudioVolume = 8 };
223 enum { kDefaultIOSize = 128 * 1024 };
224 enum { kVideoMapSize = 32 * 1024 * 1024 };
225
226 #ifndef kIOMediaPreferredBlockSizeKey
227 #define kIOMediaPreferredBlockSizeKey "Preferred Block Size"
228 #endif
229
230 #ifndef kIOBootPathKey
231 #define kIOBootPathKey "bootpath"
232 #endif
233 #ifndef kIOSelectedBootDeviceKey
234 #define kIOSelectedBootDeviceKey "boot-device"
235 #endif
236
237
238 enum { kIOHibernateMinPollersNeeded = 2 };
239
240 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
241
242 // copy from phys addr to MD
243
244 static IOReturn
245 IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
246 IOByteCount offset, addr64_t bytes, IOByteCount length)
247 {
248 addr64_t srcAddr = bytes;
249 IOByteCount remaining;
250
251 remaining = length = min(length, md->getLength() - offset);
252 while (remaining) { // (process another target segment?)
253 addr64_t dstAddr64;
254 IOByteCount dstLen;
255
256 dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
257 if (!dstAddr64)
258 break;
259
260 // Clip segment length to remaining
261 if (dstLen > remaining)
262 dstLen = remaining;
263
264 #if 1
265 bcopy_phys(srcAddr, dstAddr64, dstLen);
266 #else
267 copypv(srcAddr, dstAddr64, dstLen,
268 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
269 #endif
270 srcAddr += dstLen;
271 offset += dstLen;
272 remaining -= dstLen;
273 }
274
275 assert(!remaining);
276
277 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
278 }
279
280 // copy from MD to phys addr
281
282 static IOReturn
283 IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
284 IOByteCount offset, addr64_t bytes, IOByteCount length)
285 {
286 addr64_t dstAddr = bytes;
287 IOByteCount remaining;
288
289 remaining = length = min(length, md->getLength() - offset);
290 while (remaining) { // (process another target segment?)
291 addr64_t srcAddr64;
292 IOByteCount dstLen;
293
294 srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
295 if (!srcAddr64)
296 break;
297
298 // Clip segment length to remaining
299 if (dstLen > remaining)
300 dstLen = remaining;
301
302 #if 1
303 bcopy_phys(srcAddr64, dstAddr, dstLen);
304 #else
305 copypv(srcAddr, dstAddr64, dstLen,
306 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
307 #endif
308 dstAddr += dstLen;
309 offset += dstLen;
310 remaining -= dstLen;
311 }
312
313 assert(!remaining);
314
315 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
316 }
317
318 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
319
320 void
321 hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
322 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
323 {
324 count += ppnum;
325 switch (kind)
326 {
327 case kIOHibernatePageStateUnwiredSave:
328 // unwired save
329 for (; ppnum < count; ppnum++)
330 {
331 hibernate_page_bitset(page_list, FALSE, ppnum);
332 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
333 }
334 break;
335 case kIOHibernatePageStateWiredSave:
336 // wired save
337 for (; ppnum < count; ppnum++)
338 {
339 hibernate_page_bitset(page_list, FALSE, ppnum);
340 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
341 }
342 break;
343 case kIOHibernatePageStateFree:
344 // free page
345 for (; ppnum < count; ppnum++)
346 {
347 hibernate_page_bitset(page_list, TRUE, ppnum);
348 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
349 }
350 break;
351 default:
352 panic("hibernate_set_page_state");
353 }
354 }
355
356 static vm_offset_t
357 hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
358 {
359 uint32_t page = *pPage;
360 uint32_t count;
361 hibernate_bitmap_t * bitmap;
362
363 while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
364 {
365 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
366 if (!count)
367 break;
368 page += count;
369 if (page <= bitmap->last_page)
370 break;
371 }
372
373 *pPage = page;
374 if (bitmap)
375 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
376 else
377 count = 0;
378
379 return (count);
380 }
381
382 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
383
384 static IOReturn
385 IOHibernatePollerProbe(IOPolledFileIOVars * vars, IOService * target)
386 {
387 IOReturn err = kIOReturnError;
388 int32_t idx;
389 IOPolledInterface * poller;
390
391 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
392 {
393 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
394 err = poller->probe(target);
395 if (err)
396 {
397 HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
398 break;
399 }
400 }
401
402 return (err);
403 }
404
405 static IOReturn
406 IOHibernatePollerOpen(IOPolledFileIOVars * vars, uint32_t state, IOMemoryDescriptor * md)
407 {
408 IOReturn err = kIOReturnError;
409 int32_t idx;
410 IOPolledInterface * poller;
411
412 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
413 {
414 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
415 err = poller->open(state, md);
416 if (err)
417 {
418 HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
419 break;
420 }
421 }
422
423 return (err);
424 }
425
426 static IOReturn
427 IOHibernatePollerClose(IOPolledFileIOVars * vars, uint32_t state)
428 {
429 IOReturn err = kIOReturnError;
430 int32_t idx;
431 IOPolledInterface * poller;
432
433 for (idx = 0;
434 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
435 idx++)
436 {
437 err = poller->close(state);
438 if (err)
439 HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
440 }
441
442 return (err);
443 }
444
445 static void
446 IOHibernatePollerIOComplete(void * target,
447 void * parameter,
448 IOReturn status,
449 UInt64 actualByteCount)
450 {
451 IOPolledFileIOVars * vars = (IOPolledFileIOVars *) parameter;
452
453 vars->ioStatus = status;
454 }
455
456 static IOReturn
457 IOHibernatePollerIO(IOPolledFileIOVars * vars,
458 uint32_t operation, uint32_t bufferOffset,
459 uint64_t deviceOffset, uint64_t length)
460 {
461
462 IOReturn err = kIOReturnError;
463 IOPolledInterface * poller;
464 IOPolledCompletion completion;
465
466 completion.target = 0;
467 completion.action = &IOHibernatePollerIOComplete;
468 completion.parameter = vars;
469
470 vars->ioStatus = -1;
471
472 poller = (IOPolledInterface *) vars->pollers->getObject(0);
473 err = poller->startIO(operation, bufferOffset, deviceOffset + vars->block0, length, completion);
474 if (err)
475 HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
476
477 return (err);
478 }
479
480 static IOReturn
481 IOHibernatePollerIODone(IOPolledFileIOVars * vars, bool abortable)
482 {
483 IOReturn err = kIOReturnSuccess;
484 int32_t idx = 0;
485 IOPolledInterface * poller;
486
487 while (-1 == vars->ioStatus)
488 {
489 for (idx = 0;
490 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
491 idx++)
492 {
493 IOReturn newErr;
494 newErr = poller->checkForWork();
495 if ((newErr == kIOReturnAborted) && !abortable)
496 newErr = kIOReturnSuccess;
497 if (kIOReturnSuccess == err)
498 err = newErr;
499 }
500 }
501
502 if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort())
503 {
504 err = kIOReturnAborted;
505 HIBLOG("IOPolledInterface::checkForWork sw abort\n");
506 }
507
508 if (err)
509 {
510 HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
511 }
512 else
513 {
514 err = vars->ioStatus;
515 if (kIOReturnSuccess != err)
516 HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
517 }
518
519 return (err);
520 }
521
522 IOReturn
523 IOPolledInterface::checkAllForWork(void)
524 {
525 IOReturn err = kIOReturnNotReady;
526 int32_t idx;
527 IOPolledInterface * poller;
528
529 IOHibernateVars * vars = &gIOHibernateVars;
530
531 if (!vars->fileVars || !vars->fileVars->pollers)
532 return (err);
533
534 for (idx = 0;
535 (poller = (IOPolledInterface *) vars->fileVars->pollers->getObject(idx));
536 idx++)
537 {
538 err = poller->checkForWork();
539 if (err)
540 HIBLOG("IOPolledInterface::checkAllForWork[%d] 0x%x\n", idx, err);
541 }
542
543 return (err);
544 }
545
546 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
547
548 struct _OpenFileContext
549 {
550 OSData * extents;
551 uint64_t size;
552 };
553
554 static void
555 file_extent_callback(void * ref, uint64_t start, uint64_t length)
556 {
557 _OpenFileContext * ctx = (_OpenFileContext *) ref;
558 IOPolledFileExtent extent;
559
560 extent.start = start;
561 extent.length = length;
562
563 ctx->extents->appendBytes(&extent, sizeof(extent));
564 ctx->size += length;
565 }
566
567 static IOService *
568 IOCopyMediaForDev(dev_t device)
569 {
570 OSDictionary * matching;
571 OSNumber * num;
572 OSIterator * iter;
573 IOService * result = 0;
574
575 matching = IOService::serviceMatching("IOMedia");
576 if (!matching)
577 return (0);
578 do
579 {
580 num = OSNumber::withNumber(major(device), 32);
581 if (!num)
582 break;
583 matching->setObject(kIOBSDMajorKey, num);
584 num->release();
585 num = OSNumber::withNumber(minor(device), 32);
586 if (!num)
587 break;
588 matching->setObject(kIOBSDMinorKey, num);
589 num->release();
590 if (!num)
591 break;
592 iter = IOService::getMatchingServices(matching);
593 if (iter)
594 {
595 result = (IOService *) iter->getNextObject();
596 result->retain();
597 iter->release();
598 }
599 }
600 while (false);
601 matching->release();
602
603 return (result);
604 }
605
606 IOReturn
607 IOPolledFileOpen( const char * filename, uint64_t setFileSize,
608 IOBufferMemoryDescriptor * ioBuffer,
609 IOPolledFileIOVars ** fileVars, OSData ** fileExtents,
610 OSData ** imagePath, uint8_t * volumeCryptKey)
611 {
612 IOReturn err = kIOReturnSuccess;
613 IOPolledFileIOVars * vars;
614 _OpenFileContext ctx;
615 OSData * extentsData;
616 OSNumber * num;
617 IOService * part = 0;
618 OSString * keyUUID = 0;
619 OSString * keyStoreUUID = 0;
620 dev_t block_dev;
621 dev_t hibernate_image_dev;
622 uint64_t maxiobytes;
623 AbsoluteTime startTime, endTime;
624 uint64_t nsec;
625
626 vars = IONew(IOPolledFileIOVars, 1);
627 if (!vars) return (kIOReturnNoMemory);
628 bzero(vars, sizeof(*vars));
629
630 do
631 {
632 HIBLOG("sizeof(IOHibernateImageHeader) == %ld\n", sizeof(IOHibernateImageHeader));
633 if (sizeof(IOHibernateImageHeader) != 512)
634 continue;
635
636 vars->io = false;
637 vars->buffer = (uint8_t *) ioBuffer->getBytesNoCopy();
638 vars->bufferHalf = 0;
639 vars->bufferOffset = 0;
640 vars->bufferSize = ioBuffer->getLength() >> 1;
641
642 extentsData = OSData::withCapacity(32);
643 ctx.extents = extentsData;
644 ctx.size = 0;
645 clock_get_uptime(&startTime);
646 vars->fileRef = kern_open_file_for_direct_io(filename,
647 &file_extent_callback, &ctx,
648 setFileSize,
649 // write file:
650 0, (caddr_t) gIOHibernateCurrentHeader,
651 sizeof(IOHibernateImageHeader),
652 // results
653 &block_dev,
654 &hibernate_image_dev,
655 &vars->block0,
656 &maxiobytes,
657 &vars->flags);
658 #if 0
659 uint32_t msDelay = (131071 & random());
660 HIBLOG("sleep %d\n", msDelay);
661 IOSleep(msDelay);
662 #endif
663 clock_get_uptime(&endTime);
664 SUB_ABSOLUTETIME(&endTime, &startTime);
665 absolutetime_to_nanoseconds(endTime, &nsec);
666
667 if (!vars->fileRef) err = kIOReturnNoSpace;
668
669 IOLockLock(gFSLock);
670 if (kFSOpening != gFSState) err = kIOReturnTimeout;
671 IOLockUnlock(gFSLock);
672
673 HIBLOG("kern_open_file_for_direct_io(%d) took %qd ms\n", err, nsec / 1000000ULL);
674 if (kIOReturnSuccess != err) break;
675
676 if (kIOHibernateModeSSDInvert & gIOHibernateMode)
677 vars->flags ^= kIOHibernateOptionSSD;
678
679 HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx ssd %d\n", filename, ctx.size,
680 vars->block0, maxiobytes, kIOHibernateOptionSSD & vars->flags);
681 if (ctx.size < 1*1024*1024) // check against image size estimate!
682 {
683 err = kIOReturnNoSpace;
684 break;
685 }
686
687 vars->fileSize = ctx.size;
688 if (maxiobytes < vars->bufferSize)
689 vars->bufferSize = maxiobytes;
690
691 vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
692
693 part = IOCopyMediaForDev(block_dev);
694 if (!part)
695 break;
696
697 err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false,
698 (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
699 if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID)
700 {
701 // IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
702 uuid_t volumeKeyUUID;
703 aks_volume_key_t vek;
704 static IOService * sKeyStore;
705 static const OSSymbol * sAKSGetKey;
706
707 if (!sAKSGetKey)
708 sAKSGetKey = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY);
709 if (!sKeyStore)
710 sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
711 if (sKeyStore)
712 err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
713 else
714 err = kIOReturnNoResources;
715 if (kIOReturnSuccess == err)
716 err = sKeyStore->callPlatformFunction(sAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
717 if (kIOReturnSuccess != err)
718 IOLog("volume key err 0x%x\n", err);
719 else
720 {
721 size_t bytes = (kIOHibernateAESKeySize / 8);
722 if (vek.key.keybytecount < bytes)
723 bytes = vek.key.keybytecount;
724 bcopy(&vek.key.keybytes[0], volumeCryptKey, bytes);
725 }
726 bzero(&vek, sizeof(vek));
727 }
728 part->release();
729
730 part = IOCopyMediaForDev(hibernate_image_dev);
731 if (!part)
732 break;
733
734 IORegistryEntry * next;
735 IORegistryEntry * child;
736 OSData * data;
737
738 vars->pollers = OSArray::withCapacity(4);
739 if (!vars->pollers)
740 break;
741
742 vars->blockSize = 512;
743 next = part;
744 do
745 {
746 IOPolledInterface * poller;
747 OSObject * obj;
748
749 obj = next->getProperty(kIOPolledInterfaceSupportKey);
750 if (kOSBooleanFalse == obj)
751 {
752 vars->pollers->flushCollection();
753 break;
754 }
755 else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
756 vars->pollers->setObject(poller);
757 if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey))))
758 vars->blockSize = num->unsigned32BitValue();
759 child = next;
760 }
761 while ((next = child->getParentEntry(gIOServicePlane))
762 && child->isParent(next, gIOServicePlane, true));
763
764 HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
765 major(hibernate_image_dev), minor(hibernate_image_dev), (long)vars->blockSize, vars->pollers->getCount());
766 if (vars->pollers->getCount() < kIOHibernateMinPollersNeeded)
767 continue;
768
769 err = IOHibernatePollerProbe(vars, (IOService *) part);
770 if (kIOReturnSuccess != err)
771 break;
772
773 err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer);
774 if (kIOReturnSuccess != err)
775 break;
776
777 *fileVars = vars;
778 *fileExtents = extentsData;
779
780 // make imagePath
781
782 if ((extentsData->getLength() >= sizeof(IOPolledFileExtent)))
783 {
784 char str2[24 + sizeof(uuid_string_t) + 2];
785
786 #if defined(__i386__) || defined(__x86_64__)
787 if (!gIOCreateEFIDevicePathSymbol)
788 gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
789
790 if (keyUUID)
791 snprintf(str2, sizeof(str2), "%qx:%s",
792 vars->extentMap[0].start, keyUUID->getCStringNoCopy());
793 else
794 snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
795
796 err = IOService::getPlatform()->callPlatformFunction(
797 gIOCreateEFIDevicePathSymbol, false,
798 (void *) part, (void *) str2,
799 (void *) (uintptr_t) true, (void *) &data);
800 #else
801 char str1[256];
802 int len = sizeof(str1);
803
804 if (!part->getPath(str1, &len, gIODTPlane))
805 err = kIOReturnNotFound;
806 else
807 {
808 snprintf(str2, sizeof(str2), ",%qx", vars->extentMap[0].start);
809 // (strip the plane name)
810 char * tail = strchr(str1, ':');
811 if (!tail)
812 tail = str1 - 1;
813 data = OSData::withBytes(tail + 1, strlen(tail + 1));
814 data->appendBytes(str2, strlen(str2));
815 }
816 #endif
817 if (kIOReturnSuccess == err)
818 *imagePath = data;
819 else
820 HIBLOG("error 0x%x getting path\n", err);
821 }
822 }
823 while (false);
824
825 if (kIOReturnSuccess != err)
826 {
827 HIBLOG("error 0x%x opening hibernation file\n", err);
828 if (vars->fileRef)
829 {
830 kern_close_file_for_direct_io(vars->fileRef, 0, 0, 0, 0, 0);
831 vars->fileRef = NULL;
832 }
833 }
834
835 if (part)
836 part->release();
837
838 return (err);
839 }
840
841 IOReturn
842 IOPolledFileClose( IOPolledFileIOVars * vars )
843 {
844 if (vars->pollers)
845 {
846 IOHibernatePollerClose(vars, kIOPolledPostflightState);
847 vars->pollers->release();
848 }
849
850 bzero(vars, sizeof(IOPolledFileIOVars));
851
852 return (kIOReturnSuccess);
853 }
854
855 static IOReturn
856 IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
857 {
858 IOPolledFileExtent * extentMap;
859
860 extentMap = vars->extentMap;
861
862 vars->position = position;
863
864 while (position >= extentMap->length)
865 {
866 position -= extentMap->length;
867 extentMap++;
868 }
869
870 vars->currentExtent = extentMap;
871 vars->extentRemaining = extentMap->length - position;
872 vars->extentPosition = vars->position - position;
873
874 if (vars->bufferSize <= vars->extentRemaining)
875 vars->bufferLimit = vars->bufferSize;
876 else
877 vars->bufferLimit = vars->extentRemaining;
878
879 return (kIOReturnSuccess);
880 }
881
882 static IOReturn
883 IOPolledFileWrite(IOPolledFileIOVars * vars,
884 const uint8_t * bytes, IOByteCount size,
885 hibernate_cryptvars_t * cryptvars)
886 {
887 IOReturn err = kIOReturnSuccess;
888 IOByteCount copy;
889 bool flush = false;
890
891 do
892 {
893 if (!bytes && !size)
894 {
895 // seek to end of block & flush
896 size = vars->position & (vars->blockSize - 1);
897 if (size)
898 size = vars->blockSize - size;
899 flush = true;
900 // use some garbage for the fill
901 bytes = vars->buffer + vars->bufferOffset;
902 }
903
904 copy = vars->bufferLimit - vars->bufferOffset;
905 if (copy > size)
906 copy = size;
907 else
908 flush = true;
909
910 if (bytes)
911 {
912 bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
913 bytes += copy;
914 }
915 else
916 bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
917
918 size -= copy;
919 vars->bufferOffset += copy;
920 vars->position += copy;
921
922 if (flush && vars->bufferOffset)
923 {
924 uint64_t offset = (vars->position - vars->bufferOffset
925 - vars->extentPosition + vars->currentExtent->start);
926 uint32_t length = (vars->bufferOffset);
927
928 #if CRYPTO
929 if (cryptvars && vars->encryptStart
930 && (vars->position > vars->encryptStart)
931 && ((vars->position - length) < vars->encryptEnd))
932 {
933 AbsoluteTime startTime, endTime;
934
935 uint64_t encryptLen, encryptStart;
936 encryptLen = vars->position - vars->encryptStart;
937 if (encryptLen > length)
938 encryptLen = length;
939 encryptStart = length - encryptLen;
940 if (vars->position > vars->encryptEnd)
941 encryptLen -= (vars->position - vars->encryptEnd);
942
943 clock_get_uptime(&startTime);
944
945 // encrypt the buffer
946 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
947 &cryptvars->aes_iv[0],
948 encryptLen / AES_BLOCK_SIZE,
949 vars->buffer + vars->bufferHalf + encryptStart,
950 &cryptvars->ctx.encrypt);
951
952 clock_get_uptime(&endTime);
953 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
954 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
955 vars->cryptBytes += encryptLen;
956
957 // save initial vector for following encrypts
958 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
959 &cryptvars->aes_iv[0],
960 AES_BLOCK_SIZE);
961 }
962 #endif /* CRYPTO */
963
964 if (vars->io)
965 {
966 err = IOHibernatePollerIODone(vars, true);
967 if (kIOReturnSuccess != err)
968 break;
969 }
970
971 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
972 //if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
973
974 err = IOHibernatePollerIO(vars, kIOPolledWrite, vars->bufferHalf, offset, length);
975 if (kIOReturnSuccess != err)
976 break;
977 vars->io = true;
978
979 vars->extentRemaining -= vars->bufferOffset;
980 if (!vars->extentRemaining)
981 {
982 vars->currentExtent++;
983 vars->extentRemaining = vars->currentExtent->length;
984 vars->extentPosition = vars->position;
985 if (!vars->extentRemaining)
986 {
987 err = kIOReturnOverrun;
988 break;
989 }
990 }
991
992 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
993 vars->bufferOffset = 0;
994 if (vars->bufferSize <= vars->extentRemaining)
995 vars->bufferLimit = vars->bufferSize;
996 else
997 vars->bufferLimit = vars->extentRemaining;
998
999 flush = false;
1000 }
1001 }
1002 while (size);
1003
1004 return (err);
1005 }
1006
1007 static IOReturn
1008 IOPolledFileRead(IOPolledFileIOVars * vars,
1009 uint8_t * bytes, IOByteCount size,
1010 hibernate_cryptvars_t * cryptvars)
1011 {
1012 IOReturn err = kIOReturnSuccess;
1013 IOByteCount copy;
1014
1015 // bytesWritten += size;
1016
1017 do
1018 {
1019 copy = vars->bufferLimit - vars->bufferOffset;
1020 if (copy > size)
1021 copy = size;
1022
1023 if (bytes)
1024 {
1025 bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
1026 bytes += copy;
1027 }
1028 size -= copy;
1029 vars->bufferOffset += copy;
1030 // vars->position += copy;
1031
1032 if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd))
1033 {
1034 if (vars->io)
1035 {
1036 err = IOHibernatePollerIODone(vars, false);
1037 if (kIOReturnSuccess != err)
1038 break;
1039 }
1040 else
1041 cryptvars = 0;
1042
1043 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
1044
1045 vars->position += vars->lastRead;
1046 vars->extentRemaining -= vars->lastRead;
1047 vars->bufferLimit = vars->lastRead;
1048
1049 if (!vars->extentRemaining)
1050 {
1051 vars->currentExtent++;
1052 vars->extentRemaining = vars->currentExtent->length;
1053 vars->extentPosition = vars->position;
1054 if (!vars->extentRemaining)
1055 {
1056 err = kIOReturnOverrun;
1057 break;
1058 }
1059 }
1060
1061 uint64_t length;
1062 uint64_t lastReadLength = vars->lastRead;
1063 uint64_t offset = (vars->position
1064 - vars->extentPosition + vars->currentExtent->start);
1065 if (vars->extentRemaining <= vars->bufferSize)
1066 length = vars->extentRemaining;
1067 else
1068 length = vars->bufferSize;
1069 if ((length + vars->position) > vars->readEnd)
1070 length = vars->readEnd - vars->position;
1071
1072 vars->lastRead = length;
1073 if (length)
1074 {
1075 //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
1076 err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
1077 if (kIOReturnSuccess != err)
1078 break;
1079 vars->io = true;
1080 }
1081
1082 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1083 vars->bufferOffset = 0;
1084
1085 #if CRYPTO
1086 if (cryptvars)
1087 {
1088 uint8_t thisVector[AES_BLOCK_SIZE];
1089 AbsoluteTime startTime, endTime;
1090
1091 // save initial vector for following decrypts
1092 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
1093 bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
1094 &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1095
1096 // decrypt the buffer
1097 clock_get_uptime(&startTime);
1098
1099 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
1100 &thisVector[0],
1101 lastReadLength / AES_BLOCK_SIZE,
1102 vars->buffer + vars->bufferHalf,
1103 &cryptvars->ctx.decrypt);
1104
1105 clock_get_uptime(&endTime);
1106 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
1107 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
1108 vars->cryptBytes += lastReadLength;
1109 }
1110 #endif /* CRYPTO */
1111 }
1112 }
1113 while (size);
1114
1115 return (err);
1116 }
1117
1118 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1119
1120 IOReturn
1121 IOHibernateSystemSleep(void)
1122 {
1123 IOReturn err;
1124 OSData * data;
1125 OSObject * obj;
1126 OSString * str;
1127 OSNumber * num;
1128 bool dsSSD;
1129 IOHibernateVars * vars;
1130
1131 gIOHibernateState = kIOHibernateStateInactive;
1132
1133 if (!gIOChosenEntry)
1134 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1135
1136 gIOHibernateDebugFlags = 0;
1137 if (kIOLogHibernate & gIOKitDebug)
1138 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
1139
1140 if (IOService::getPMRootDomain()->getHibernateSettings(
1141 &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime))
1142 {
1143 if (kIOHibernateModeSleep & gIOHibernateMode)
1144 // default to discard clean for safe sleep
1145 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
1146 | kIOHibernateModeDiscardCleanActive);
1147 }
1148
1149 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
1150 {
1151 if ((str = OSDynamicCast(OSString, obj)))
1152 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
1153 sizeof(gIOHibernateFilename));
1154 obj->release();
1155 }
1156
1157 if (!gIOHibernateMode || !gIOHibernateFilename[0])
1158 return (kIOReturnUnsupported);
1159
1160 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
1161
1162 vars = IONew(IOHibernateVars, 1);
1163 if (!vars) return (kIOReturnNoMemory);
1164 bzero(vars, sizeof(*vars));
1165
1166 IOLockLock(gFSLock);
1167 if (kFSIdle != gFSState)
1168 {
1169 HIBLOG("hibernate file busy\n");
1170 IOLockUnlock(gFSLock);
1171 IODelete(vars, IOHibernateVars, 1);
1172 return (kIOReturnBusy);
1173 }
1174 gFSState = kFSOpening;
1175 IOLockUnlock(gFSLock);
1176
1177 do
1178 {
1179 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1180 4 * page_size, page_size);
1181 vars->ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1182 2 * kDefaultIOSize, page_size);
1183
1184 vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1185 ptoa_64(gIOHibernateHandoffPageCount), page_size);
1186
1187 if (!vars->srcBuffer || !vars->ioBuffer || !vars->handoffBuffer)
1188 {
1189 err = kIOReturnNoMemory;
1190 break;
1191 }
1192
1193 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey)))
1194 {
1195 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMinSize = num->unsigned64BitValue();
1196 obj->release();
1197 }
1198 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey)))
1199 {
1200 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMaxSize = num->unsigned64BitValue();
1201 obj->release();
1202 }
1203
1204 boolean_t encryptedswap = true;
1205 uint32_t pageCount;
1206 AbsoluteTime startTime, endTime;
1207 uint64_t nsec;
1208
1209 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1210 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
1211 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1212
1213 dsSSD = (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey));
1214 clock_get_uptime(&startTime);
1215 err = hibernate_setup(gIOHibernateCurrentHeader,
1216 gIOHibernateFreeRatio, gIOHibernateFreeTime,
1217 dsSSD,
1218 &vars->page_list, &vars->page_list_wired, &vars->page_list_pal);
1219 clock_get_uptime(&endTime);
1220 SUB_ABSOLUTETIME(&endTime, &startTime);
1221 absolutetime_to_nanoseconds(endTime, &nsec);
1222 HIBLOG("hibernate_setup(%d) took %qd ms\n", err, nsec / 1000000ULL);
1223
1224 uint64_t setFileSize = 0;
1225
1226 if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode))
1227 {
1228 hibernate_page_list_setall(vars->page_list,
1229 vars->page_list_wired,
1230 vars->page_list_pal,
1231 true /* preflight */,
1232 &pageCount);
1233 PE_Video consoleInfo;
1234 bzero(&consoleInfo, sizeof(consoleInfo));
1235 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1236
1237 // estimate: 5% increase in pages compressed
1238 // screen preview 2 images compressed 50%
1239 setFileSize = ((ptoa_64((105 * pageCount) / 100) * gIOHibernateCompression) >> 8)
1240 + vars->page_list->list_size
1241 + (consoleInfo.v_width * consoleInfo.v_height * 4);
1242
1243 HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
1244 pageCount, (100ULL * gIOHibernateCompression) >> 8,
1245 setFileSize, vars->fileMinSize);
1246
1247 if (!(kIOHibernateModeFileResize & gIOHibernateMode)
1248 && (setFileSize < vars->fileMinSize))
1249 {
1250 setFileSize = vars->fileMinSize;
1251 }
1252 }
1253
1254 // open & invalidate the image file
1255
1256 err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, vars->ioBuffer,
1257 &vars->fileVars, &vars->fileExtents, &data,
1258 &vars->volumeCryptKey[0]);
1259 if (KERN_SUCCESS != err)
1260 {
1261 HIBLOG("IOPolledFileOpen(%x)\n", err);
1262 break;
1263 }
1264
1265 dsSSD = ((0 != (kIOHibernateOptionSSD & vars->fileVars->flags))
1266 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
1267 if (dsSSD)
1268 {
1269 gIOHibernateCurrentHeader->options |=
1270 kIOHibernateOptionSSD
1271 | kIOHibernateOptionColor;
1272
1273 #if defined(__i386__) || defined(__x86_64__)
1274 if (!uuid_is_null(vars->volumeCryptKey) &&
1275 (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey)))
1276 {
1277 uintptr_t smcVars[2];
1278 smcVars[0] = sizeof(vars->volumeCryptKey);
1279 smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
1280
1281 IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
1282 bzero(smcVars, sizeof(smcVars));
1283 }
1284 #endif
1285 }
1286 else
1287 {
1288 gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
1289 }
1290
1291
1292 if (KERN_SUCCESS != err)
1293 break;
1294
1295 if (encryptedswap || !uuid_is_null(vars->volumeCryptKey))
1296 gIOHibernateMode ^= kIOHibernateModeEncrypt;
1297
1298 if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1299 {
1300 vars->videoAllocSize = kVideoMapSize;
1301 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
1302 vars->videoMapping = 0;
1303 }
1304
1305 // generate crypt keys
1306 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
1307 vars->wiredCryptKey[i] = random();
1308 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
1309 vars->cryptKey[i] = random();
1310
1311 // set nvram
1312
1313 IORegistryEntry * regEntry;
1314 if (!gIOOptionsEntry)
1315 {
1316 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
1317 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
1318 if (regEntry && !gIOOptionsEntry)
1319 regEntry->release();
1320 }
1321
1322 if (gIOOptionsEntry)
1323 {
1324 const OSSymbol * sym;
1325
1326 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1327 if (sym)
1328 {
1329 gIOOptionsEntry->setProperty(sym, data);
1330 sym->release();
1331 }
1332 data->release();
1333
1334 #if defined(__i386__) || defined(__x86_64__)
1335 struct AppleRTCHibernateVars
1336 {
1337 uint8_t signature[4];
1338 uint32_t revision;
1339 uint8_t booterSignature[20];
1340 uint8_t wiredCryptKey[16];
1341 };
1342 AppleRTCHibernateVars rtcVars;
1343
1344 rtcVars.signature[0] = 'A';
1345 rtcVars.signature[1] = 'A';
1346 rtcVars.signature[2] = 'P';
1347 rtcVars.signature[3] = 'L';
1348 rtcVars.revision = 1;
1349 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
1350 if (gIOHibernateBootSignature[0])
1351 {
1352 char c;
1353 uint8_t value = 0;
1354 for (uint32_t i = 0;
1355 (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
1356 i++)
1357 {
1358 if (c >= 'a')
1359 c -= 'a' - 10;
1360 else if (c >= 'A')
1361 c -= 'A' - 10;
1362 else if (c >= '0')
1363 c -= '0';
1364 else
1365 continue;
1366 value = (value << 4) | c;
1367 if (i & 1)
1368 rtcVars.booterSignature[i >> 1] = value;
1369 }
1370 }
1371 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
1372 if (data)
1373 {
1374 if (!gIOHibernateRTCVariablesKey)
1375 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1376 if (gIOHibernateRTCVariablesKey)
1377 IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
1378
1379 if( gIOOptionsEntry )
1380 {
1381 if( gIOHibernateMode & kIOHibernateModeSwitch )
1382 {
1383 const OSSymbol *sym;
1384 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSwitchVarsKey);
1385 if( sym )
1386 {
1387 gIOOptionsEntry->setProperty(sym, data); /* intentional insecure backup of rtc boot vars */
1388 sym->release();
1389 }
1390 }
1391 }
1392
1393 data->release();
1394 }
1395 if (gIOChosenEntry)
1396 {
1397 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
1398 if (data)
1399 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
1400 {
1401 // set BootNext
1402
1403 if (!gIOHibernateBoot0082Data)
1404 {
1405 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
1406 if (data)
1407 {
1408 // AppleNVRAM_EFI_LOAD_OPTION
1409 struct {
1410 uint32_t Attributes;
1411 uint16_t FilePathLength;
1412 uint16_t Desc;
1413 } loadOptionHeader;
1414 loadOptionHeader.Attributes = 1;
1415 loadOptionHeader.FilePathLength = data->getLength();
1416 loadOptionHeader.Desc = 0;
1417 gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
1418 if (gIOHibernateBoot0082Data)
1419 {
1420 gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
1421 gIOHibernateBoot0082Data->appendBytes(data);
1422 }
1423 }
1424 }
1425 if (!gIOHibernateBoot0082Key)
1426 gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
1427 if (!gIOHibernateBootNextKey)
1428 gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
1429 if (!gIOHibernateBootNextData)
1430 {
1431 uint16_t bits = 0x0082;
1432 gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
1433 }
1434 if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData)
1435 {
1436 gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
1437 gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
1438 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
1439 }
1440 }
1441 }
1442 #else /* !i386 && !x86_64 */
1443 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1444 {
1445 data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1446 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
1447 if (sym && data)
1448 gIOOptionsEntry->setProperty(sym, data);
1449 if (sym)
1450 sym->release();
1451 if (data)
1452 data->release();
1453 if (false && gIOHibernateBootSignature[0])
1454 {
1455 data = OSData::withCapacity(16);
1456 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
1457 if (sym && data)
1458 {
1459 char c;
1460 uint8_t value = 0;
1461 for (uint32_t i = 0; (c = gIOHibernateBootSignature[i]); i++)
1462 {
1463 if (c >= 'a')
1464 c -= 'a' - 10;
1465 else if (c >= 'A')
1466 c -= 'A' - 10;
1467 else if (c >= '0')
1468 c -= '0';
1469 else
1470 continue;
1471 value = (value << 4) | c;
1472 if (i & 1)
1473 data->appendBytes(&value, sizeof(value));
1474 }
1475 gIOOptionsEntry->setProperty(sym, data);
1476 }
1477 if (sym)
1478 sym->release();
1479 if (data)
1480 data->release();
1481 }
1482 }
1483 if (!vars->haveFastBoot)
1484 {
1485 // set boot volume to zero
1486 IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
1487 if (platform && (kIOReturnSuccess == platform->readXPRAM(kXPRamAudioVolume,
1488 &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume))))
1489 {
1490 uint8_t newVolume;
1491 newVolume = vars->saveBootAudioVolume & 0xf8;
1492 platform->writeXPRAM(kXPRamAudioVolume,
1493 &newVolume, sizeof(newVolume));
1494 }
1495 }
1496 #endif /* !i386 && !x86_64 */
1497 }
1498 // --
1499
1500 }
1501 while (false);
1502
1503 IOLockLock(gFSLock);
1504 if ((kIOReturnSuccess == err) && (kFSOpening == gFSState))
1505 {
1506 gFSState = kFSOpened;
1507 gIOHibernateVars = *vars;
1508 gFileVars = *vars->fileVars;
1509 gIOHibernateVars.fileVars = &gFileVars;
1510 gIOHibernateFileRef = gFileVars.fileRef;
1511 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
1512 gIOHibernateState = kIOHibernateStateHibernating;
1513 }
1514 else
1515 {
1516 HIBLOG("hibernate file close due timeout\n");
1517 if (vars->fileVars && vars->fileVars->fileRef) kern_close_file_for_direct_io(vars->fileVars->fileRef, 0, 0, 0, 0, 0);
1518 IOHibernateDone(vars);
1519 gFSState = kFSIdle;
1520 }
1521 IOLockUnlock(gFSLock);
1522
1523 if (vars->fileVars) IODelete(vars->fileVars, IOPolledFileIOVars, 1);
1524 IODelete(vars, IOHibernateVars, 1);
1525
1526 return (err);
1527 }
1528
1529 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1530
1531 DECLARE_IOHIBERNATEPROGRESSALPHA
1532
1533 static void
1534 ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
1535 {
1536 uint32_t rowBytes, pixelShift;
1537 uint32_t x, y;
1538 int32_t blob;
1539 uint32_t alpha, in, color, result;
1540 uint8_t * out;
1541 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1542
1543 rowBytes = display->rowBytes;
1544 pixelShift = display->depth >> 4;
1545 if (pixelShift < 1) return;
1546
1547 screen += ((display->width
1548 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1549 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1550
1551 for (y = 0; y < kIOHibernateProgressHeight; y++)
1552 {
1553 out = screen + y * rowBytes;
1554 for (blob = 0; blob < kIOHibernateProgressCount; blob++)
1555 {
1556 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
1557 for (x = 0; x < kIOHibernateProgressWidth; x++)
1558 {
1559 alpha = gIOHibernateProgressAlpha[y][x];
1560 result = color;
1561 if (alpha)
1562 {
1563 if (0xff != alpha)
1564 {
1565 if (1 == pixelShift)
1566 {
1567 in = *((uint16_t *)out) & 0x1f; // 16
1568 in = (in << 3) | (in >> 2);
1569 }
1570 else
1571 in = *((uint32_t *)out) & 0xff; // 32
1572 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
1573 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
1574 }
1575 if (1 == pixelShift)
1576 {
1577 result >>= 3;
1578 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1579 }
1580 else
1581 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1582 }
1583 out += (1 << pixelShift);
1584 }
1585 out += (kIOHibernateProgressSpacing << pixelShift);
1586 }
1587 }
1588 }
1589
1590
1591 static void
1592 ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
1593 {
1594 uint32_t rowBytes, pixelShift;
1595 uint32_t x, y;
1596 int32_t blob, lastBlob;
1597 uint32_t alpha, in, color, result;
1598 uint8_t * out;
1599 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1600
1601 pixelShift = display->depth >> 4;
1602 if (pixelShift < 1)
1603 return;
1604
1605 rowBytes = display->rowBytes;
1606
1607 screen += ((display->width
1608 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1609 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1610
1611 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
1612
1613 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
1614
1615 for (y = 0; y < kIOHibernateProgressHeight; y++)
1616 {
1617 out = screen + y * rowBytes;
1618 for (blob = firstBlob; blob <= lastBlob; blob++)
1619 {
1620 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
1621 for (x = 0; x < kIOHibernateProgressWidth; x++)
1622 {
1623 alpha = gIOHibernateProgressAlpha[y][x];
1624 result = color;
1625 if (alpha)
1626 {
1627 if (0xff != alpha)
1628 {
1629 in = display->progressSaveUnder[blob][saveindex[blob]++];
1630 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
1631 }
1632 if (1 == pixelShift)
1633 {
1634 result >>= 3;
1635 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1636 }
1637 else
1638 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1639 }
1640 out += (1 << pixelShift);
1641 }
1642 out += (kIOHibernateProgressSpacing << pixelShift);
1643 }
1644 }
1645 }
1646
1647 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1648
1649 IOReturn
1650 IOHibernateIOKitSleep(void)
1651 {
1652 IOReturn ret = kIOReturnSuccess;
1653 IOLockLock(gFSLock);
1654 if (kFSOpening == gFSState)
1655 {
1656 gFSState = kFSTimedOut;
1657 HIBLOG("hibernate file open timed out\n");
1658 ret = kIOReturnTimeout;
1659 }
1660 IOLockUnlock(gFSLock);
1661 return (ret);
1662 }
1663
1664 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1665
1666 IOReturn
1667 IOHibernateSystemHasSlept(void)
1668 {
1669 IOReturn ret = kIOReturnSuccess;
1670 IOHibernateVars * vars = &gIOHibernateVars;
1671 OSObject * obj = 0;
1672 OSData * data;
1673
1674 IOLockLock(gFSLock);
1675 if ((kFSOpened != gFSState) && gIOHibernateMode)
1676 {
1677 ret = kIOReturnTimeout;
1678 }
1679 IOLockUnlock(gFSLock);
1680 if (kIOReturnSuccess != ret) return (ret);
1681
1682 if (gIOHibernateMode) obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
1683 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1684 if (obj && !vars->previewBuffer)
1685 obj->release();
1686
1687 vars->consoleMapping = NULL;
1688 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
1689 {
1690 vars->previewBuffer->release();
1691 vars->previewBuffer = 0;
1692 }
1693
1694 if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1695 && vars->previewBuffer
1696 && (data = OSDynamicCast(OSData,
1697 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
1698 {
1699 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
1700 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
1701
1702 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1703
1704 if (kIOHibernatePreviewUpdates & flags)
1705 {
1706 PE_Video consoleInfo;
1707 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1708
1709 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1710
1711 graphicsInfo->width = consoleInfo.v_width;
1712 graphicsInfo->height = consoleInfo.v_height;
1713 graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
1714 graphicsInfo->depth = consoleInfo.v_depth;
1715 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1716
1717 HIBPRINT("video %p %d %d %d\n",
1718 vars->consoleMapping, graphicsInfo->depth,
1719 graphicsInfo->width, graphicsInfo->height);
1720 if (vars->consoleMapping)
1721 ProgressInit(graphicsInfo, vars->consoleMapping,
1722 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1723 }
1724 }
1725
1726 if (gIOOptionsEntry)
1727 gIOOptionsEntry->sync();
1728
1729 return (ret);
1730 }
1731
1732 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1733
1734 static DeviceTreeNode *
1735 MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry)
1736 {
1737 DeviceTreeNodeProperty * prop;
1738 DeviceTreeNode * child;
1739 IORegistryEntry * childRegEntry;
1740 const char * nameProp;
1741 unsigned int propLen, idx;
1742
1743 prop = (DeviceTreeNodeProperty *) (entry + 1);
1744 for (idx = 0; idx < entry->nProperties; idx++)
1745 {
1746 if (regEntry && (0 != strcmp("name", prop->name)))
1747 {
1748 regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
1749 // HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
1750 }
1751 prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
1752 }
1753
1754 child = (DeviceTreeNode *) prop;
1755 for (idx = 0; idx < entry->nChildren; idx++)
1756 {
1757 if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen))
1758 panic("no name");
1759 childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
1760 // HIBPRINT("%s == %p\n", nameProp, childRegEntry);
1761 child = MergeDeviceTree(child, childRegEntry);
1762 }
1763 return (child);
1764 }
1765
1766 IOReturn
1767 IOHibernateSystemWake(void)
1768 {
1769 if (kFSOpened == gFSState)
1770 {
1771 IOHibernateDone(&gIOHibernateVars);
1772 }
1773 else
1774 {
1775 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1776 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1777 }
1778 return (kIOReturnSuccess);
1779 }
1780
1781 static IOReturn
1782 IOHibernateDone(IOHibernateVars * vars)
1783 {
1784 hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
1785
1786 if (vars->videoMapping)
1787 {
1788 if (vars->videoMapSize)
1789 // remove mappings
1790 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1791 if (vars->videoAllocSize)
1792 // dealloc range
1793 kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
1794 }
1795
1796 if (vars->previewBuffer)
1797 {
1798 vars->previewBuffer->release();
1799 vars->previewBuffer = 0;
1800 }
1801
1802 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1803 {
1804 IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
1805 gIOHibernateCurrentHeader->options, 32);
1806 }
1807 else
1808 {
1809 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1810 }
1811
1812 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1813 && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus))
1814 {
1815 IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
1816 &gIOHibernateGraphicsInfo->gfxStatus,
1817 sizeof(gIOHibernateGraphicsInfo->gfxStatus));
1818 }
1819 else
1820 {
1821 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1822 }
1823
1824
1825 if (vars->fileVars)
1826 {
1827 IOPolledFileClose(vars->fileVars);
1828 }
1829
1830 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1831
1832 #if defined(__i386__) || defined(__x86_64__)
1833 IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
1834 IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
1835
1836 /*
1837 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1838 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1839 */
1840 if (gIOOptionsEntry) {
1841
1842 if (gIOHibernateRTCVariablesKey) {
1843 if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
1844 gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
1845 }
1846 }
1847
1848 if (gIOHibernateBootNextKey)
1849 {
1850 if (gIOHibernateBootNextSave)
1851 {
1852 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
1853 gIOHibernateBootNextSave->release();
1854 gIOHibernateBootNextSave = NULL;
1855 }
1856 else
1857 gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
1858 }
1859 gIOOptionsEntry->sync();
1860 }
1861 #endif
1862
1863 if (vars->srcBuffer)
1864 vars->srcBuffer->release();
1865 if (vars->ioBuffer)
1866 vars->ioBuffer->release();
1867 bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
1868 if (vars->handoffBuffer)
1869 {
1870 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1871 {
1872 IOHibernateHandoff * handoff;
1873 bool done = false;
1874 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
1875 !done;
1876 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
1877 {
1878 HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
1879 uint8_t * data = &handoff->data[0];
1880 switch (handoff->type)
1881 {
1882 case kIOHibernateHandoffTypeEnd:
1883 done = true;
1884 break;
1885
1886 case kIOHibernateHandoffTypeDeviceTree:
1887 MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
1888 break;
1889
1890 case kIOHibernateHandoffTypeKeyStore:
1891 #if defined(__i386__) || defined(__x86_64__)
1892 {
1893 IOBufferMemoryDescriptor *
1894 md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
1895 if (md)
1896 {
1897 IOSetKeyStoreData(md);
1898 }
1899 }
1900 #endif
1901 break;
1902
1903 default:
1904 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
1905 break;
1906 }
1907 }
1908 }
1909 vars->handoffBuffer->release();
1910 }
1911 if (vars->fileExtents)
1912 vars->fileExtents->release();
1913
1914 bzero(vars, sizeof(*vars));
1915
1916 // gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1917
1918 return (kIOReturnSuccess);
1919 }
1920
1921 IOReturn
1922 IOHibernateSystemPostWake(void)
1923 {
1924 struct kern_direct_file_io_ref_t * fileRef;
1925
1926 if (kFSOpened == gFSState)
1927 {
1928 // invalidate & close the image file
1929 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1930 if ((fileRef = gIOHibernateFileRef))
1931 {
1932 gIOHibernateFileRef = 0;
1933 kern_close_file_for_direct_io(fileRef,
1934 0, (caddr_t) gIOHibernateCurrentHeader,
1935 sizeof(IOHibernateImageHeader),
1936 sizeof(IOHibernateImageHeader),
1937 gIOHibernateCurrentHeader->imageSize);
1938 }
1939 gFSState = kFSIdle;
1940 }
1941 return (kIOReturnSuccess);
1942 }
1943
1944 bool IOHibernateWasScreenLocked(void)
1945 {
1946 bool ret = false;
1947 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry)
1948 {
1949 OSData *
1950 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
1951 if (data) switch (*((uint32_t *)data->getBytesNoCopy()))
1952 {
1953 case kIOScreenLockLocked:
1954 case kIOScreenLockFileVaultDialog:
1955 ret = true;
1956 break;
1957 case kIOScreenLockNoLock:
1958 case kIOScreenLockUnlocked:
1959 default:
1960 ret = false;
1961 break;
1962 }
1963 }
1964 return (ret);
1965 }
1966
1967 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1968
1969 SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
1970 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1971 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
1972 SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
1973 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1974 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
1975 SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
1976 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1977 &gIOHibernateMode, 0, "");
1978
1979 void
1980 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1981 {
1982 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
1983 if (data)
1984 {
1985 rootDomain->setProperty(kIOHibernateStateKey, data);
1986 data->release();
1987 }
1988
1989 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))
1990 gIOHibernateMode = kIOHibernateModeOn;
1991 else
1992 gIOHibernateFilename[0] = 0;
1993
1994 sysctl_register_oid(&sysctl__kern_hibernatefile);
1995 sysctl_register_oid(&sysctl__kern_bootsignature);
1996 sysctl_register_oid(&sysctl__kern_hibernatemode);
1997
1998 gFSLock = IOLockAlloc();
1999 }
2000
2001
2002 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2003
2004 static void
2005 hibernate_setup_for_wake(void)
2006 {
2007 }
2008
2009 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2010
2011 #define C_ASSERT(e) typedef char __C_ASSERT__[(e) ? 1 : -1]
2012
2013 static bool
2014 no_encrypt_page(vm_offset_t ppnum)
2015 {
2016 if (pmap_is_noencrypt((ppnum_t)ppnum) == TRUE)
2017 {
2018 return true;
2019 }
2020 return false;
2021 }
2022
2023 uint32_t wired_pages_encrypted = 0;
2024 uint32_t dirty_pages_encrypted = 0;
2025 uint32_t wired_pages_clear = 0;
2026
2027 static void
2028 hibernate_pal_callback(void *vars_arg, vm_offset_t addr)
2029 {
2030 IOHibernateVars *vars = (IOHibernateVars *)vars_arg;
2031 /* Make sure it's not in either of the save lists */
2032 hibernate_set_page_state(vars->page_list, vars->page_list_wired, atop_64(addr), 1, kIOHibernatePageStateFree);
2033
2034 /* Set it in the bitmap of pages owned by the PAL */
2035 hibernate_page_bitset(vars->page_list_pal, TRUE, atop_64(addr));
2036 }
2037
2038 static struct hibernate_cryptvars_t *local_cryptvars;
2039
2040 extern "C" int
2041 hibernate_pal_write(void *buffer, size_t size)
2042 {
2043 IOHibernateVars * vars = &gIOHibernateVars;
2044
2045 IOReturn err = IOPolledFileWrite(vars->fileVars, (const uint8_t *)buffer, size, local_cryptvars);
2046 if (kIOReturnSuccess != err) {
2047 kprintf("epic hibernate fail! %d\n", err);
2048 return err;
2049 }
2050
2051 return 0;
2052 }
2053
2054
2055 extern "C" uint32_t
2056 hibernate_write_image(void)
2057 {
2058 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
2059 IOHibernateVars * vars = &gIOHibernateVars;
2060 IOPolledFileExtent * fileExtents;
2061
2062 C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
2063
2064 uint32_t pageCount, pagesDone;
2065 IOReturn err;
2066 vm_offset_t ppnum, page;
2067 IOItemCount count;
2068 uint8_t * src;
2069 uint8_t * data;
2070 IOByteCount pageCompressedSize;
2071 uint64_t compressedSize, uncompressedSize;
2072 uint64_t image1Size = 0;
2073 uint32_t bitmap_size;
2074 bool iterDone, pollerOpen, needEncrypt;
2075 uint32_t restore1Sum, sum, sum1, sum2;
2076 uint32_t tag;
2077 uint32_t pageType;
2078 uint32_t pageAndCount[2];
2079 addr64_t phys64;
2080 IOByteCount segLen;
2081
2082 AbsoluteTime startTime, endTime;
2083 AbsoluteTime allTime, compTime;
2084 uint64_t compBytes;
2085 uint64_t nsec;
2086 uint32_t lastProgressStamp = 0;
2087 uint32_t progressStamp;
2088 uint32_t blob, lastBlob = (uint32_t) -1L;
2089
2090 hibernate_cryptvars_t _cryptvars;
2091 hibernate_cryptvars_t * cryptvars = 0;
2092
2093 wired_pages_encrypted = 0;
2094 dirty_pages_encrypted = 0;
2095 wired_pages_clear = 0;
2096
2097 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
2098 return (false /* sleep */ );
2099
2100 if (kIOHibernateModeSleep & gIOHibernateMode)
2101 kdebug_enable = save_kdebug_enable;
2102
2103 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START, 0, 0, 0, 0, 0);
2104 IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
2105
2106 restore1Sum = sum1 = sum2 = 0;
2107
2108 hibernate_pal_prepare();
2109
2110 #if CRYPTO
2111 // encryption data. "iv" is the "initial vector".
2112 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2113 {
2114 static const unsigned char first_iv[AES_BLOCK_SIZE]
2115 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
2116 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
2117
2118 cryptvars = &gIOHibernateCryptWakeContext;
2119 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
2120 aes_encrypt_key(vars->cryptKey,
2121 kIOHibernateAESKeySize,
2122 &cryptvars->ctx.encrypt);
2123 aes_decrypt_key(vars->cryptKey,
2124 kIOHibernateAESKeySize,
2125 &cryptvars->ctx.decrypt);
2126
2127 cryptvars = &_cryptvars;
2128 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
2129 for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++)
2130 vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
2131 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
2132 aes_encrypt_key(vars->wiredCryptKey,
2133 kIOHibernateAESKeySize,
2134 &cryptvars->ctx.encrypt);
2135
2136 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
2137 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
2138 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
2139
2140 local_cryptvars = cryptvars;
2141 }
2142 #endif /* CRYPTO */
2143
2144 hibernate_setup_for_wake();
2145
2146 hibernate_page_list_setall(vars->page_list,
2147 vars->page_list_wired,
2148 vars->page_list_pal,
2149 false /* !preflight */,
2150 &pageCount);
2151
2152 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
2153
2154 fileExtents = (IOPolledFileExtent *) vars->fileExtents->getBytesNoCopy();
2155
2156 #if 0
2157 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
2158 for (page = 0; page < count; page++)
2159 {
2160 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
2161 fileExtents[page].start, fileExtents[page].length,
2162 fileExtents[page].start + fileExtents[page].length);
2163 }
2164 #endif
2165
2166 needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
2167 AbsoluteTime_to_scalar(&compTime) = 0;
2168 compBytes = 0;
2169
2170 clock_get_uptime(&allTime);
2171 IOService::getPMRootDomain()->pmStatsRecordEvent(
2172 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
2173
2174 do
2175 {
2176 compressedSize = 0;
2177 uncompressedSize = 0;
2178
2179 IOPolledFileSeek(vars->fileVars, sizeof(IOHibernateImageHeader));
2180
2181 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
2182 ml_get_interrupts_enabled());
2183 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledBeforeSleepState, vars->ioBuffer);
2184 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
2185 pollerOpen = (kIOReturnSuccess == err);
2186 if (!pollerOpen)
2187 break;
2188
2189 // copy file block extent list if larger than header
2190
2191 count = vars->fileExtents->getLength();
2192 if (count > sizeof(header->fileExtentMap))
2193 {
2194 count -= sizeof(header->fileExtentMap);
2195 err = IOPolledFileWrite(vars->fileVars,
2196 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
2197 if (kIOReturnSuccess != err)
2198 break;
2199 }
2200
2201 uintptr_t hibernateBase;
2202 uintptr_t hibernateEnd;
2203
2204 hibernateBase = HIB_BASE; /* Defined in PAL headers */
2205
2206 hibernateEnd = (segHIBB + segSizeHIB);
2207
2208 // copy out restore1 code
2209
2210 for (count = 0;
2211 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2212 count += segLen)
2213 {
2214 for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++)
2215 {
2216 gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
2217 }
2218 }
2219
2220 page = atop_32(kvtophys(hibernateBase));
2221 count = atop_32(round_page(hibernateEnd) - hibernateBase);
2222 header->restore1CodePhysPage = page;
2223 header->restore1CodeVirt = hibernateBase;
2224 header->restore1PageCount = count;
2225 header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
2226 header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
2227
2228 // sum __HIB seg, with zeros for the stack
2229 src = (uint8_t *) trunc_page(hibernateBase);
2230 for (page = 0; page < count; page++)
2231 {
2232 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
2233 restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
2234 else
2235 restore1Sum += 0x00000000;
2236 src += page_size;
2237 }
2238 sum1 = restore1Sum;
2239
2240 // write the __HIB seg, with zeros for the stack
2241
2242 src = (uint8_t *) trunc_page(hibernateBase);
2243 count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
2244 if (count)
2245 {
2246 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
2247 if (kIOReturnSuccess != err)
2248 break;
2249 }
2250 err = IOPolledFileWrite(vars->fileVars,
2251 (uint8_t *) 0,
2252 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
2253 cryptvars);
2254 if (kIOReturnSuccess != err)
2255 break;
2256 src = &gIOHibernateRestoreStackEnd[0];
2257 count = round_page(hibernateEnd) - ((uintptr_t) src);
2258 if (count)
2259 {
2260 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
2261 if (kIOReturnSuccess != err)
2262 break;
2263 }
2264
2265 vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
2266 vars->fileVars->encryptEnd = UINT64_MAX;
2267 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
2268
2269 // write the preview buffer
2270
2271 if (vars->previewBuffer)
2272 {
2273 ppnum = 0;
2274 count = 0;
2275 do
2276 {
2277 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
2278 pageAndCount[0] = atop_64(phys64);
2279 pageAndCount[1] = atop_32(segLen);
2280 err = IOPolledFileWrite(vars->fileVars,
2281 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
2282 cryptvars);
2283 if (kIOReturnSuccess != err)
2284 break;
2285 count += segLen;
2286 ppnum += sizeof(pageAndCount);
2287 }
2288 while (phys64);
2289 if (kIOReturnSuccess != err)
2290 break;
2291
2292 src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
2293
2294 ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
2295
2296 count = vars->previewBuffer->getLength();
2297
2298 header->previewPageListSize = ppnum;
2299 header->previewSize = count + ppnum;
2300
2301 for (page = 0; page < count; page += page_size)
2302 {
2303 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
2304 sum1 += hibernate_sum_page(src + page, atop_64(phys64));
2305 }
2306 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
2307 if (kIOReturnSuccess != err)
2308 break;
2309 }
2310
2311 // mark areas for no save
2312
2313 for (count = 0;
2314 (phys64 = vars->ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2315 count += segLen)
2316 {
2317 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2318 atop_64(phys64), atop_32(segLen),
2319 kIOHibernatePageStateFree);
2320 pageCount -= atop_32(segLen);
2321 }
2322
2323 for (count = 0;
2324 (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2325 count += segLen)
2326 {
2327 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2328 atop_64(phys64), atop_32(segLen),
2329 kIOHibernatePageStateFree);
2330 pageCount -= atop_32(segLen);
2331 }
2332
2333 // copy out bitmap of pages available for trashing during restore
2334
2335 bitmap_size = vars->page_list_wired->list_size;
2336 src = (uint8_t *) vars->page_list_wired;
2337 err = IOPolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
2338 if (kIOReturnSuccess != err)
2339 break;
2340
2341 // mark more areas for no save, but these are not available
2342 // for trashing during restore
2343
2344 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
2345
2346
2347 page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
2348 count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
2349 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2350 page, count,
2351 kIOHibernatePageStateFree);
2352 pageCount -= count;
2353
2354 if (vars->previewBuffer) for (count = 0;
2355 (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2356 count += segLen)
2357 {
2358 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2359 atop_64(phys64), atop_32(segLen),
2360 kIOHibernatePageStateFree);
2361 pageCount -= atop_32(segLen);
2362 }
2363
2364 for (count = 0;
2365 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2366 count += segLen)
2367 {
2368 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2369 atop_64(phys64), atop_32(segLen),
2370 kIOHibernatePageStateFree);
2371 pageCount -= atop_32(segLen);
2372 }
2373
2374 (void)hibernate_pal_callback;
2375
2376 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2377
2378 pagesDone = 0;
2379 lastBlob = 0;
2380
2381 HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
2382 bitmap_size, header->previewSize,
2383 pageCount, vars->fileVars->position);
2384
2385 enum
2386 // pageType
2387 {
2388 kWired = 0x02,
2389 kEncrypt = 0x01,
2390 kWiredEncrypt = kWired | kEncrypt,
2391 kWiredClear = kWired,
2392 kUnwiredEncrypt = kEncrypt
2393 };
2394
2395 for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)
2396 {
2397 if (kUnwiredEncrypt == pageType)
2398 {
2399 // start unwired image
2400 vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
2401 vars->fileVars->encryptEnd = UINT64_MAX;
2402 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
2403 bcopy(&cryptvars->aes_iv[0],
2404 &gIOHibernateCryptWakeContext.aes_iv[0],
2405 sizeof(cryptvars->aes_iv));
2406 cryptvars = &gIOHibernateCryptWakeContext;
2407 }
2408 for (iterDone = false, ppnum = 0; !iterDone; )
2409 {
2410 count = hibernate_page_list_iterate((kWired & pageType)
2411 ? vars->page_list_wired : vars->page_list,
2412 &ppnum);
2413 // kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
2414 iterDone = !count;
2415
2416 if (count && (kWired & pageType) && needEncrypt)
2417 {
2418 uint32_t checkIndex;
2419 for (checkIndex = 0;
2420 (checkIndex < count)
2421 && (((kEncrypt & pageType) == 0) == no_encrypt_page(ppnum + checkIndex));
2422 checkIndex++)
2423 {}
2424 if (!checkIndex)
2425 {
2426 ppnum++;
2427 continue;
2428 }
2429 count = checkIndex;
2430 }
2431
2432 switch (pageType)
2433 {
2434 case kWiredEncrypt: wired_pages_encrypted += count; break;
2435 case kWiredClear: wired_pages_clear += count; break;
2436 case kUnwiredEncrypt: dirty_pages_encrypted += count; break;
2437 }
2438
2439 if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */}
2440 else
2441 {
2442 pageAndCount[0] = ppnum;
2443 pageAndCount[1] = count;
2444 err = IOPolledFileWrite(vars->fileVars,
2445 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
2446 cryptvars);
2447 if (kIOReturnSuccess != err)
2448 break;
2449 }
2450
2451 for (page = ppnum; page < (ppnum + count); page++)
2452 {
2453 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
2454 if (err)
2455 {
2456 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
2457 break;
2458 }
2459
2460 sum = hibernate_sum_page(src, page);
2461 if (kWired & pageType)
2462 sum1 += sum;
2463 else
2464 sum2 += sum;
2465
2466 clock_get_uptime(&startTime);
2467
2468 pageCompressedSize = WKdm_compress ((WK_word*) src, (WK_word*) (src + page_size), PAGE_SIZE_IN_WORDS);
2469
2470 clock_get_uptime(&endTime);
2471 ADD_ABSOLUTETIME(&compTime, &endTime);
2472 SUB_ABSOLUTETIME(&compTime, &startTime);
2473 compBytes += page_size;
2474
2475 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2476 pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
2477
2478 if (pageCompressedSize > page_size)
2479 {
2480 // HIBLOG("------------lose: %d\n", pageCompressedSize);
2481 pageCompressedSize = page_size;
2482 }
2483
2484 if (pageCompressedSize != page_size)
2485 data = (src + page_size);
2486 else
2487 data = src;
2488
2489 tag = pageCompressedSize | kIOHibernateTagSignature;
2490 err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
2491 if (kIOReturnSuccess != err)
2492 break;
2493
2494 err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
2495 if (kIOReturnSuccess != err)
2496 break;
2497
2498 compressedSize += pageCompressedSize;
2499 if (pageCompressedSize)
2500 uncompressedSize += page_size;
2501 pagesDone++;
2502
2503 if (vars->consoleMapping && (0 == (1023 & pagesDone)))
2504 {
2505 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
2506 if (blob != lastBlob)
2507 {
2508 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
2509 lastBlob = blob;
2510 }
2511 }
2512 if (0 == (8191 & pagesDone))
2513 {
2514 clock_get_uptime(&endTime);
2515 SUB_ABSOLUTETIME(&endTime, &allTime);
2516 absolutetime_to_nanoseconds(endTime, &nsec);
2517 progressStamp = nsec / 750000000ULL;
2518 if (progressStamp != lastProgressStamp)
2519 {
2520 lastProgressStamp = progressStamp;
2521 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
2522 }
2523 }
2524 }
2525 if (kIOReturnSuccess != err)
2526 break;
2527 ppnum = page;
2528 }
2529
2530 if (kIOReturnSuccess != err)
2531 break;
2532
2533 if ((kEncrypt & pageType))
2534 {
2535 vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
2536 HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
2537 }
2538
2539 if (kWiredEncrypt != pageType)
2540 {
2541 // end of image1/2 - fill to next block
2542 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2543 if (kIOReturnSuccess != err)
2544 break;
2545 }
2546 if (kWiredClear == pageType)
2547 {
2548 // enlarge wired image for test
2549 // err = IOPolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
2550
2551 // end wired image
2552 header->encryptStart = vars->fileVars->encryptStart;
2553 header->encryptEnd = vars->fileVars->encryptEnd;
2554 image1Size = vars->fileVars->position;
2555 HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
2556 image1Size, header->encryptStart, header->encryptEnd);
2557 }
2558 }
2559 if (kIOReturnSuccess != err)
2560 break;
2561
2562 // Header:
2563
2564 header->imageSize = vars->fileVars->position;
2565 header->image1Size = image1Size;
2566 header->bitmapSize = bitmap_size;
2567 header->pageCount = pageCount;
2568
2569 header->restore1Sum = restore1Sum;
2570 header->image1Sum = sum1;
2571 header->image2Sum = sum2;
2572 header->sleepTime = gIOLastSleepTime.tv_sec;
2573
2574 header->compression = (compressedSize << 8) / uncompressedSize;
2575 gIOHibernateCompression = header->compression;
2576
2577 count = vars->fileExtents->getLength();
2578 if (count > sizeof(header->fileExtentMap))
2579 {
2580 header->fileExtentMapSize = count;
2581 count = sizeof(header->fileExtentMap);
2582 }
2583 else
2584 header->fileExtentMapSize = sizeof(header->fileExtentMap);
2585 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
2586
2587 header->deviceBase = vars->fileVars->block0;
2588
2589 IOPolledFileSeek(vars->fileVars, 0);
2590 err = IOPolledFileWrite(vars->fileVars,
2591 (uint8_t *) header, sizeof(IOHibernateImageHeader),
2592 cryptvars);
2593 if (kIOReturnSuccess != err)
2594 break;
2595 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2596 if (kIOReturnSuccess != err)
2597 break;
2598 err = IOHibernatePollerIODone(vars->fileVars, true);
2599 if (kIOReturnSuccess != err)
2600 break;
2601 }
2602 while (false);
2603
2604 clock_get_uptime(&endTime);
2605
2606 IOService::getPMRootDomain()->pmStatsRecordEvent(
2607 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
2608
2609 SUB_ABSOLUTETIME(&endTime, &allTime);
2610 absolutetime_to_nanoseconds(endTime, &nsec);
2611 HIBLOG("all time: %qd ms, ",
2612 nsec / 1000000ULL);
2613
2614 absolutetime_to_nanoseconds(compTime, &nsec);
2615 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2616 compBytes,
2617 nsec / 1000000ULL,
2618 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2619
2620 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2621 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
2622 vars->fileVars->cryptBytes,
2623 nsec / 1000000ULL,
2624 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2625
2626 HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
2627 header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
2628 uncompressedSize, atop_32(uncompressedSize), compressedSize,
2629 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
2630 sum1, sum2);
2631
2632 HIBLOG("wired_pages_encrypted %d, wired_pages_clear %d, dirty_pages_encrypted %d\n",
2633 wired_pages_encrypted, wired_pages_clear, dirty_pages_encrypted);
2634
2635 if (vars->fileVars->io)
2636 (void) IOHibernatePollerIODone(vars->fileVars, false);
2637
2638 if (pollerOpen)
2639 IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
2640
2641 if (vars->consoleMapping)
2642 ProgressUpdate(gIOHibernateGraphicsInfo,
2643 vars->consoleMapping, 0, kIOHibernateProgressCount);
2644
2645 HIBLOG("hibernate_write_image done(%x)\n", err);
2646
2647 // should we come back via regular wake, set the state in memory.
2648 gIOHibernateState = kIOHibernateStateInactive;
2649
2650 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END,
2651 wired_pages_encrypted, wired_pages_clear, dirty_pages_encrypted, 0, 0);
2652
2653 if (kIOReturnSuccess == err)
2654 {
2655 if (kIOHibernateModeSleep & gIOHibernateMode)
2656 {
2657 return (kIOHibernatePostWriteSleep);
2658 }
2659 else if(kIOHibernateModeRestart & gIOHibernateMode)
2660 {
2661 return (kIOHibernatePostWriteRestart);
2662 }
2663 else
2664 {
2665 /* by default, power down */
2666 return (kIOHibernatePostWriteHalt);
2667 }
2668 }
2669 else if (kIOReturnAborted == err)
2670 {
2671 return (kIOHibernatePostWriteWake);
2672 }
2673 else
2674 {
2675 /* on error, sleep */
2676 return (kIOHibernatePostWriteSleep);
2677 }
2678 }
2679
2680 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2681
2682 extern "C" void
2683 hibernate_machine_init(void)
2684 {
2685 IOReturn err;
2686 uint32_t sum;
2687 uint32_t pagesDone;
2688 uint32_t pagesRead = 0;
2689 AbsoluteTime startTime, compTime;
2690 AbsoluteTime allTime, endTime;
2691 uint64_t compBytes;
2692 uint64_t nsec;
2693 uint32_t lastProgressStamp = 0;
2694 uint32_t progressStamp;
2695 hibernate_cryptvars_t * cryptvars = 0;
2696
2697 IOHibernateVars * vars = &gIOHibernateVars;
2698
2699 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
2700 return;
2701
2702 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2703 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2704
2705 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, image1Size %qx, conflictCount %d, nextFree %x\n",
2706 gIOHibernateState, pagesDone, sum, gIOHibernateCurrentHeader->image1Size,
2707 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
2708
2709 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
2710 {
2711 HIBLOG("regular wake\n");
2712 return;
2713 }
2714
2715 HIBPRINT("diag %x %x %x %x\n",
2716 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
2717 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
2718
2719 if ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)
2720 hibernate_page_list_discard(vars->page_list);
2721
2722 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
2723
2724 if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount)
2725 panic("handoff overflow");
2726
2727 IOHibernateHandoff * handoff;
2728 bool done = false;
2729 bool foundCryptData = false;
2730
2731 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
2732 !done;
2733 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
2734 {
2735 // HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
2736 uint8_t * data = &handoff->data[0];
2737 switch (handoff->type)
2738 {
2739 case kIOHibernateHandoffTypeEnd:
2740 done = true;
2741 break;
2742
2743 case kIOHibernateHandoffTypeGraphicsInfo:
2744 bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
2745 break;
2746
2747 case kIOHibernateHandoffTypeCryptVars:
2748 if (cryptvars)
2749 {
2750 hibernate_cryptwakevars_t *
2751 wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
2752 bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
2753 }
2754 foundCryptData = true;
2755 bzero(data, handoff->bytecount);
2756 break;
2757
2758 case kIOHibernateHandoffTypeMemoryMap:
2759 hibernate_newruntime_map(data, handoff->bytecount,
2760 gIOHibernateCurrentHeader->systemTableOffset);
2761 break;
2762
2763 case kIOHibernateHandoffTypeDeviceTree:
2764 {
2765 // DTEntry chosen = NULL;
2766 // HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen));
2767 }
2768 break;
2769
2770 default:
2771 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
2772 break;
2773 }
2774 }
2775 if (cryptvars && !foundCryptData)
2776 panic("hibernate handoff");
2777
2778 HIBPRINT("video %x %d %d %d status %x\n",
2779 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2780 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
2781
2782 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)
2783 {
2784 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2785 * gIOHibernateGraphicsInfo->rowBytes);
2786 IOMapPages(kernel_map,
2787 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2788 vars->videoMapSize, kIOMapInhibitCache );
2789 }
2790
2791 if (vars->videoMapSize)
2792 ProgressUpdate(gIOHibernateGraphicsInfo,
2793 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2794
2795 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2796 uint32_t decoOffset;
2797
2798 clock_get_uptime(&allTime);
2799 AbsoluteTime_to_scalar(&compTime) = 0;
2800 compBytes = 0;
2801
2802 HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2803 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0);
2804 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
2805
2806 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2807
2808 // kick off the read ahead
2809 vars->fileVars->io = false;
2810 vars->fileVars->bufferHalf = 0;
2811 vars->fileVars->bufferLimit = 0;
2812 vars->fileVars->lastRead = 0;
2813 vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
2814 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2815 vars->fileVars->cryptBytes = 0;
2816 AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
2817
2818 err = IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
2819 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2820 // --
2821
2822 HIBLOG("hibernate_machine_init reading\n");
2823
2824 uint32_t * header = (uint32_t *) src;
2825 sum = 0;
2826
2827 while (kIOReturnSuccess == err)
2828 {
2829 unsigned int count;
2830 unsigned int page;
2831 uint32_t tag;
2832 vm_offset_t ppnum, compressedSize;
2833
2834 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2835 if (kIOReturnSuccess != err)
2836 break;
2837
2838 ppnum = header[0];
2839 count = header[1];
2840
2841 // HIBPRINT("(%x, %x)\n", ppnum, count);
2842
2843 if (!count)
2844 break;
2845
2846 for (page = 0; page < count; page++)
2847 {
2848 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2849 if (kIOReturnSuccess != err)
2850 break;
2851
2852 compressedSize = kIOHibernateTagLength & tag;
2853 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength))
2854 {
2855 err = kIOReturnIPCError;
2856 break;
2857 }
2858
2859 if (!compressedSize)
2860 {
2861 ppnum++;
2862 pagesDone++;
2863 continue;
2864 }
2865
2866 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2867 if (kIOReturnSuccess != err)
2868 break;
2869
2870 if (compressedSize < page_size)
2871 {
2872 decoOffset = page_size;
2873
2874 clock_get_uptime(&startTime);
2875 WKdm_decompress((WK_word*) src, (WK_word*) (src + decoOffset), PAGE_SIZE_IN_WORDS);
2876 clock_get_uptime(&endTime);
2877 ADD_ABSOLUTETIME(&compTime, &endTime);
2878 SUB_ABSOLUTETIME(&compTime, &startTime);
2879
2880 compBytes += page_size;
2881 }
2882 else
2883 decoOffset = 0;
2884
2885 sum += hibernate_sum_page((src + decoOffset), ppnum);
2886
2887 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2888 if (err)
2889 {
2890 HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
2891 break;
2892 }
2893
2894 ppnum++;
2895 pagesDone++;
2896 pagesRead++;
2897
2898 if (0 == (8191 & pagesDone))
2899 {
2900 clock_get_uptime(&endTime);
2901 SUB_ABSOLUTETIME(&endTime, &allTime);
2902 absolutetime_to_nanoseconds(endTime, &nsec);
2903 progressStamp = nsec / 750000000ULL;
2904 if (progressStamp != lastProgressStamp)
2905 {
2906 lastProgressStamp = progressStamp;
2907 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2908 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2909 }
2910 }
2911 }
2912 }
2913 if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages))
2914 err = kIOReturnLockedRead;
2915
2916 if (kIOReturnSuccess != err)
2917 panic("Hibernate restore error %x", err);
2918
2919 gIOHibernateCurrentHeader->actualImage2Sum = sum;
2920 gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
2921
2922 if (vars->fileVars->io)
2923 (void) IOHibernatePollerIODone(vars->fileVars, false);
2924
2925 err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
2926
2927 clock_get_uptime(&endTime);
2928
2929 IOService::getPMRootDomain()->pmStatsRecordEvent(
2930 kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
2931 IOService::getPMRootDomain()->pmStatsRecordEvent(
2932 kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
2933
2934 SUB_ABSOLUTETIME(&endTime, &allTime);
2935 absolutetime_to_nanoseconds(endTime, &nsec);
2936
2937 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %qd ms, ",
2938 pagesDone, sum, nsec / 1000000ULL);
2939
2940 absolutetime_to_nanoseconds(compTime, &nsec);
2941 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2942 compBytes,
2943 nsec / 1000000ULL,
2944 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2945
2946 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2947 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
2948 vars->fileVars->cryptBytes,
2949 nsec / 1000000ULL,
2950 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2951
2952 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_NONE, pagesRead, pagesDone, 0, 0, 0);
2953 }
2954
2955 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2956
2957 void IOHibernateSystemRestart(void)
2958 {
2959 static uint8_t noteStore[32] __attribute__((aligned(32)));
2960 IORegistryEntry * regEntry;
2961 const OSSymbol * sym;
2962 OSData * noteProp;
2963 OSData * data;
2964 uintptr_t * smcVars;
2965 uint8_t * smcBytes;
2966 size_t len;
2967 addr64_t element;
2968
2969 data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
2970 if (!data) return;
2971
2972 smcVars = (typeof(smcVars)) data->getBytesNoCopy();
2973 smcBytes = (typeof(smcBytes)) smcVars[1];
2974 len = smcVars[0];
2975 if (len > sizeof(noteStore)) len = sizeof(noteStore);
2976 noteProp = OSData::withCapacity(3 * sizeof(element));
2977 if (!noteProp) return;
2978 element = len;
2979 noteProp->appendBytes(&element, sizeof(element));
2980 element = crc32(0, smcBytes, len);
2981 noteProp->appendBytes(&element, sizeof(element));
2982
2983 bcopy(smcBytes, noteStore, len);
2984 element = (addr64_t) &noteStore[0];
2985 element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
2986 noteProp->appendBytes(&element, sizeof(element));
2987
2988 if (!gIOOptionsEntry)
2989 {
2990 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
2991 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
2992 if (regEntry && !gIOOptionsEntry)
2993 regEntry->release();
2994 }
2995
2996 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
2997 if (gIOOptionsEntry && sym) gIOOptionsEntry->setProperty(sym, noteProp);
2998 if (noteProp) noteProp->release();
2999 if (sym) sym->release();
3000 }
3001
3002
3003
mirror of XNU