]> git.saurik.com Git - apple/xnu.git/blob - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-2050.9.2.tar.gz
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
1 /*
2 * Copyright (c) 2004-2008 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29
30 /*
31
32 Sleep:
33
34 - PMRootDomain calls IOHibernateSystemSleep() before system sleep
35 (devices awake, normal execution context)
36 - IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
37 grabs its extents and searches for a polling driver willing to work with that IOMedia.
38 The BSD code makes an ioctl to the storage driver to get the partition base offset to
39 the disk, and other ioctls to get the transfer constraints
40 If successful, the file is written to make sure its initially not bootable (in case of
41 later failure) and nvram set to point to the first block of the file. (Has to be done
42 here so blocking is possible in nvram support).
43 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
44 page out any pages it wants to (currently zero, but probably some percentage of memory).
45 Its assumed just allocating pages will cause the VM system to naturally select the best
46 pages for eviction. It also copies processor flags needed for the restore path and sets
47 a flag in the boot processor proc info.
48 gIOHibernateState = kIOHibernateStateHibernating.
49 - Regular sleep progresses - some drivers may inspect the root domain property
50 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
51 as usual but leaves motherboard I/O on.
52 - Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
53 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
54 all ppc RC bits out of the hash table and caches into the mapping structures.
55 - hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
56 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
57 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
58 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
59 The image header and block list are written. The header includes the second file extent so
60 only the header block is needed to read the file, regardless of filesystem.
61 The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
62 (only) is used to decompress the image during wake/boot.
63 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
64 The bitmaps are written to the image.
65 More areas are removed from the bitmaps (after they have been written to the image) - the
66 segment "__HIB" pages and interrupt stack.
67 Each wired page is compressed and written and then each non-wired page. Compression and
68 disk writes are in parallel.
69 The image header is written to the start of the file and the polling driver closed.
70 The machine powers down (or sleeps).
71
72 Boot/Wake:
73
74 - BootX sees the boot-image nvram variable containing the device and block number of the image,
75 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
76 - BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
77 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
78 that is in the kernel's __HIB section.
79 - BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
80 hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
81 only code & data in that section is safe to call since all the other wired pages are still
82 compressed in the image.
83 - hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
84 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
85 location directly, and copies those that can't to interim free pages. When the image has been
86 completed, the copies are uncompressed, overwriting the wired image pages.
87 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
88 is used to get pages into place for 64bit.
89 - the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
90 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
91 are removed from the software strutures, and the hash table is reinitialized.
92 - After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
93 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
94 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
95 for the remaining non wired pages.
96 - The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
97 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
98 is closed via bsd.
99
100 Polled Mode I/O:
101
102 IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
103 registry, specifying an object of calls IOPolledInterface.
104
105 Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
106 partition) that the image is going to live, looking for polled interface properties. If it finds
107 one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
108 interfaces found are kept in an ordered list.
109
110 There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
111 few different contexts things happen in:
112
113 - there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
114 up and running) and after wake - this is safe to allocate memory and do anything. The device
115 ignores sleep requests from that point since its a waste of time if it goes to sleep and
116 immediately wakes back up for the image write.
117
118 - there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
119 immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
120 from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
121 used to flush and set the disk to sleep.
122
123 - there is an Open/Close (AfterSleep) pair made around the image read operations that happen
124 immediately after sleep. These can't block or allocate memory. This is happening after the platform
125 expert has woken the low level bits of the system, but most of the I/O system has not. There is only
126 one thread running.
127
128 For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
129 (prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
130 the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
131 Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
132 that is called for the hardware to check for events, and complete the I/O via the callback.
133 The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
134 to restrict I/O ops.
135 */
136
137 #include <sys/systm.h>
138
139 #include <IOKit/IOWorkLoop.h>
140 #include <IOKit/IOCommandGate.h>
141 #include <IOKit/IOTimerEventSource.h>
142 #include <IOKit/IOPlatformExpert.h>
143 #include <IOKit/IOKitDebug.h>
144 #include <IOKit/IOTimeStamp.h>
145 #include <IOKit/pwr_mgt/RootDomain.h>
146 #include <IOKit/pwr_mgt/IOPMPrivate.h>
147 #include <IOKit/IOMessage.h>
148 #include <IOKit/IODeviceTreeSupport.h>
149 #include <IOKit/IOBSD.h>
150 #include "RootDomainUserClient.h"
151 #include <IOKit/pwr_mgt/IOPowerConnection.h>
152 #include "IOPMPowerStateQueue.h"
153 #include <IOKit/IOBufferMemoryDescriptor.h>
154 #include <IOKit/AppleKeyStoreInterface.h>
155 #include <libkern/crypto/aes.h>
156
157 #include <sys/uio.h>
158 #include <sys/conf.h>
159 #include <sys/stat.h>
160 #include <sys/fcntl.h> // (FWRITE, ...)
161 #include <sys/sysctl.h>
162 #include <sys/kdebug.h>
163
164 #include <IOKit/IOHibernatePrivate.h>
165 #include <IOKit/IOPolledInterface.h>
166 #include <IOKit/IONVRAM.h>
167 #include "IOHibernateInternal.h"
168 #include <libkern/WKdm.h>
169 #include "IOKitKernelInternal.h"
170 #include <pexpert/device_tree.h>
171
172 #include <machine/pal_routines.h>
173 #include <machine/pal_hibernate.h>
174
175 extern "C" addr64_t kvtophys(vm_offset_t va);
176
177 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
178
179 extern unsigned int save_kdebug_enable;
180 extern uint32_t gIOHibernateState;
181 uint32_t gIOHibernateMode;
182 static char gIOHibernateBootSignature[256+1];
183 static char gIOHibernateFilename[MAXPATHLEN+1];
184 static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
185 uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
186
187 static IODTNVRAM * gIOOptionsEntry;
188 static IORegistryEntry * gIOChosenEntry;
189 #if defined(__i386__) || defined(__x86_64__)
190 static const OSSymbol * gIOCreateEFIDevicePathSymbol;
191 static const OSSymbol * gIOHibernateRTCVariablesKey;
192 static const OSSymbol * gIOHibernateBoot0082Key;
193 static const OSSymbol * gIOHibernateBootNextKey;
194 static OSData * gIOHibernateBoot0082Data;
195 static OSData * gIOHibernateBootNextData;
196 static OSObject * gIOHibernateBootNextSave;
197 #endif
198
199 static IOLock * gFSLock;
200 static uint32_t gFSState;
201 static IOPolledFileIOVars gFileVars;
202 static IOHibernateVars gIOHibernateVars;
203 static struct kern_direct_file_io_ref_t * gIOHibernateFileRef;
204 static hibernate_cryptvars_t gIOHibernateCryptWakeContext;
205 static hibernate_graphics_t _hibernateGraphics;
206 static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
207
208 enum
209 {
210 kFSIdle = 0,
211 kFSOpening = 2,
212 kFSOpened = 3,
213 kFSTimedOut = 4,
214 };
215
216 static IOReturn IOHibernateDone(IOHibernateVars * vars);
217
218 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
219
220 enum { kXPRamAudioVolume = 8 };
221 enum { kDefaultIOSize = 128 * 1024 };
222 enum { kVideoMapSize = 32 * 1024 * 1024 };
223
224 #ifndef kIOMediaPreferredBlockSizeKey
225 #define kIOMediaPreferredBlockSizeKey "Preferred Block Size"
226 #endif
227
228 #ifndef kIOBootPathKey
229 #define kIOBootPathKey "bootpath"
230 #endif
231 #ifndef kIOSelectedBootDeviceKey
232 #define kIOSelectedBootDeviceKey "boot-device"
233 #endif
234
235
236 enum { kIOHibernateMinPollersNeeded = 2 };
237
238 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
239
240 // copy from phys addr to MD
241
242 static IOReturn
243 IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
244 IOByteCount offset, addr64_t bytes, IOByteCount length)
245 {
246 addr64_t srcAddr = bytes;
247 IOByteCount remaining;
248
249 remaining = length = min(length, md->getLength() - offset);
250 while (remaining) { // (process another target segment?)
251 addr64_t dstAddr64;
252 IOByteCount dstLen;
253
254 dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
255 if (!dstAddr64)
256 break;
257
258 // Clip segment length to remaining
259 if (dstLen > remaining)
260 dstLen = remaining;
261
262 #if 1
263 bcopy_phys(srcAddr, dstAddr64, dstLen);
264 #else
265 copypv(srcAddr, dstAddr64, dstLen,
266 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
267 #endif
268 srcAddr += dstLen;
269 offset += dstLen;
270 remaining -= dstLen;
271 }
272
273 assert(!remaining);
274
275 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
276 }
277
278 // copy from MD to phys addr
279
280 static IOReturn
281 IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
282 IOByteCount offset, addr64_t bytes, IOByteCount length)
283 {
284 addr64_t dstAddr = bytes;
285 IOByteCount remaining;
286
287 remaining = length = min(length, md->getLength() - offset);
288 while (remaining) { // (process another target segment?)
289 addr64_t srcAddr64;
290 IOByteCount dstLen;
291
292 srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
293 if (!srcAddr64)
294 break;
295
296 // Clip segment length to remaining
297 if (dstLen > remaining)
298 dstLen = remaining;
299
300 #if 1
301 bcopy_phys(srcAddr64, dstAddr, dstLen);
302 #else
303 copypv(srcAddr, dstAddr64, dstLen,
304 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
305 #endif
306 dstAddr += dstLen;
307 offset += dstLen;
308 remaining -= dstLen;
309 }
310
311 assert(!remaining);
312
313 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
314 }
315
316 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
317
318 void
319 hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
320 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
321 {
322 count += ppnum;
323 switch (kind)
324 {
325 case kIOHibernatePageStateUnwiredSave:
326 // unwired save
327 for (; ppnum < count; ppnum++)
328 {
329 hibernate_page_bitset(page_list, FALSE, ppnum);
330 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
331 }
332 break;
333 case kIOHibernatePageStateWiredSave:
334 // wired save
335 for (; ppnum < count; ppnum++)
336 {
337 hibernate_page_bitset(page_list, FALSE, ppnum);
338 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
339 }
340 break;
341 case kIOHibernatePageStateFree:
342 // free page
343 for (; ppnum < count; ppnum++)
344 {
345 hibernate_page_bitset(page_list, TRUE, ppnum);
346 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
347 }
348 break;
349 default:
350 panic("hibernate_set_page_state");
351 }
352 }
353
354 static vm_offset_t
355 hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
356 {
357 uint32_t page = *pPage;
358 uint32_t count;
359 hibernate_bitmap_t * bitmap;
360
361 while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
362 {
363 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
364 if (!count)
365 break;
366 page += count;
367 if (page <= bitmap->last_page)
368 break;
369 }
370
371 *pPage = page;
372 if (bitmap)
373 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
374 else
375 count = 0;
376
377 return (count);
378 }
379
380 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
381
382 static IOReturn
383 IOHibernatePollerProbe(IOPolledFileIOVars * vars, IOService * target)
384 {
385 IOReturn err = kIOReturnError;
386 int32_t idx;
387 IOPolledInterface * poller;
388
389 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
390 {
391 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
392 err = poller->probe(target);
393 if (err)
394 {
395 HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
396 break;
397 }
398 }
399
400 return (err);
401 }
402
403 static IOReturn
404 IOHibernatePollerOpen(IOPolledFileIOVars * vars, uint32_t state, IOMemoryDescriptor * md)
405 {
406 IOReturn err = kIOReturnError;
407 int32_t idx;
408 IOPolledInterface * poller;
409
410 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
411 {
412 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
413 err = poller->open(state, md);
414 if (err)
415 {
416 HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
417 break;
418 }
419 }
420
421 return (err);
422 }
423
424 static IOReturn
425 IOHibernatePollerClose(IOPolledFileIOVars * vars, uint32_t state)
426 {
427 IOReturn err = kIOReturnError;
428 int32_t idx;
429 IOPolledInterface * poller;
430
431 for (idx = 0;
432 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
433 idx++)
434 {
435 err = poller->close(state);
436 if (err)
437 HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
438 }
439
440 return (err);
441 }
442
443 static void
444 IOHibernatePollerIOComplete(void * target,
445 void * parameter,
446 IOReturn status,
447 UInt64 actualByteCount)
448 {
449 IOPolledFileIOVars * vars = (IOPolledFileIOVars *) parameter;
450
451 vars->ioStatus = status;
452 }
453
454 static IOReturn
455 IOHibernatePollerIO(IOPolledFileIOVars * vars,
456 uint32_t operation, uint32_t bufferOffset,
457 uint64_t deviceOffset, uint64_t length)
458 {
459
460 IOReturn err = kIOReturnError;
461 IOPolledInterface * poller;
462 IOPolledCompletion completion;
463
464 completion.target = 0;
465 completion.action = &IOHibernatePollerIOComplete;
466 completion.parameter = vars;
467
468 vars->ioStatus = -1;
469
470 poller = (IOPolledInterface *) vars->pollers->getObject(0);
471 err = poller->startIO(operation, bufferOffset, deviceOffset + vars->block0, length, completion);
472 if (err)
473 HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
474
475 return (err);
476 }
477
478 static IOReturn
479 IOHibernatePollerIODone(IOPolledFileIOVars * vars, bool abortable)
480 {
481 IOReturn err = kIOReturnSuccess;
482 int32_t idx = 0;
483 IOPolledInterface * poller;
484
485 while (-1 == vars->ioStatus)
486 {
487 for (idx = 0;
488 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
489 idx++)
490 {
491 IOReturn newErr;
492 newErr = poller->checkForWork();
493 if ((newErr == kIOReturnAborted) && !abortable)
494 newErr = kIOReturnSuccess;
495 if (kIOReturnSuccess == err)
496 err = newErr;
497 }
498 }
499
500 if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort())
501 {
502 err = kIOReturnAborted;
503 HIBLOG("IOPolledInterface::checkForWork sw abort\n");
504 }
505
506 if (err)
507 {
508 HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
509 }
510 else
511 {
512 err = vars->ioStatus;
513 if (kIOReturnSuccess != err)
514 HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
515 }
516
517 return (err);
518 }
519
520 IOReturn
521 IOPolledInterface::checkAllForWork(void)
522 {
523 IOReturn err = kIOReturnNotReady;
524 int32_t idx;
525 IOPolledInterface * poller;
526
527 IOHibernateVars * vars = &gIOHibernateVars;
528
529 if (!vars->fileVars || !vars->fileVars->pollers)
530 return (err);
531
532 for (idx = 0;
533 (poller = (IOPolledInterface *) vars->fileVars->pollers->getObject(idx));
534 idx++)
535 {
536 err = poller->checkForWork();
537 if (err)
538 HIBLOG("IOPolledInterface::checkAllForWork[%d] 0x%x\n", idx, err);
539 }
540
541 return (err);
542 }
543
544 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
545
546 struct _OpenFileContext
547 {
548 OSData * extents;
549 uint64_t size;
550 };
551
552 static void
553 file_extent_callback(void * ref, uint64_t start, uint64_t length)
554 {
555 _OpenFileContext * ctx = (_OpenFileContext *) ref;
556 IOPolledFileExtent extent;
557
558 extent.start = start;
559 extent.length = length;
560
561 ctx->extents->appendBytes(&extent, sizeof(extent));
562 ctx->size += length;
563 }
564
565 static IOService *
566 IOCopyMediaForDev(dev_t device)
567 {
568 OSDictionary * matching;
569 OSNumber * num;
570 OSIterator * iter;
571 IOService * result = 0;
572
573 matching = IOService::serviceMatching("IOMedia");
574 if (!matching)
575 return (0);
576 do
577 {
578 num = OSNumber::withNumber(major(device), 32);
579 if (!num)
580 break;
581 matching->setObject(kIOBSDMajorKey, num);
582 num->release();
583 num = OSNumber::withNumber(minor(device), 32);
584 if (!num)
585 break;
586 matching->setObject(kIOBSDMinorKey, num);
587 num->release();
588 if (!num)
589 break;
590 iter = IOService::getMatchingServices(matching);
591 if (iter)
592 {
593 result = (IOService *) iter->getNextObject();
594 result->retain();
595 iter->release();
596 }
597 }
598 while (false);
599 matching->release();
600
601 return (result);
602 }
603
604 IOReturn
605 IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
606 IOPolledFileIOVars ** fileVars, OSData ** fileExtents,
607 OSData ** imagePath, uint8_t * volumeCryptKey)
608 {
609 IOReturn err = kIOReturnSuccess;
610 IOPolledFileIOVars * vars;
611 _OpenFileContext ctx;
612 OSData * extentsData;
613 OSNumber * num;
614 IOService * part = 0;
615 OSString * keyUUID = 0;
616 OSString * keyStoreUUID = 0;
617 dev_t block_dev;
618 dev_t hibernate_image_dev;
619 uint64_t maxiobytes;
620 AbsoluteTime startTime, endTime;
621 uint64_t nsec;
622
623 vars = IONew(IOPolledFileIOVars, 1);
624 if (!vars) return (kIOReturnNoMemory);
625 bzero(vars, sizeof(*vars));
626
627 do
628 {
629 HIBLOG("sizeof(IOHibernateImageHeader) == %ld\n", sizeof(IOHibernateImageHeader));
630 if (sizeof(IOHibernateImageHeader) != 512)
631 continue;
632
633 vars->io = false;
634 vars->buffer = (uint8_t *) ioBuffer->getBytesNoCopy();
635 vars->bufferHalf = 0;
636 vars->bufferOffset = 0;
637 vars->bufferSize = ioBuffer->getLength() >> 1;
638
639 extentsData = OSData::withCapacity(32);
640 ctx.extents = extentsData;
641 ctx.size = 0;
642 clock_get_uptime(&startTime);
643 vars->fileRef = kern_open_file_for_direct_io(filename,
644 &file_extent_callback, &ctx,
645 &block_dev,
646 &hibernate_image_dev,
647 &vars->block0,
648 &maxiobytes,
649 &vars->flags,
650 0, (caddr_t) gIOHibernateCurrentHeader,
651 sizeof(IOHibernateImageHeader));
652 #if 0
653 uint32_t msDelay = (131071 & random());
654 HIBLOG("sleep %d\n", msDelay);
655 IOSleep(msDelay);
656 #endif
657 clock_get_uptime(&endTime);
658 SUB_ABSOLUTETIME(&endTime, &startTime);
659 absolutetime_to_nanoseconds(endTime, &nsec);
660
661 if (!vars->fileRef) err = kIOReturnNoSpace;
662
663 IOLockLock(gFSLock);
664 if (kFSOpening != gFSState) err = kIOReturnTimeout;
665 IOLockUnlock(gFSLock);
666
667 HIBLOG("kern_open_file_for_direct_io(%d) took %qd ms\n", err, nsec / 1000000ULL);
668 if (kIOReturnSuccess != err) break;
669
670 if (kIOHibernateModeSSDInvert & gIOHibernateMode)
671 vars->flags ^= kIOHibernateOptionSSD;
672
673 HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx ssd %d\n", filename, ctx.size,
674 vars->block0, maxiobytes, kIOHibernateOptionSSD & vars->flags);
675 if (ctx.size < 1*1024*1024) // check against image size estimate!
676 {
677 err = kIOReturnNoSpace;
678 break;
679 }
680
681 if (maxiobytes < vars->bufferSize)
682 vars->bufferSize = maxiobytes;
683
684 vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
685
686 part = IOCopyMediaForDev(block_dev);
687 if (!part)
688 break;
689
690 err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false,
691 (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
692 if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID)
693 {
694 // IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
695 uuid_t volumeKeyUUID;
696 aks_volume_key_t vek;
697 static IOService * sKeyStore;
698 static const OSSymbol * sAKSGetKey;
699
700 if (!sAKSGetKey)
701 sAKSGetKey = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY);
702 if (!sKeyStore)
703 sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
704 if (sKeyStore)
705 err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
706 else
707 err = kIOReturnNoResources;
708 if (kIOReturnSuccess == err)
709 err = sKeyStore->callPlatformFunction(sAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
710 if (kIOReturnSuccess != err)
711 IOLog("volume key err 0x%x\n", err);
712 else
713 {
714 size_t bytes = (kIOHibernateAESKeySize / 8);
715 if (vek.key.keybytecount < bytes)
716 bytes = vek.key.keybytecount;
717 bcopy(&vek.key.keybytes[0], volumeCryptKey, bytes);
718 }
719 bzero(&vek, sizeof(vek));
720 }
721 part->release();
722
723 part = IOCopyMediaForDev(hibernate_image_dev);
724 if (!part)
725 break;
726
727 IORegistryEntry * next;
728 IORegistryEntry * child;
729 OSData * data;
730
731 vars->pollers = OSArray::withCapacity(4);
732 if (!vars->pollers)
733 break;
734
735 vars->blockSize = 512;
736 next = part;
737 do
738 {
739 IOPolledInterface * poller;
740 OSObject * obj;
741
742 obj = next->getProperty(kIOPolledInterfaceSupportKey);
743 if (kOSBooleanFalse == obj)
744 {
745 vars->pollers->flushCollection();
746 break;
747 }
748 else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
749 vars->pollers->setObject(poller);
750 if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey))))
751 vars->blockSize = num->unsigned32BitValue();
752 child = next;
753 }
754 while ((next = child->getParentEntry(gIOServicePlane))
755 && child->isParent(next, gIOServicePlane, true));
756
757 HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
758 major(hibernate_image_dev), minor(hibernate_image_dev), (long)vars->blockSize, vars->pollers->getCount());
759 if (vars->pollers->getCount() < kIOHibernateMinPollersNeeded)
760 continue;
761
762 err = IOHibernatePollerProbe(vars, (IOService *) part);
763 if (kIOReturnSuccess != err)
764 break;
765
766 err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer);
767 if (kIOReturnSuccess != err)
768 break;
769
770 *fileVars = vars;
771 *fileExtents = extentsData;
772
773 // make imagePath
774
775 if ((extentsData->getLength() >= sizeof(IOPolledFileExtent)))
776 {
777 char str2[24 + sizeof(uuid_string_t) + 2];
778
779 #if defined(__i386__) || defined(__x86_64__)
780 if (!gIOCreateEFIDevicePathSymbol)
781 gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
782
783 if (keyUUID)
784 snprintf(str2, sizeof(str2), "%qx:%s",
785 vars->extentMap[0].start, keyUUID->getCStringNoCopy());
786 else
787 snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
788
789 err = IOService::getPlatform()->callPlatformFunction(
790 gIOCreateEFIDevicePathSymbol, false,
791 (void *) part, (void *) str2,
792 (void *) (uintptr_t) true, (void *) &data);
793 #else
794 char str1[256];
795 int len = sizeof(str1);
796
797 if (!part->getPath(str1, &len, gIODTPlane))
798 err = kIOReturnNotFound;
799 else
800 {
801 snprintf(str2, sizeof(str2), ",%qx", vars->extentMap[0].start);
802 // (strip the plane name)
803 char * tail = strchr(str1, ':');
804 if (!tail)
805 tail = str1 - 1;
806 data = OSData::withBytes(tail + 1, strlen(tail + 1));
807 data->appendBytes(str2, strlen(str2));
808 }
809 #endif
810 if (kIOReturnSuccess == err)
811 *imagePath = data;
812 else
813 HIBLOG("error 0x%x getting path\n", err);
814 }
815 }
816 while (false);
817
818 if (kIOReturnSuccess != err)
819 {
820 HIBLOG("error 0x%x opening hibernation file\n", err);
821 if (vars->fileRef)
822 {
823 kern_close_file_for_direct_io(vars->fileRef, 0, 0, 0, 0, 0);
824 vars->fileRef = NULL;
825 }
826 }
827
828 if (part)
829 part->release();
830
831 return (err);
832 }
833
834 IOReturn
835 IOPolledFileClose( IOPolledFileIOVars * vars )
836 {
837 if (vars->pollers)
838 {
839 IOHibernatePollerClose(vars, kIOPolledPostflightState);
840 vars->pollers->release();
841 }
842
843 bzero(vars, sizeof(IOPolledFileIOVars));
844
845 return (kIOReturnSuccess);
846 }
847
848 static IOReturn
849 IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
850 {
851 IOPolledFileExtent * extentMap;
852
853 extentMap = vars->extentMap;
854
855 vars->position = position;
856
857 while (position >= extentMap->length)
858 {
859 position -= extentMap->length;
860 extentMap++;
861 }
862
863 vars->currentExtent = extentMap;
864 vars->extentRemaining = extentMap->length - position;
865 vars->extentPosition = vars->position - position;
866
867 if (vars->bufferSize <= vars->extentRemaining)
868 vars->bufferLimit = vars->bufferSize;
869 else
870 vars->bufferLimit = vars->extentRemaining;
871
872 return (kIOReturnSuccess);
873 }
874
875 static IOReturn
876 IOPolledFileWrite(IOPolledFileIOVars * vars,
877 const uint8_t * bytes, IOByteCount size,
878 hibernate_cryptvars_t * cryptvars)
879 {
880 IOReturn err = kIOReturnSuccess;
881 IOByteCount copy;
882 bool flush = false;
883
884 do
885 {
886 if (!bytes && !size)
887 {
888 // seek to end of block & flush
889 size = vars->position & (vars->blockSize - 1);
890 if (size)
891 size = vars->blockSize - size;
892 flush = true;
893 // use some garbage for the fill
894 bytes = vars->buffer + vars->bufferOffset;
895 }
896
897 copy = vars->bufferLimit - vars->bufferOffset;
898 if (copy > size)
899 copy = size;
900 else
901 flush = true;
902
903 if (bytes)
904 {
905 bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
906 bytes += copy;
907 }
908 else
909 bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
910
911 size -= copy;
912 vars->bufferOffset += copy;
913 vars->position += copy;
914
915 if (flush && vars->bufferOffset)
916 {
917 uint64_t offset = (vars->position - vars->bufferOffset
918 - vars->extentPosition + vars->currentExtent->start);
919 uint32_t length = (vars->bufferOffset);
920
921 #if CRYPTO
922 if (cryptvars && vars->encryptStart
923 && (vars->position > vars->encryptStart)
924 && ((vars->position - length) < vars->encryptEnd))
925 {
926 AbsoluteTime startTime, endTime;
927
928 uint64_t encryptLen, encryptStart;
929 encryptLen = vars->position - vars->encryptStart;
930 if (encryptLen > length)
931 encryptLen = length;
932 encryptStart = length - encryptLen;
933 if (vars->position > vars->encryptEnd)
934 encryptLen -= (vars->position - vars->encryptEnd);
935
936 clock_get_uptime(&startTime);
937
938 // encrypt the buffer
939 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
940 &cryptvars->aes_iv[0],
941 encryptLen / AES_BLOCK_SIZE,
942 vars->buffer + vars->bufferHalf + encryptStart,
943 &cryptvars->ctx.encrypt);
944
945 clock_get_uptime(&endTime);
946 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
947 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
948 vars->cryptBytes += encryptLen;
949
950 // save initial vector for following encrypts
951 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
952 &cryptvars->aes_iv[0],
953 AES_BLOCK_SIZE);
954 }
955 #endif /* CRYPTO */
956
957 if (vars->io)
958 {
959 err = IOHibernatePollerIODone(vars, true);
960 if (kIOReturnSuccess != err)
961 break;
962 }
963
964 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
965 //if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
966
967 err = IOHibernatePollerIO(vars, kIOPolledWrite, vars->bufferHalf, offset, length);
968 if (kIOReturnSuccess != err)
969 break;
970 vars->io = true;
971
972 vars->extentRemaining -= vars->bufferOffset;
973 if (!vars->extentRemaining)
974 {
975 vars->currentExtent++;
976 vars->extentRemaining = vars->currentExtent->length;
977 vars->extentPosition = vars->position;
978 if (!vars->extentRemaining)
979 {
980 err = kIOReturnOverrun;
981 break;
982 }
983 }
984
985 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
986 vars->bufferOffset = 0;
987 if (vars->bufferSize <= vars->extentRemaining)
988 vars->bufferLimit = vars->bufferSize;
989 else
990 vars->bufferLimit = vars->extentRemaining;
991
992 flush = false;
993 }
994 }
995 while (size);
996
997 return (err);
998 }
999
1000 static IOReturn
1001 IOPolledFileRead(IOPolledFileIOVars * vars,
1002 uint8_t * bytes, IOByteCount size,
1003 hibernate_cryptvars_t * cryptvars)
1004 {
1005 IOReturn err = kIOReturnSuccess;
1006 IOByteCount copy;
1007
1008 // bytesWritten += size;
1009
1010 do
1011 {
1012 copy = vars->bufferLimit - vars->bufferOffset;
1013 if (copy > size)
1014 copy = size;
1015
1016 if (bytes)
1017 {
1018 bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
1019 bytes += copy;
1020 }
1021 size -= copy;
1022 vars->bufferOffset += copy;
1023 // vars->position += copy;
1024
1025 if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd))
1026 {
1027 if (vars->io)
1028 {
1029 err = IOHibernatePollerIODone(vars, false);
1030 if (kIOReturnSuccess != err)
1031 break;
1032 }
1033 else
1034 cryptvars = 0;
1035
1036 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
1037
1038 vars->position += vars->lastRead;
1039 vars->extentRemaining -= vars->lastRead;
1040 vars->bufferLimit = vars->lastRead;
1041
1042 if (!vars->extentRemaining)
1043 {
1044 vars->currentExtent++;
1045 vars->extentRemaining = vars->currentExtent->length;
1046 vars->extentPosition = vars->position;
1047 if (!vars->extentRemaining)
1048 {
1049 err = kIOReturnOverrun;
1050 break;
1051 }
1052 }
1053
1054 uint64_t length;
1055 uint64_t lastReadLength = vars->lastRead;
1056 uint64_t offset = (vars->position
1057 - vars->extentPosition + vars->currentExtent->start);
1058 if (vars->extentRemaining <= vars->bufferSize)
1059 length = vars->extentRemaining;
1060 else
1061 length = vars->bufferSize;
1062 if ((length + vars->position) > vars->readEnd)
1063 length = vars->readEnd - vars->position;
1064
1065 vars->lastRead = length;
1066 if (length)
1067 {
1068 //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
1069 err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
1070 if (kIOReturnSuccess != err)
1071 break;
1072 vars->io = true;
1073 }
1074
1075 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1076 vars->bufferOffset = 0;
1077
1078 #if CRYPTO
1079 if (cryptvars)
1080 {
1081 uint8_t thisVector[AES_BLOCK_SIZE];
1082 AbsoluteTime startTime, endTime;
1083
1084 // save initial vector for following decrypts
1085 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
1086 bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
1087 &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1088
1089 // decrypt the buffer
1090 clock_get_uptime(&startTime);
1091
1092 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
1093 &thisVector[0],
1094 lastReadLength / AES_BLOCK_SIZE,
1095 vars->buffer + vars->bufferHalf,
1096 &cryptvars->ctx.decrypt);
1097
1098 clock_get_uptime(&endTime);
1099 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
1100 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
1101 vars->cryptBytes += lastReadLength;
1102 }
1103 #endif /* CRYPTO */
1104 }
1105 }
1106 while (size);
1107
1108 return (err);
1109 }
1110
1111 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1112
1113 IOReturn
1114 IOHibernateSystemSleep(void)
1115 {
1116 IOReturn err;
1117 OSData * data;
1118 OSObject * obj;
1119 OSString * str;
1120 bool dsSSD;
1121 IOHibernateVars * vars;
1122
1123 gIOHibernateState = kIOHibernateStateInactive;
1124
1125 if (!gIOChosenEntry)
1126 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1127
1128 gIOHibernateDebugFlags = 0;
1129 if (kIOLogHibernate & gIOKitDebug)
1130 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
1131
1132 if (IOService::getPMRootDomain()->getHibernateSettings(
1133 &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime))
1134 {
1135 if (kIOHibernateModeSleep & gIOHibernateMode)
1136 // default to discard clean for safe sleep
1137 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
1138 | kIOHibernateModeDiscardCleanActive);
1139 }
1140
1141 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
1142 {
1143 if ((str = OSDynamicCast(OSString, obj)))
1144 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
1145 sizeof(gIOHibernateFilename));
1146 obj->release();
1147 }
1148
1149 if (!gIOHibernateMode || !gIOHibernateFilename[0])
1150 return (kIOReturnUnsupported);
1151
1152 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
1153
1154 vars = IONew(IOHibernateVars, 1);
1155 if (!vars) return (kIOReturnNoMemory);
1156 bzero(vars, sizeof(*vars));
1157
1158 IOLockLock(gFSLock);
1159 if (kFSIdle != gFSState)
1160 {
1161 HIBLOG("hibernate file busy\n");
1162 IOLockUnlock(gFSLock);
1163 IODelete(vars, IOHibernateVars, 1);
1164 return (kIOReturnBusy);
1165 }
1166 gFSState = kFSOpening;
1167 IOLockUnlock(gFSLock);
1168
1169 do
1170 {
1171 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1172 4 * page_size, page_size);
1173 vars->ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1174 2 * kDefaultIOSize, page_size);
1175
1176 vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1177 ptoa_64(gIOHibernateHandoffPageCount), page_size);
1178
1179 if (!vars->srcBuffer || !vars->ioBuffer || !vars->handoffBuffer)
1180 {
1181 err = kIOReturnNoMemory;
1182 break;
1183 }
1184
1185 // open & invalidate the image file
1186 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1187 err = IOPolledFileOpen(gIOHibernateFilename, vars->ioBuffer,
1188 &vars->fileVars, &vars->fileExtents, &data,
1189 &vars->volumeCryptKey[0]);
1190 if (KERN_SUCCESS != err)
1191 {
1192 HIBLOG("IOPolledFileOpen(%x)\n", err);
1193 break;
1194 }
1195
1196 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1197 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
1198 dsSSD = ((0 != (kIOHibernateOptionSSD & vars->fileVars->flags))
1199 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
1200 if (dsSSD)
1201 {
1202 gIOHibernateCurrentHeader->options |=
1203 kIOHibernateOptionSSD
1204 | kIOHibernateOptionColor;
1205
1206 #if defined(__i386__) || defined(__x86_64__)
1207 if (!uuid_is_null(vars->volumeCryptKey) &&
1208 (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey)))
1209 {
1210 uintptr_t smcVars[2];
1211 smcVars[0] = sizeof(vars->volumeCryptKey);
1212 smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
1213
1214 IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
1215 bzero(smcVars, sizeof(smcVars));
1216 }
1217 #endif
1218 }
1219 else
1220 {
1221 gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
1222 }
1223
1224 boolean_t encryptedswap;
1225 AbsoluteTime startTime, endTime;
1226 uint64_t nsec;
1227
1228 clock_get_uptime(&startTime);
1229 err = hibernate_setup(gIOHibernateCurrentHeader,
1230 gIOHibernateFreeRatio, gIOHibernateFreeTime,
1231 dsSSD,
1232 &vars->page_list, &vars->page_list_wired, &vars->page_list_pal, &encryptedswap);
1233 clock_get_uptime(&endTime);
1234 SUB_ABSOLUTETIME(&endTime, &startTime);
1235 absolutetime_to_nanoseconds(endTime, &nsec);
1236 HIBLOG("hibernate_setup(%d) took %qd ms\n", err, nsec / 1000000ULL);
1237
1238 if (KERN_SUCCESS != err)
1239 break;
1240
1241 if (encryptedswap || !uuid_is_null(vars->volumeCryptKey))
1242 gIOHibernateMode ^= kIOHibernateModeEncrypt;
1243
1244 if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1245 {
1246 vars->videoAllocSize = kVideoMapSize;
1247 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
1248 vars->videoMapping = 0;
1249 }
1250
1251 // generate crypt keys
1252 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
1253 vars->wiredCryptKey[i] = random();
1254 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
1255 vars->cryptKey[i] = random();
1256
1257 // set nvram
1258
1259 IORegistryEntry * regEntry;
1260 if (!gIOOptionsEntry)
1261 {
1262 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
1263 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
1264 if (regEntry && !gIOOptionsEntry)
1265 regEntry->release();
1266 }
1267
1268 if (gIOOptionsEntry)
1269 {
1270 const OSSymbol * sym;
1271
1272 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1273 if (sym)
1274 {
1275 gIOOptionsEntry->setProperty(sym, data);
1276 sym->release();
1277 }
1278 data->release();
1279
1280 #if defined(__i386__) || defined(__x86_64__)
1281 struct AppleRTCHibernateVars
1282 {
1283 uint8_t signature[4];
1284 uint32_t revision;
1285 uint8_t booterSignature[20];
1286 uint8_t wiredCryptKey[16];
1287 };
1288 AppleRTCHibernateVars rtcVars;
1289
1290 rtcVars.signature[0] = 'A';
1291 rtcVars.signature[1] = 'A';
1292 rtcVars.signature[2] = 'P';
1293 rtcVars.signature[3] = 'L';
1294 rtcVars.revision = 1;
1295 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
1296 if (gIOHibernateBootSignature[0])
1297 {
1298 char c;
1299 uint8_t value = 0;
1300 for (uint32_t i = 0;
1301 (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
1302 i++)
1303 {
1304 if (c >= 'a')
1305 c -= 'a' - 10;
1306 else if (c >= 'A')
1307 c -= 'A' - 10;
1308 else if (c >= '0')
1309 c -= '0';
1310 else
1311 continue;
1312 value = (value << 4) | c;
1313 if (i & 1)
1314 rtcVars.booterSignature[i >> 1] = value;
1315 }
1316 }
1317 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
1318 if (data)
1319 {
1320 if (!gIOHibernateRTCVariablesKey)
1321 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1322 if (gIOHibernateRTCVariablesKey)
1323 IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
1324
1325 if( gIOOptionsEntry )
1326 {
1327 if( gIOHibernateMode & kIOHibernateModeSwitch )
1328 {
1329 const OSSymbol *sym;
1330 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSwitchVarsKey);
1331 if( sym )
1332 {
1333 gIOOptionsEntry->setProperty(sym, data); /* intentional insecure backup of rtc boot vars */
1334 sym->release();
1335 }
1336 }
1337 }
1338
1339 data->release();
1340 }
1341 if (gIOChosenEntry)
1342 {
1343 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
1344 if (data)
1345 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
1346 {
1347 // set BootNext
1348
1349 if (!gIOHibernateBoot0082Data)
1350 {
1351 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
1352 if (data)
1353 {
1354 // AppleNVRAM_EFI_LOAD_OPTION
1355 struct {
1356 uint32_t Attributes;
1357 uint16_t FilePathLength;
1358 uint16_t Desc;
1359 } loadOptionHeader;
1360 loadOptionHeader.Attributes = 1;
1361 loadOptionHeader.FilePathLength = data->getLength();
1362 loadOptionHeader.Desc = 0;
1363 gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
1364 if (gIOHibernateBoot0082Data)
1365 {
1366 gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
1367 gIOHibernateBoot0082Data->appendBytes(data);
1368 }
1369 }
1370 }
1371 if (!gIOHibernateBoot0082Key)
1372 gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
1373 if (!gIOHibernateBootNextKey)
1374 gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
1375 if (!gIOHibernateBootNextData)
1376 {
1377 uint16_t bits = 0x0082;
1378 gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
1379 }
1380 if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData)
1381 {
1382 gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
1383 gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
1384 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
1385 }
1386 }
1387 }
1388 #else /* !i386 && !x86_64 */
1389 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1390 {
1391 data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1392 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
1393 if (sym && data)
1394 gIOOptionsEntry->setProperty(sym, data);
1395 if (sym)
1396 sym->release();
1397 if (data)
1398 data->release();
1399 if (false && gIOHibernateBootSignature[0])
1400 {
1401 data = OSData::withCapacity(16);
1402 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
1403 if (sym && data)
1404 {
1405 char c;
1406 uint8_t value = 0;
1407 for (uint32_t i = 0; (c = gIOHibernateBootSignature[i]); i++)
1408 {
1409 if (c >= 'a')
1410 c -= 'a' - 10;
1411 else if (c >= 'A')
1412 c -= 'A' - 10;
1413 else if (c >= '0')
1414 c -= '0';
1415 else
1416 continue;
1417 value = (value << 4) | c;
1418 if (i & 1)
1419 data->appendBytes(&value, sizeof(value));
1420 }
1421 gIOOptionsEntry->setProperty(sym, data);
1422 }
1423 if (sym)
1424 sym->release();
1425 if (data)
1426 data->release();
1427 }
1428 }
1429 if (!vars->haveFastBoot)
1430 {
1431 // set boot volume to zero
1432 IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
1433 if (platform && (kIOReturnSuccess == platform->readXPRAM(kXPRamAudioVolume,
1434 &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume))))
1435 {
1436 uint8_t newVolume;
1437 newVolume = vars->saveBootAudioVolume & 0xf8;
1438 platform->writeXPRAM(kXPRamAudioVolume,
1439 &newVolume, sizeof(newVolume));
1440 }
1441 }
1442 #endif /* !i386 && !x86_64 */
1443 }
1444 // --
1445
1446 }
1447 while (false);
1448
1449 IOLockLock(gFSLock);
1450 if ((kIOReturnSuccess == err) && (kFSOpening == gFSState))
1451 {
1452 gFSState = kFSOpened;
1453 gIOHibernateVars = *vars;
1454 gFileVars = *vars->fileVars;
1455 gIOHibernateVars.fileVars = &gFileVars;
1456 gIOHibernateFileRef = gFileVars.fileRef;
1457 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
1458 gIOHibernateState = kIOHibernateStateHibernating;
1459 }
1460 else
1461 {
1462 HIBLOG("hibernate file close due timeout\n");
1463 if (vars->fileVars && vars->fileVars->fileRef) kern_close_file_for_direct_io(vars->fileVars->fileRef, 0, 0, 0, 0, 0);
1464 IOHibernateDone(vars);
1465 gFSState = kFSIdle;
1466 }
1467 IOLockUnlock(gFSLock);
1468
1469 if (vars->fileVars) IODelete(vars->fileVars, IOPolledFileIOVars, 1);
1470 IODelete(vars, IOHibernateVars, 1);
1471
1472 return (err);
1473 }
1474
1475 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1476
1477 DECLARE_IOHIBERNATEPROGRESSALPHA
1478
1479 static void
1480 ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
1481 {
1482 uint32_t rowBytes, pixelShift;
1483 uint32_t x, y;
1484 int32_t blob;
1485 uint32_t alpha, in, color, result;
1486 uint8_t * out;
1487 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1488
1489 rowBytes = display->rowBytes;
1490 pixelShift = display->depth >> 4;
1491 if (pixelShift < 1) return;
1492
1493 screen += ((display->width
1494 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1495 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1496
1497 for (y = 0; y < kIOHibernateProgressHeight; y++)
1498 {
1499 out = screen + y * rowBytes;
1500 for (blob = 0; blob < kIOHibernateProgressCount; blob++)
1501 {
1502 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
1503 for (x = 0; x < kIOHibernateProgressWidth; x++)
1504 {
1505 alpha = gIOHibernateProgressAlpha[y][x];
1506 result = color;
1507 if (alpha)
1508 {
1509 if (0xff != alpha)
1510 {
1511 if (1 == pixelShift)
1512 {
1513 in = *((uint16_t *)out) & 0x1f; // 16
1514 in = (in << 3) | (in >> 2);
1515 }
1516 else
1517 in = *((uint32_t *)out) & 0xff; // 32
1518 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
1519 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
1520 }
1521 if (1 == pixelShift)
1522 {
1523 result >>= 3;
1524 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1525 }
1526 else
1527 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1528 }
1529 out += (1 << pixelShift);
1530 }
1531 out += (kIOHibernateProgressSpacing << pixelShift);
1532 }
1533 }
1534 }
1535
1536
1537 static void
1538 ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
1539 {
1540 uint32_t rowBytes, pixelShift;
1541 uint32_t x, y;
1542 int32_t blob, lastBlob;
1543 uint32_t alpha, in, color, result;
1544 uint8_t * out;
1545 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1546
1547 pixelShift = display->depth >> 4;
1548 if (pixelShift < 1)
1549 return;
1550
1551 rowBytes = display->rowBytes;
1552
1553 screen += ((display->width
1554 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1555 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1556
1557 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
1558
1559 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
1560
1561 for (y = 0; y < kIOHibernateProgressHeight; y++)
1562 {
1563 out = screen + y * rowBytes;
1564 for (blob = firstBlob; blob <= lastBlob; blob++)
1565 {
1566 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
1567 for (x = 0; x < kIOHibernateProgressWidth; x++)
1568 {
1569 alpha = gIOHibernateProgressAlpha[y][x];
1570 result = color;
1571 if (alpha)
1572 {
1573 if (0xff != alpha)
1574 {
1575 in = display->progressSaveUnder[blob][saveindex[blob]++];
1576 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
1577 }
1578 if (1 == pixelShift)
1579 {
1580 result >>= 3;
1581 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1582 }
1583 else
1584 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1585 }
1586 out += (1 << pixelShift);
1587 }
1588 out += (kIOHibernateProgressSpacing << pixelShift);
1589 }
1590 }
1591 }
1592
1593 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1594
1595 IOReturn
1596 IOHibernateIOKitSleep(void)
1597 {
1598 IOReturn ret = kIOReturnSuccess;
1599 IOLockLock(gFSLock);
1600 if (kFSOpening == gFSState)
1601 {
1602 gFSState = kFSTimedOut;
1603 HIBLOG("hibernate file open timed out\n");
1604 ret = kIOReturnTimeout;
1605 }
1606 IOLockUnlock(gFSLock);
1607 return (ret);
1608 }
1609
1610 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1611
1612 IOReturn
1613 IOHibernateSystemHasSlept(void)
1614 {
1615 IOReturn ret = kIOReturnSuccess;
1616 IOHibernateVars * vars = &gIOHibernateVars;
1617 OSObject * obj = 0;
1618 OSData * data;
1619
1620 IOLockLock(gFSLock);
1621 if ((kFSOpened != gFSState) && gIOHibernateMode)
1622 {
1623 ret = kIOReturnTimeout;
1624 }
1625 IOLockUnlock(gFSLock);
1626 if (kIOReturnSuccess != ret) return (ret);
1627
1628 if (gIOHibernateMode) obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
1629 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1630 if (obj && !vars->previewBuffer)
1631 obj->release();
1632
1633 vars->consoleMapping = NULL;
1634 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
1635 {
1636 vars->previewBuffer->release();
1637 vars->previewBuffer = 0;
1638 }
1639
1640 if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1641 && vars->previewBuffer
1642 && (data = OSDynamicCast(OSData,
1643 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
1644 {
1645 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
1646 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
1647
1648 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1649
1650 if (kIOHibernatePreviewUpdates & flags)
1651 {
1652 PE_Video consoleInfo;
1653 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1654
1655 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1656
1657 graphicsInfo->width = consoleInfo.v_width;
1658 graphicsInfo->height = consoleInfo.v_height;
1659 graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
1660 graphicsInfo->depth = consoleInfo.v_depth;
1661 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1662
1663 HIBPRINT("video %p %d %d %d\n",
1664 vars->consoleMapping, graphicsInfo->depth,
1665 graphicsInfo->width, graphicsInfo->height);
1666 if (vars->consoleMapping)
1667 ProgressInit(graphicsInfo, vars->consoleMapping,
1668 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1669 }
1670 }
1671
1672 if (gIOOptionsEntry)
1673 gIOOptionsEntry->sync();
1674
1675 return (ret);
1676 }
1677
1678 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1679
1680 static DeviceTreeNode *
1681 MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry)
1682 {
1683 DeviceTreeNodeProperty * prop;
1684 DeviceTreeNode * child;
1685 IORegistryEntry * childRegEntry;
1686 const char * nameProp;
1687 unsigned int propLen, idx;
1688
1689 prop = (DeviceTreeNodeProperty *) (entry + 1);
1690 for (idx = 0; idx < entry->nProperties; idx++)
1691 {
1692 if (regEntry && (0 != strcmp("name", prop->name)))
1693 {
1694 regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
1695 // HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
1696 }
1697 prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
1698 }
1699
1700 child = (DeviceTreeNode *) prop;
1701 for (idx = 0; idx < entry->nChildren; idx++)
1702 {
1703 if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen))
1704 panic("no name");
1705 childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
1706 // HIBPRINT("%s == %p\n", nameProp, childRegEntry);
1707 child = MergeDeviceTree(child, childRegEntry);
1708 }
1709 return (child);
1710 }
1711
1712 IOReturn
1713 IOHibernateSystemWake(void)
1714 {
1715 if (kFSOpened == gFSState)
1716 {
1717 IOHibernateDone(&gIOHibernateVars);
1718 }
1719 else
1720 {
1721 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1722 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1723 }
1724 return (kIOReturnSuccess);
1725 }
1726
1727 static IOReturn
1728 IOHibernateDone(IOHibernateVars * vars)
1729 {
1730 hibernate_teardown(vars->page_list, vars->page_list_wired);
1731
1732 if (vars->videoMapping)
1733 {
1734 if (vars->videoMapSize)
1735 // remove mappings
1736 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1737 if (vars->videoAllocSize)
1738 // dealloc range
1739 kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
1740 }
1741
1742 if (vars->previewBuffer)
1743 {
1744 vars->previewBuffer->release();
1745 vars->previewBuffer = 0;
1746 }
1747
1748 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1749 {
1750 IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
1751 gIOHibernateCurrentHeader->options, 32);
1752 }
1753 else
1754 {
1755 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1756 }
1757
1758 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1759 && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus))
1760 {
1761 IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
1762 &gIOHibernateGraphicsInfo->gfxStatus,
1763 sizeof(gIOHibernateGraphicsInfo->gfxStatus));
1764 }
1765 else
1766 {
1767 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1768 }
1769
1770
1771 if (vars->fileVars)
1772 {
1773 IOPolledFileClose(vars->fileVars);
1774 }
1775
1776 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1777
1778 #if defined(__i386__) || defined(__x86_64__)
1779 IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
1780 IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
1781
1782 /*
1783 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1784 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1785 */
1786 if (gIOOptionsEntry) {
1787
1788 if (gIOHibernateRTCVariablesKey) {
1789 if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
1790 gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
1791 }
1792 }
1793
1794 if (gIOHibernateBootNextKey)
1795 {
1796 if (gIOHibernateBootNextSave)
1797 {
1798 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
1799 gIOHibernateBootNextSave->release();
1800 gIOHibernateBootNextSave = NULL;
1801 }
1802 else
1803 gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
1804 }
1805 gIOOptionsEntry->sync();
1806 }
1807 #endif
1808
1809 if (vars->srcBuffer)
1810 vars->srcBuffer->release();
1811 if (vars->ioBuffer)
1812 vars->ioBuffer->release();
1813 bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
1814 if (vars->handoffBuffer && (kIOHibernateStateWakingFromHibernate == gIOHibernateState))
1815 {
1816 IOHibernateHandoff * handoff;
1817 bool done = false;
1818 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
1819 !done;
1820 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
1821 {
1822 HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
1823 uint8_t * data = &handoff->data[0];
1824 switch (handoff->type)
1825 {
1826 case kIOHibernateHandoffTypeEnd:
1827 done = true;
1828 break;
1829
1830 case kIOHibernateHandoffTypeDeviceTree:
1831 MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
1832 break;
1833
1834 case kIOHibernateHandoffTypeKeyStore:
1835 #if defined(__i386__) || defined(__x86_64__)
1836 {
1837 IOBufferMemoryDescriptor *
1838 md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
1839 if (md)
1840 {
1841 IOSetKeyStoreData(md);
1842 }
1843 }
1844 #endif
1845 break;
1846
1847 default:
1848 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
1849 break;
1850 }
1851 }
1852 vars->handoffBuffer->release();
1853 }
1854 if (vars->fileExtents)
1855 vars->fileExtents->release();
1856
1857 bzero(vars, sizeof(*vars));
1858
1859 // gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1860
1861 return (kIOReturnSuccess);
1862 }
1863
1864 IOReturn
1865 IOHibernateSystemPostWake(void)
1866 {
1867 struct kern_direct_file_io_ref_t * fileRef;
1868
1869 if (kFSOpened == gFSState)
1870 {
1871 // invalidate & close the image file
1872 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1873 if ((fileRef = gIOHibernateFileRef))
1874 {
1875 gIOHibernateFileRef = 0;
1876 kern_close_file_for_direct_io(fileRef,
1877 0, (caddr_t) gIOHibernateCurrentHeader,
1878 sizeof(IOHibernateImageHeader),
1879 sizeof(IOHibernateImageHeader),
1880 gIOHibernateCurrentHeader->imageSize);
1881 }
1882 gFSState = kFSIdle;
1883 }
1884 return (kIOReturnSuccess);
1885 }
1886
1887 bool IOHibernateWasScreenLocked(void)
1888 {
1889 bool ret = false;
1890 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry)
1891 {
1892 OSData *
1893 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
1894 if (data) switch (*((uint32_t *)data->getBytesNoCopy()))
1895 {
1896 case kIOScreenLockLocked:
1897 case kIOScreenLockFileVaultDialog:
1898 ret = true;
1899 break;
1900 case kIOScreenLockNoLock:
1901 case kIOScreenLockUnlocked:
1902 default:
1903 ret = false;
1904 break;
1905 }
1906 }
1907 return (ret);
1908 }
1909
1910 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1911
1912 SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
1913 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1914 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
1915 SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
1916 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1917 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
1918 SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
1919 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1920 &gIOHibernateMode, 0, "");
1921
1922 void
1923 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1924 {
1925 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
1926 if (data)
1927 {
1928 rootDomain->setProperty(kIOHibernateStateKey, data);
1929 data->release();
1930 }
1931
1932 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))
1933 gIOHibernateMode = kIOHibernateModeOn;
1934 else
1935 gIOHibernateFilename[0] = 0;
1936
1937 sysctl_register_oid(&sysctl__kern_hibernatefile);
1938 sysctl_register_oid(&sysctl__kern_bootsignature);
1939 sysctl_register_oid(&sysctl__kern_hibernatemode);
1940
1941 gFSLock = IOLockAlloc();
1942 }
1943
1944
1945 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1946
1947 static void
1948 hibernate_setup_for_wake(void)
1949 {
1950 }
1951
1952 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1953
1954 #define C_ASSERT(e) typedef char __C_ASSERT__[(e) ? 1 : -1]
1955
1956 static bool
1957 no_encrypt_page(vm_offset_t ppnum)
1958 {
1959 if (pmap_is_noencrypt((ppnum_t)ppnum) == TRUE)
1960 {
1961 return true;
1962 }
1963 return false;
1964 }
1965
1966 uint32_t wired_pages_encrypted = 0;
1967 uint32_t dirty_pages_encrypted = 0;
1968 uint32_t wired_pages_clear = 0;
1969
1970 static void
1971 hibernate_pal_callback(void *vars_arg, vm_offset_t addr)
1972 {
1973 IOHibernateVars *vars = (IOHibernateVars *)vars_arg;
1974 /* Make sure it's not in either of the save lists */
1975 hibernate_set_page_state(vars->page_list, vars->page_list_wired, atop_64(addr), 1, kIOHibernatePageStateFree);
1976
1977 /* Set it in the bitmap of pages owned by the PAL */
1978 hibernate_page_bitset(vars->page_list_pal, TRUE, atop_64(addr));
1979 }
1980
1981 static struct hibernate_cryptvars_t *local_cryptvars;
1982
1983 extern "C" int
1984 hibernate_pal_write(void *buffer, size_t size)
1985 {
1986 IOHibernateVars * vars = &gIOHibernateVars;
1987
1988 IOReturn err = IOPolledFileWrite(vars->fileVars, (const uint8_t *)buffer, size, local_cryptvars);
1989 if (kIOReturnSuccess != err) {
1990 kprintf("epic hibernate fail! %d\n", err);
1991 return err;
1992 }
1993
1994 return 0;
1995 }
1996
1997
1998 extern "C" uint32_t
1999 hibernate_write_image(void)
2000 {
2001 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
2002 IOHibernateVars * vars = &gIOHibernateVars;
2003 IOPolledFileExtent * fileExtents;
2004
2005 C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
2006
2007 uint32_t pageCount, pagesDone;
2008 IOReturn err;
2009 vm_offset_t ppnum, page;
2010 IOItemCount count;
2011 uint8_t * src;
2012 uint8_t * data;
2013 IOByteCount pageCompressedSize;
2014 uint64_t compressedSize, uncompressedSize;
2015 uint64_t image1Size = 0;
2016 uint32_t bitmap_size;
2017 bool iterDone, pollerOpen, needEncrypt;
2018 uint32_t restore1Sum, sum, sum1, sum2;
2019 uint32_t tag;
2020 uint32_t pageType;
2021 uint32_t pageAndCount[2];
2022 addr64_t phys64;
2023 IOByteCount segLen;
2024
2025 AbsoluteTime startTime, endTime;
2026 AbsoluteTime allTime, compTime;
2027 uint64_t compBytes;
2028 uint64_t nsec;
2029 uint32_t lastProgressStamp = 0;
2030 uint32_t progressStamp;
2031 uint32_t blob, lastBlob = (uint32_t) -1L;
2032
2033 hibernate_cryptvars_t _cryptvars;
2034 hibernate_cryptvars_t * cryptvars = 0;
2035
2036 wired_pages_encrypted = 0;
2037 dirty_pages_encrypted = 0;
2038 wired_pages_clear = 0;
2039
2040 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
2041 return (false /* sleep */ );
2042
2043 if (kIOHibernateModeSleep & gIOHibernateMode)
2044 kdebug_enable = save_kdebug_enable;
2045
2046 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START, 0, 0, 0, 0, 0);
2047 IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
2048
2049 restore1Sum = sum1 = sum2 = 0;
2050
2051 hibernate_pal_prepare();
2052
2053 #if CRYPTO
2054 // encryption data. "iv" is the "initial vector".
2055 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2056 {
2057 static const unsigned char first_iv[AES_BLOCK_SIZE]
2058 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
2059 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
2060
2061 cryptvars = &gIOHibernateCryptWakeContext;
2062 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
2063 aes_encrypt_key(vars->cryptKey,
2064 kIOHibernateAESKeySize,
2065 &cryptvars->ctx.encrypt);
2066 aes_decrypt_key(vars->cryptKey,
2067 kIOHibernateAESKeySize,
2068 &cryptvars->ctx.decrypt);
2069
2070 cryptvars = &_cryptvars;
2071 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
2072 for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++)
2073 vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
2074 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
2075 aes_encrypt_key(vars->wiredCryptKey,
2076 kIOHibernateAESKeySize,
2077 &cryptvars->ctx.encrypt);
2078
2079 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
2080 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
2081 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
2082
2083 local_cryptvars = cryptvars;
2084 }
2085 #endif /* CRYPTO */
2086
2087 hibernate_setup_for_wake();
2088
2089 hibernate_page_list_setall(vars->page_list,
2090 vars->page_list_wired,
2091 vars->page_list_pal,
2092 &pageCount);
2093
2094 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
2095
2096 fileExtents = (IOPolledFileExtent *) vars->fileExtents->getBytesNoCopy();
2097
2098 #if 0
2099 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
2100 for (page = 0; page < count; page++)
2101 {
2102 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
2103 fileExtents[page].start, fileExtents[page].length,
2104 fileExtents[page].start + fileExtents[page].length);
2105 }
2106 #endif
2107
2108 needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
2109 AbsoluteTime_to_scalar(&compTime) = 0;
2110 compBytes = 0;
2111
2112 clock_get_uptime(&allTime);
2113 IOService::getPMRootDomain()->pmStatsRecordEvent(
2114 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
2115
2116 do
2117 {
2118 compressedSize = 0;
2119 uncompressedSize = 0;
2120
2121 IOPolledFileSeek(vars->fileVars, sizeof(IOHibernateImageHeader));
2122
2123 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
2124 ml_get_interrupts_enabled());
2125 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledBeforeSleepState, vars->ioBuffer);
2126 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
2127 pollerOpen = (kIOReturnSuccess == err);
2128 if (!pollerOpen)
2129 break;
2130
2131 // copy file block extent list if larger than header
2132
2133 count = vars->fileExtents->getLength();
2134 if (count > sizeof(header->fileExtentMap))
2135 {
2136 count -= sizeof(header->fileExtentMap);
2137 err = IOPolledFileWrite(vars->fileVars,
2138 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
2139 if (kIOReturnSuccess != err)
2140 break;
2141 }
2142
2143 uintptr_t hibernateBase;
2144 uintptr_t hibernateEnd;
2145
2146 hibernateBase = HIB_BASE; /* Defined in PAL headers */
2147
2148 hibernateEnd = (segHIBB + segSizeHIB);
2149
2150 // copy out restore1 code
2151
2152 for (count = 0;
2153 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2154 count += segLen)
2155 {
2156 for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++)
2157 {
2158 gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
2159 }
2160 }
2161
2162 page = atop_32(kvtophys(hibernateBase));
2163 count = atop_32(round_page(hibernateEnd) - hibernateBase);
2164 header->restore1CodePhysPage = page;
2165 header->restore1CodeVirt = hibernateBase;
2166 header->restore1PageCount = count;
2167 header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
2168 header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
2169
2170 // sum __HIB seg, with zeros for the stack
2171 src = (uint8_t *) trunc_page(hibernateBase);
2172 for (page = 0; page < count; page++)
2173 {
2174 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
2175 restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
2176 else
2177 restore1Sum += 0x00000000;
2178 src += page_size;
2179 }
2180 sum1 = restore1Sum;
2181
2182 // write the __HIB seg, with zeros for the stack
2183
2184 src = (uint8_t *) trunc_page(hibernateBase);
2185 count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
2186 if (count)
2187 {
2188 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
2189 if (kIOReturnSuccess != err)
2190 break;
2191 }
2192 err = IOPolledFileWrite(vars->fileVars,
2193 (uint8_t *) 0,
2194 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
2195 cryptvars);
2196 if (kIOReturnSuccess != err)
2197 break;
2198 src = &gIOHibernateRestoreStackEnd[0];
2199 count = round_page(hibernateEnd) - ((uintptr_t) src);
2200 if (count)
2201 {
2202 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
2203 if (kIOReturnSuccess != err)
2204 break;
2205 }
2206
2207 vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
2208 vars->fileVars->encryptEnd = UINT64_MAX;
2209 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
2210
2211 // write the preview buffer
2212
2213 if (vars->previewBuffer)
2214 {
2215 ppnum = 0;
2216 count = 0;
2217 do
2218 {
2219 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
2220 pageAndCount[0] = atop_64(phys64);
2221 pageAndCount[1] = atop_32(segLen);
2222 err = IOPolledFileWrite(vars->fileVars,
2223 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
2224 cryptvars);
2225 if (kIOReturnSuccess != err)
2226 break;
2227 count += segLen;
2228 ppnum += sizeof(pageAndCount);
2229 }
2230 while (phys64);
2231 if (kIOReturnSuccess != err)
2232 break;
2233
2234 src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
2235
2236 ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
2237
2238 count = vars->previewBuffer->getLength();
2239
2240 header->previewPageListSize = ppnum;
2241 header->previewSize = count + ppnum;
2242
2243 for (page = 0; page < count; page += page_size)
2244 {
2245 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
2246 sum1 += hibernate_sum_page(src + page, atop_64(phys64));
2247 }
2248 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
2249 if (kIOReturnSuccess != err)
2250 break;
2251 }
2252
2253 // mark areas for no save
2254
2255 for (count = 0;
2256 (phys64 = vars->ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2257 count += segLen)
2258 {
2259 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2260 atop_64(phys64), atop_32(segLen),
2261 kIOHibernatePageStateFree);
2262 pageCount -= atop_32(segLen);
2263 }
2264
2265 for (count = 0;
2266 (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2267 count += segLen)
2268 {
2269 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2270 atop_64(phys64), atop_32(segLen),
2271 kIOHibernatePageStateFree);
2272 pageCount -= atop_32(segLen);
2273 }
2274
2275 // copy out bitmap of pages available for trashing during restore
2276
2277 bitmap_size = vars->page_list_wired->list_size;
2278 src = (uint8_t *) vars->page_list_wired;
2279 err = IOPolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
2280 if (kIOReturnSuccess != err)
2281 break;
2282
2283 // mark more areas for no save, but these are not available
2284 // for trashing during restore
2285
2286 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
2287
2288
2289 page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
2290 count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
2291 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2292 page, count,
2293 kIOHibernatePageStateFree);
2294 pageCount -= count;
2295
2296 if (vars->previewBuffer) for (count = 0;
2297 (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2298 count += segLen)
2299 {
2300 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2301 atop_64(phys64), atop_32(segLen),
2302 kIOHibernatePageStateFree);
2303 pageCount -= atop_32(segLen);
2304 }
2305
2306 for (count = 0;
2307 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2308 count += segLen)
2309 {
2310 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2311 atop_64(phys64), atop_32(segLen),
2312 kIOHibernatePageStateFree);
2313 pageCount -= atop_32(segLen);
2314 }
2315
2316 (void)hibernate_pal_callback;
2317
2318 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2319
2320 pagesDone = 0;
2321 lastBlob = 0;
2322
2323 HIBLOG("writing %d pages\n", pageCount);
2324
2325 enum
2326 // pageType
2327 {
2328 kWired = 0x02,
2329 kEncrypt = 0x01,
2330 kWiredEncrypt = kWired | kEncrypt,
2331 kWiredClear = kWired,
2332 kUnwiredEncrypt = kEncrypt
2333 };
2334
2335 for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)
2336 {
2337 if (kUnwiredEncrypt == pageType)
2338 {
2339 // start unwired image
2340 vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
2341 vars->fileVars->encryptEnd = UINT64_MAX;
2342 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
2343 bcopy(&cryptvars->aes_iv[0],
2344 &gIOHibernateCryptWakeContext.aes_iv[0],
2345 sizeof(cryptvars->aes_iv));
2346 cryptvars = &gIOHibernateCryptWakeContext;
2347 }
2348 for (iterDone = false, ppnum = 0; !iterDone; )
2349 {
2350 count = hibernate_page_list_iterate((kWired & pageType)
2351 ? vars->page_list_wired : vars->page_list,
2352 &ppnum);
2353 // kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
2354 iterDone = !count;
2355
2356 if (count && (kWired & pageType) && needEncrypt)
2357 {
2358 uint32_t checkIndex;
2359 for (checkIndex = 0;
2360 (checkIndex < count)
2361 && (((kEncrypt & pageType) == 0) == no_encrypt_page(ppnum + checkIndex));
2362 checkIndex++)
2363 {}
2364 if (!checkIndex)
2365 {
2366 ppnum++;
2367 continue;
2368 }
2369 count = checkIndex;
2370 }
2371
2372 switch (pageType)
2373 {
2374 case kWiredEncrypt: wired_pages_encrypted += count; break;
2375 case kWiredClear: wired_pages_clear += count; break;
2376 case kUnwiredEncrypt: dirty_pages_encrypted += count; break;
2377 }
2378
2379 if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */}
2380 else
2381 {
2382 pageAndCount[0] = ppnum;
2383 pageAndCount[1] = count;
2384 err = IOPolledFileWrite(vars->fileVars,
2385 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
2386 cryptvars);
2387 if (kIOReturnSuccess != err)
2388 break;
2389 }
2390
2391 for (page = ppnum; page < (ppnum + count); page++)
2392 {
2393 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
2394 if (err)
2395 {
2396 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
2397 break;
2398 }
2399
2400 sum = hibernate_sum_page(src, page);
2401 if (kWired & pageType)
2402 sum1 += sum;
2403 else
2404 sum2 += sum;
2405
2406 clock_get_uptime(&startTime);
2407
2408 pageCompressedSize = WKdm_compress ((WK_word*) src, (WK_word*) (src + page_size), PAGE_SIZE_IN_WORDS);
2409
2410 clock_get_uptime(&endTime);
2411 ADD_ABSOLUTETIME(&compTime, &endTime);
2412 SUB_ABSOLUTETIME(&compTime, &startTime);
2413 compBytes += page_size;
2414
2415 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2416 pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
2417
2418 if (pageCompressedSize > page_size)
2419 {
2420 // HIBLOG("------------lose: %d\n", pageCompressedSize);
2421 pageCompressedSize = page_size;
2422 }
2423
2424 if (pageCompressedSize != page_size)
2425 data = (src + page_size);
2426 else
2427 data = src;
2428
2429 tag = pageCompressedSize | kIOHibernateTagSignature;
2430 err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
2431 if (kIOReturnSuccess != err)
2432 break;
2433
2434 err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
2435 if (kIOReturnSuccess != err)
2436 break;
2437
2438 compressedSize += pageCompressedSize;
2439 if (pageCompressedSize)
2440 uncompressedSize += page_size;
2441 pagesDone++;
2442
2443 if (vars->consoleMapping && (0 == (1023 & pagesDone)))
2444 {
2445 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
2446 if (blob != lastBlob)
2447 {
2448 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
2449 lastBlob = blob;
2450 }
2451 }
2452 if (0 == (8191 & pagesDone))
2453 {
2454 clock_get_uptime(&endTime);
2455 SUB_ABSOLUTETIME(&endTime, &allTime);
2456 absolutetime_to_nanoseconds(endTime, &nsec);
2457 progressStamp = nsec / 750000000ULL;
2458 if (progressStamp != lastProgressStamp)
2459 {
2460 lastProgressStamp = progressStamp;
2461 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
2462 }
2463 }
2464 }
2465 if (kIOReturnSuccess != err)
2466 break;
2467 ppnum = page;
2468 }
2469
2470 if (kIOReturnSuccess != err)
2471 break;
2472
2473 if ((kEncrypt & pageType))
2474 {
2475 vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
2476 HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
2477 }
2478
2479 if (kWiredEncrypt != pageType)
2480 {
2481 // end of image1/2 - fill to next block
2482 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2483 if (kIOReturnSuccess != err)
2484 break;
2485 }
2486 if (kWiredClear == pageType)
2487 {
2488 // enlarge wired image for test
2489 // err = IOPolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
2490
2491 // end wired image
2492 header->encryptStart = vars->fileVars->encryptStart;
2493 header->encryptEnd = vars->fileVars->encryptEnd;
2494 image1Size = vars->fileVars->position;
2495 HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
2496 image1Size, header->encryptStart, header->encryptEnd);
2497 }
2498 }
2499 if (kIOReturnSuccess != err)
2500 break;
2501
2502 // Header:
2503
2504 header->imageSize = vars->fileVars->position;
2505 header->image1Size = image1Size;
2506 header->bitmapSize = bitmap_size;
2507 header->pageCount = pageCount;
2508
2509 header->restore1Sum = restore1Sum;
2510 header->image1Sum = sum1;
2511 header->image2Sum = sum2;
2512 header->sleepTime = gIOLastSleepTime.tv_sec;
2513
2514 count = vars->fileExtents->getLength();
2515 if (count > sizeof(header->fileExtentMap))
2516 {
2517 header->fileExtentMapSize = count;
2518 count = sizeof(header->fileExtentMap);
2519 }
2520 else
2521 header->fileExtentMapSize = sizeof(header->fileExtentMap);
2522 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
2523
2524 header->deviceBase = vars->fileVars->block0;
2525
2526 IOPolledFileSeek(vars->fileVars, 0);
2527 err = IOPolledFileWrite(vars->fileVars,
2528 (uint8_t *) header, sizeof(IOHibernateImageHeader),
2529 cryptvars);
2530 if (kIOReturnSuccess != err)
2531 break;
2532 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2533 if (kIOReturnSuccess != err)
2534 break;
2535 err = IOHibernatePollerIODone(vars->fileVars, true);
2536 if (kIOReturnSuccess != err)
2537 break;
2538 }
2539 while (false);
2540
2541 clock_get_uptime(&endTime);
2542
2543 IOService::getPMRootDomain()->pmStatsRecordEvent(
2544 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
2545
2546 SUB_ABSOLUTETIME(&endTime, &allTime);
2547 absolutetime_to_nanoseconds(endTime, &nsec);
2548 HIBLOG("all time: %qd ms, ",
2549 nsec / 1000000ULL);
2550
2551 absolutetime_to_nanoseconds(compTime, &nsec);
2552 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2553 compBytes,
2554 nsec / 1000000ULL,
2555 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2556
2557 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2558 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
2559 vars->fileVars->cryptBytes,
2560 nsec / 1000000ULL,
2561 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2562
2563 HIBLOG("\nimage %qd, uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
2564 header->imageSize,
2565 uncompressedSize, atop_32(uncompressedSize), compressedSize,
2566 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
2567 sum1, sum2);
2568
2569 HIBLOG("wired_pages_encrypted %d, wired_pages_clear %d, dirty_pages_encrypted %d\n",
2570 wired_pages_encrypted, wired_pages_clear, dirty_pages_encrypted);
2571
2572 if (vars->fileVars->io)
2573 (void) IOHibernatePollerIODone(vars->fileVars, false);
2574
2575 if (pollerOpen)
2576 IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
2577
2578 if (vars->consoleMapping)
2579 ProgressUpdate(gIOHibernateGraphicsInfo,
2580 vars->consoleMapping, 0, kIOHibernateProgressCount);
2581
2582 HIBLOG("hibernate_write_image done(%x)\n", err);
2583
2584 // should we come back via regular wake, set the state in memory.
2585 gIOHibernateState = kIOHibernateStateInactive;
2586
2587 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END,
2588 wired_pages_encrypted, wired_pages_clear, dirty_pages_encrypted, 0, 0);
2589
2590 if (kIOReturnSuccess == err)
2591 {
2592 if (kIOHibernateModeSleep & gIOHibernateMode)
2593 {
2594 return (kIOHibernatePostWriteSleep);
2595 }
2596 else if(kIOHibernateModeRestart & gIOHibernateMode)
2597 {
2598 return (kIOHibernatePostWriteRestart);
2599 }
2600 else
2601 {
2602 /* by default, power down */
2603 return (kIOHibernatePostWriteHalt);
2604 }
2605 }
2606 else if (kIOReturnAborted == err)
2607 {
2608 return (kIOHibernatePostWriteWake);
2609 }
2610 else
2611 {
2612 /* on error, sleep */
2613 return (kIOHibernatePostWriteSleep);
2614 }
2615 }
2616
2617 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2618
2619 extern "C" void
2620 hibernate_machine_init(void)
2621 {
2622 IOReturn err;
2623 uint32_t sum;
2624 uint32_t pagesDone;
2625 uint32_t pagesRead = 0;
2626 AbsoluteTime startTime, compTime;
2627 AbsoluteTime allTime, endTime;
2628 uint64_t compBytes;
2629 uint64_t nsec;
2630 uint32_t lastProgressStamp = 0;
2631 uint32_t progressStamp;
2632 hibernate_cryptvars_t * cryptvars = 0;
2633
2634 IOHibernateVars * vars = &gIOHibernateVars;
2635
2636 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
2637 return;
2638
2639 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2640 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2641
2642 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, image1Size %qx, conflictCount %d, nextFree %x\n",
2643 gIOHibernateState, pagesDone, sum, gIOHibernateCurrentHeader->image1Size,
2644 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
2645
2646 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
2647 {
2648 HIBLOG("regular wake\n");
2649 return;
2650 }
2651
2652 HIBPRINT("diag %x %x %x %x\n",
2653 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
2654 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
2655
2656 if ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)
2657 hibernate_page_list_discard(vars->page_list);
2658
2659 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
2660
2661 if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount)
2662 panic("handoff overflow");
2663
2664 IOHibernateHandoff * handoff;
2665 bool done = false;
2666 bool foundCryptData = false;
2667
2668 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
2669 !done;
2670 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
2671 {
2672 // HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
2673 uint8_t * data = &handoff->data[0];
2674 switch (handoff->type)
2675 {
2676 case kIOHibernateHandoffTypeEnd:
2677 done = true;
2678 break;
2679
2680 case kIOHibernateHandoffTypeGraphicsInfo:
2681 bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
2682 break;
2683
2684 case kIOHibernateHandoffTypeCryptVars:
2685 if (cryptvars)
2686 {
2687 hibernate_cryptwakevars_t *
2688 wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
2689 bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
2690 }
2691 foundCryptData = true;
2692 bzero(data, handoff->bytecount);
2693 break;
2694
2695 case kIOHibernateHandoffTypeMemoryMap:
2696 hibernate_newruntime_map(data, handoff->bytecount,
2697 gIOHibernateCurrentHeader->systemTableOffset);
2698 break;
2699
2700 case kIOHibernateHandoffTypeDeviceTree:
2701 {
2702 // DTEntry chosen = NULL;
2703 // HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen));
2704 }
2705 break;
2706
2707 default:
2708 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
2709 break;
2710 }
2711 }
2712 if (cryptvars && !foundCryptData)
2713 panic("hibernate handoff");
2714
2715 HIBPRINT("video %x %d %d %d status %x\n",
2716 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2717 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
2718
2719 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)
2720 {
2721 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2722 * gIOHibernateGraphicsInfo->rowBytes);
2723 IOMapPages(kernel_map,
2724 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2725 vars->videoMapSize, kIOMapInhibitCache );
2726 }
2727
2728 if (vars->videoMapSize)
2729 ProgressUpdate(gIOHibernateGraphicsInfo,
2730 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2731
2732 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2733 uint32_t decoOffset;
2734
2735 clock_get_uptime(&allTime);
2736 AbsoluteTime_to_scalar(&compTime) = 0;
2737 compBytes = 0;
2738
2739 HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2740 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0);
2741 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
2742
2743 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2744
2745 // kick off the read ahead
2746 vars->fileVars->io = false;
2747 vars->fileVars->bufferHalf = 0;
2748 vars->fileVars->bufferLimit = 0;
2749 vars->fileVars->lastRead = 0;
2750 vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
2751 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2752 vars->fileVars->cryptBytes = 0;
2753 AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
2754
2755 err = IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
2756 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2757 // --
2758
2759 HIBLOG("hibernate_machine_init reading\n");
2760
2761 uint32_t * header = (uint32_t *) src;
2762 sum = 0;
2763
2764 while (kIOReturnSuccess == err)
2765 {
2766 unsigned int count;
2767 unsigned int page;
2768 uint32_t tag;
2769 vm_offset_t ppnum, compressedSize;
2770
2771 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2772 if (kIOReturnSuccess != err)
2773 break;
2774
2775 ppnum = header[0];
2776 count = header[1];
2777
2778 // HIBPRINT("(%x, %x)\n", ppnum, count);
2779
2780 if (!count)
2781 break;
2782
2783 for (page = 0; page < count; page++)
2784 {
2785 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2786 if (kIOReturnSuccess != err)
2787 break;
2788
2789 compressedSize = kIOHibernateTagLength & tag;
2790 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength))
2791 {
2792 err = kIOReturnIPCError;
2793 break;
2794 }
2795
2796 if (!compressedSize)
2797 {
2798 ppnum++;
2799 pagesDone++;
2800 continue;
2801 }
2802
2803 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2804 if (kIOReturnSuccess != err)
2805 break;
2806
2807 if (compressedSize < page_size)
2808 {
2809 decoOffset = page_size;
2810
2811 clock_get_uptime(&startTime);
2812 WKdm_decompress((WK_word*) src, (WK_word*) (src + decoOffset), PAGE_SIZE_IN_WORDS);
2813 clock_get_uptime(&endTime);
2814 ADD_ABSOLUTETIME(&compTime, &endTime);
2815 SUB_ABSOLUTETIME(&compTime, &startTime);
2816
2817 compBytes += page_size;
2818 }
2819 else
2820 decoOffset = 0;
2821
2822 sum += hibernate_sum_page((src + decoOffset), ppnum);
2823
2824 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2825 if (err)
2826 {
2827 HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
2828 break;
2829 }
2830
2831 ppnum++;
2832 pagesDone++;
2833 pagesRead++;
2834
2835 if (0 == (8191 & pagesDone))
2836 {
2837 clock_get_uptime(&endTime);
2838 SUB_ABSOLUTETIME(&endTime, &allTime);
2839 absolutetime_to_nanoseconds(endTime, &nsec);
2840 progressStamp = nsec / 750000000ULL;
2841 if (progressStamp != lastProgressStamp)
2842 {
2843 lastProgressStamp = progressStamp;
2844 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2845 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2846 }
2847 }
2848 }
2849 }
2850 if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages))
2851 err = kIOReturnLockedRead;
2852
2853 if (kIOReturnSuccess != err)
2854 panic("Hibernate restore error %x", err);
2855
2856 gIOHibernateCurrentHeader->actualImage2Sum = sum;
2857
2858 if (vars->fileVars->io)
2859 (void) IOHibernatePollerIODone(vars->fileVars, false);
2860
2861 err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
2862
2863 clock_get_uptime(&endTime);
2864
2865 IOService::getPMRootDomain()->pmStatsRecordEvent(
2866 kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
2867 IOService::getPMRootDomain()->pmStatsRecordEvent(
2868 kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
2869
2870 SUB_ABSOLUTETIME(&endTime, &allTime);
2871 absolutetime_to_nanoseconds(endTime, &nsec);
2872
2873 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %qd ms, ",
2874 pagesDone, sum, nsec / 1000000ULL);
2875
2876 absolutetime_to_nanoseconds(compTime, &nsec);
2877 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2878 compBytes,
2879 nsec / 1000000ULL,
2880 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2881
2882 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2883 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
2884 vars->fileVars->cryptBytes,
2885 nsec / 1000000ULL,
2886 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2887
2888 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_NONE, pagesRead, pagesDone, 0, 0, 0);
2889 }
2890
2891 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
mirror of XNU