]> git.saurik.com Git - apple/xnu.git/blame - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-2050.22.13.tar.gz
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
CommitLineData
3a60a9f5 1/*
b0d623f7 2 * Copyright (c) 2004-2008 Apple Computer, Inc. All rights reserved.
3a60a9f5 3 *
2d21ac55 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
3a60a9f5 5 *
2d21ac55
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
8f6c56a5 14 *
2d21ac55
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5
A		 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55
A		 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
8f6c56a5 25 *
2d21ac55 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
3a60a9f5
A		 27 */
28
29
30/*
31
32Sleep:
33
34- PMRootDomain calls IOHibernateSystemSleep() before system sleep
35(devices awake, normal execution context)
36- IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
37 grabs its extents and searches for a polling driver willing to work with that IOMedia.
38 The BSD code makes an ioctl to the storage driver to get the partition base offset to
39 the disk, and other ioctls to get the transfer constraints
40 If successful, the file is written to make sure its initially not bootable (in case of
41 later failure) and nvram set to point to the first block of the file. (Has to be done
42 here so blocking is possible in nvram support).
43 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
44 page out any pages it wants to (currently zero, but probably some percentage of memory).
45 Its assumed just allocating pages will cause the VM system to naturally select the best
46 pages for eviction. It also copies processor flags needed for the restore path and sets
47 a flag in the boot processor proc info.
48 gIOHibernateState = kIOHibernateStateHibernating.
49- Regular sleep progresses - some drivers may inspect the root domain property
50 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
51 as usual but leaves motherboard I/O on.
52- Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
53 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
54 all ppc RC bits out of the hash table and caches into the mapping structures.
55- hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
56 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
57 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
58 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
59 The image header and block list are written. The header includes the second file extent so
60 only the header block is needed to read the file, regardless of filesystem.
316670eb 61 The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
3a60a9f5
A		 62 (only) is used to decompress the image during wake/boot.
63 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
64 The bitmaps are written to the image.
65 More areas are removed from the bitmaps (after they have been written to the image) - the
316670eb 66 segment "__HIB" pages and interrupt stack.
3a60a9f5
A		 67 Each wired page is compressed and written and then each non-wired page. Compression and
68 disk writes are in parallel.
69 The image header is written to the start of the file and the polling driver closed.
70 The machine powers down (or sleeps).
71
72Boot/Wake:
73
74- BootX sees the boot-image nvram variable containing the device and block number of the image,
75 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
76- BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
77 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
78 that is in the kernel's __HIB section.
79- BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
80 hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
81 only code & data in that section is safe to call since all the other wired pages are still
82 compressed in the image.
83- hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
84 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
85 location directly, and copies those that can't to interim free pages. When the image has been
86 completed, the copies are uncompressed, overwriting the wired image pages.
87 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
88 is used to get pages into place for 64bit.
89- the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
90 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
91 are removed from the software strutures, and the hash table is reinitialized.
92- After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
93 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
94 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
95 for the remaining non wired pages.
96- The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
97 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
98 is closed via bsd.
99
100Polled Mode I/O:
101
102IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
103registry, specifying an object of calls IOPolledInterface.
104
105Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
106partition) that the image is going to live, looking for polled interface properties. If it finds
107one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
108interfaces found are kept in an ordered list.
109
110There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
111few different contexts things happen in:
112
113- there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
114up and running) and after wake - this is safe to allocate memory and do anything. The device
115ignores sleep requests from that point since its a waste of time if it goes to sleep and
116immediately wakes back up for the image write.
117
118- there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
119immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
120from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
121used to flush and set the disk to sleep.
122
123- there is an Open/Close (AfterSleep) pair made around the image read operations that happen
124immediately after sleep. These can't block or allocate memory. This is happening after the platform
125expert has woken the low level bits of the system, but most of the I/O system has not. There is only
126one thread running.
127
128For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
129(prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
130the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
131Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
132that is called for the hardware to check for events, and complete the I/O via the callback.
133The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
134to restrict I/O ops.
135*/
136
137#include <sys/systm.h>
138
139#include <IOKit/IOWorkLoop.h>
140#include <IOKit/IOCommandGate.h>
141#include <IOKit/IOTimerEventSource.h>
142#include <IOKit/IOPlatformExpert.h>
143#include <IOKit/IOKitDebug.h>
144#include <IOKit/IOTimeStamp.h>
145#include <IOKit/pwr_mgt/RootDomain.h>
146#include <IOKit/pwr_mgt/IOPMPrivate.h>
147#include <IOKit/IOMessage.h>
148#include <IOKit/IODeviceTreeSupport.h>
149#include <IOKit/IOBSD.h>
150#include "RootDomainUserClient.h"
151#include <IOKit/pwr_mgt/IOPowerConnection.h>
152#include "IOPMPowerStateQueue.h"
153#include <IOKit/IOBufferMemoryDescriptor.h>
6d2010ae 154#include <IOKit/AppleKeyStoreInterface.h>
316670eb 155#include <libkern/crypto/aes.h>
3a60a9f5
A		 156
157#include <sys/uio.h>
158#include <sys/conf.h>
159#include <sys/stat.h>
160#include <sys/fcntl.h> // (FWRITE, ...)
3a60a9f5 161#include <sys/sysctl.h>
6d2010ae 162#include <sys/kdebug.h>
3a60a9f5
A		 163
164#include <IOKit/IOHibernatePrivate.h>
165#include <IOKit/IOPolledInterface.h>
166#include <IOKit/IONVRAM.h>
167#include "IOHibernateInternal.h"
6d2010ae 168#include <libkern/WKdm.h>
3a60a9f5 169#include "IOKitKernelInternal.h"
6d2010ae
A		 170#include <pexpert/device_tree.h>
171
172#include <machine/pal_routines.h>
173#include <machine/pal_hibernate.h>
174
175extern "C" addr64_t kvtophys(vm_offset_t va);
99c3a104 176extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
3a60a9f5
A		 177
178/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
179
0b4c1975 180extern unsigned int save_kdebug_enable;
3a60a9f5
A		 181extern uint32_t gIOHibernateState;
182uint32_t gIOHibernateMode;
183static char gIOHibernateBootSignature[256+1];
184static char gIOHibernateFilename[MAXPATHLEN+1];
db609669
A		 185static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
186uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
187static uint64_t gIOHibernateCompression = 0x80; // default compression 50%
3a60a9f5
A		 188
189static IODTNVRAM * gIOOptionsEntry;
190static IORegistryEntry * gIOChosenEntry;
b0d623f7 191#if defined(__i386__) || defined(__x86_64__)
0c530ab8 192static const OSSymbol * gIOCreateEFIDevicePathSymbol;
060df5ea
A		 193static const OSSymbol * gIOHibernateRTCVariablesKey;
194static const OSSymbol * gIOHibernateBoot0082Key;
195static const OSSymbol * gIOHibernateBootNextKey;
196static OSData * gIOHibernateBoot0082Data;
197static OSData * gIOHibernateBootNextData;
198static OSObject * gIOHibernateBootNextSave;
0c530ab8 199#endif
3a60a9f5 200
316670eb
A		 201static IOLock * gFSLock;
202static uint32_t gFSState;
3a60a9f5
A		 203static IOPolledFileIOVars gFileVars;
204static IOHibernateVars gIOHibernateVars;
205static struct kern_direct_file_io_ref_t * gIOHibernateFileRef;
206static hibernate_cryptvars_t gIOHibernateCryptWakeContext;
6d2010ae
A		 207static hibernate_graphics_t _hibernateGraphics;
208static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
3a60a9f5 209
316670eb
A		 210enum
211{
212 kFSIdle = 0,
213 kFSOpening = 2,
214 kFSOpened = 3,
215 kFSTimedOut = 4,
216};
217
218static IOReturn IOHibernateDone(IOHibernateVars * vars);
219
3a60a9f5
A		 220/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
221
222enum { kXPRamAudioVolume = 8 };
223enum { kDefaultIOSize = 128 * 1024 };
224enum { kVideoMapSize = 32 * 1024 * 1024 };
225
226#ifndef kIOMediaPreferredBlockSizeKey
227#define kIOMediaPreferredBlockSizeKey "Preferred Block Size"
228#endif
229
230#ifndef kIOBootPathKey
231#define kIOBootPathKey "bootpath"
232#endif
233#ifndef kIOSelectedBootDeviceKey
234#define kIOSelectedBootDeviceKey "boot-device"
235#endif
236
237
238enum { kIOHibernateMinPollersNeeded = 2 };
239
240/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
241
242// copy from phys addr to MD
243
244static IOReturn
245IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
246 IOByteCount offset, addr64_t bytes, IOByteCount length)
247{
248 addr64_t srcAddr = bytes;
249 IOByteCount remaining;
250
251 remaining = length = min(length, md->getLength() - offset);
252 while (remaining) { // (process another target segment?)
253 addr64_t dstAddr64;
254 IOByteCount dstLen;
255
b0d623f7 256 dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
3a60a9f5
A		 257 if (!dstAddr64)
258 break;
259
260 // Clip segment length to remaining
261 if (dstLen > remaining)
262 dstLen = remaining;
263
264#if 1
265 bcopy_phys(srcAddr, dstAddr64, dstLen);
266#else
267 copypv(srcAddr, dstAddr64, dstLen,
268 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
269#endif
270 srcAddr += dstLen;
271 offset += dstLen;
272 remaining -= dstLen;
273 }
274
275 assert(!remaining);
276
277 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
278}
279
280// copy from MD to phys addr
281
282static IOReturn
283IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
284 IOByteCount offset, addr64_t bytes, IOByteCount length)
285{
286 addr64_t dstAddr = bytes;
287 IOByteCount remaining;
288
289 remaining = length = min(length, md->getLength() - offset);
290 while (remaining) { // (process another target segment?)
291 addr64_t srcAddr64;
292 IOByteCount dstLen;
293
b0d623f7 294 srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
3a60a9f5
A		 295 if (!srcAddr64)
296 break;
297
298 // Clip segment length to remaining
299 if (dstLen > remaining)
300 dstLen = remaining;
301
302#if 1
303 bcopy_phys(srcAddr64, dstAddr, dstLen);
304#else
305 copypv(srcAddr, dstAddr64, dstLen,
306 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
307#endif
308 dstAddr += dstLen;
309 offset += dstLen;
310 remaining -= dstLen;
311 }
312
313 assert(!remaining);
314
315 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
316}
317
318/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
319
320void
321hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
322 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
323{
324 count += ppnum;
325 switch (kind)
326 {
327 case kIOHibernatePageStateUnwiredSave:
328 // unwired save
329 for (; ppnum < count; ppnum++)
330 {
331 hibernate_page_bitset(page_list, FALSE, ppnum);
332 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
333 }
334 break;
335 case kIOHibernatePageStateWiredSave:
336 // wired save
337 for (; ppnum < count; ppnum++)
338 {
339 hibernate_page_bitset(page_list, FALSE, ppnum);
340 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
341 }
342 break;
343 case kIOHibernatePageStateFree:
344 // free page
345 for (; ppnum < count; ppnum++)
346 {
347 hibernate_page_bitset(page_list, TRUE, ppnum);
348 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
349 }
350 break;
351 default:
352 panic("hibernate_set_page_state");
353 }
354}
355
356static vm_offset_t
0c530ab8 357hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
3a60a9f5 358{
0c530ab8
A		 359 uint32_t page = *pPage;
360 uint32_t count;
361 hibernate_bitmap_t * bitmap;
21362eb3 362
0c530ab8
A		 363 while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
364 {
365 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
366 if (!count)
367 break;
368 page += count;
369 if (page <= bitmap->last_page)
370 break;
371 }
6601e61a 372
0c530ab8
A		 373 *pPage = page;
374 if (bitmap)
375 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
376 else
377 count = 0;
3a60a9f5
A		 378
379 return (count);
380}
381
382/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
383
384static IOReturn
385IOHibernatePollerProbe(IOPolledFileIOVars * vars, IOService * target)
386{
387 IOReturn err = kIOReturnError;
388 int32_t idx;
389 IOPolledInterface * poller;
390
391 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
392 {
393 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
394 err = poller->probe(target);
395 if (err)
396 {
397 HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
398 break;
399 }
400 }
401
402 return (err);
403}
404
405static IOReturn
406IOHibernatePollerOpen(IOPolledFileIOVars * vars, uint32_t state, IOMemoryDescriptor * md)
407{
408 IOReturn err = kIOReturnError;
409 int32_t idx;
410 IOPolledInterface * poller;
411
412 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
413 {
414 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
415 err = poller->open(state, md);
416 if (err)
417 {
418 HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
419 break;
420 }
421 }
422
423 return (err);
424}
425
426static IOReturn
427IOHibernatePollerClose(IOPolledFileIOVars * vars, uint32_t state)
428{
429 IOReturn err = kIOReturnError;
430 int32_t idx;
431 IOPolledInterface * poller;
432
433 for (idx = 0;
434 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
435 idx++)
436 {
437 err = poller->close(state);
438 if (err)
439 HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
440 }
441
442 return (err);
443}
444
445static void
446IOHibernatePollerIOComplete(void * target,
447 void * parameter,
448 IOReturn status,
449 UInt64 actualByteCount)
450{
451 IOPolledFileIOVars * vars = (IOPolledFileIOVars *) parameter;
452
453 vars->ioStatus = status;
454}
455
456static IOReturn
457IOHibernatePollerIO(IOPolledFileIOVars * vars,
458 uint32_t operation, uint32_t bufferOffset,
459 uint64_t deviceOffset, uint64_t length)
460{
461
462 IOReturn err = kIOReturnError;
463 IOPolledInterface * poller;
464 IOPolledCompletion completion;
465
466 completion.target = 0;
467 completion.action = &IOHibernatePollerIOComplete;
468 completion.parameter = vars;
469
470 vars->ioStatus = -1;
471
472 poller = (IOPolledInterface *) vars->pollers->getObject(0);
473 err = poller->startIO(operation, bufferOffset, deviceOffset + vars->block0, length, completion);
474 if (err)
475 HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
476
477 return (err);
478}
479
480static IOReturn
2d21ac55 481IOHibernatePollerIODone(IOPolledFileIOVars * vars, bool abortable)
3a60a9f5 482{
2d21ac55
A		 483 IOReturn err = kIOReturnSuccess;
484 int32_t idx = 0;
3a60a9f5
A		 485 IOPolledInterface * poller;
486
487 while (-1 == vars->ioStatus)
488 {
2d21ac55
A		 489 for (idx = 0;
490 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
3a60a9f5
A		 491 idx++)
492 {
2d21ac55
A		 493 IOReturn newErr;
494 newErr = poller->checkForWork();
495 if ((newErr == kIOReturnAborted) && !abortable)
496 newErr = kIOReturnSuccess;
497 if (kIOReturnSuccess == err)
498 err = newErr;
3a60a9f5
A		 499 }
500 }
501
0b4c1975
A		 502 if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort())
503 {
504 err = kIOReturnAborted;
505 HIBLOG("IOPolledInterface::checkForWork sw abort\n");
506 }
507
2d21ac55
A		 508 if (err)
509 {
510 HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
511 }
512 else
513 {
514 err = vars->ioStatus;
515 if (kIOReturnSuccess != err)
516 HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
517 }
3a60a9f5 518
2d21ac55 519 return (err);
3a60a9f5
A		 520}
521
522IOReturn
523IOPolledInterface::checkAllForWork(void)
524{
525 IOReturn err = kIOReturnNotReady;
526 int32_t idx;
527 IOPolledInterface * poller;
528
529 IOHibernateVars * vars = &gIOHibernateVars;
530
531 if (!vars->fileVars || !vars->fileVars->pollers)
532 return (err);
533
534 for (idx = 0;
535 (poller = (IOPolledInterface *) vars->fileVars->pollers->getObject(idx));
536 idx++)
537 {
538 err = poller->checkForWork();
539 if (err)
540 HIBLOG("IOPolledInterface::checkAllForWork[%d] 0x%x\n", idx, err);
541 }
542
543 return (err);
544}
545
546/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
547
548struct _OpenFileContext
549{
550 OSData * extents;
551 uint64_t size;
552};
553
554static void
555file_extent_callback(void * ref, uint64_t start, uint64_t length)
556{
557 _OpenFileContext * ctx = (_OpenFileContext *) ref;
558 IOPolledFileExtent extent;
559
560 extent.start = start;
561 extent.length = length;
562
563 ctx->extents->appendBytes(&extent, sizeof(extent));
564 ctx->size += length;
565}
566
6d2010ae
A		 567static IOService *
568IOCopyMediaForDev(dev_t device)
569{
570 OSDictionary * matching;
571 OSNumber * num;
572 OSIterator * iter;
573 IOService * result = 0;
574
575 matching = IOService::serviceMatching("IOMedia");
576 if (!matching)
577 return (0);
578 do
579 {
580 num = OSNumber::withNumber(major(device), 32);
581 if (!num)
582 break;
583 matching->setObject(kIOBSDMajorKey, num);
584 num->release();
585 num = OSNumber::withNumber(minor(device), 32);
586 if (!num)
587 break;
588 matching->setObject(kIOBSDMinorKey, num);
589 num->release();
590 if (!num)
591 break;
592 iter = IOService::getMatchingServices(matching);
593 if (iter)
594 {
595 result = (IOService *) iter->getNextObject();
596 result->retain();
597 iter->release();
598 }
599 }
600 while (false);
601 matching->release();
602
603 return (result);
604}
605
3a60a9f5 606IOReturn
db609669
A		 607IOPolledFileOpen( const char * filename, uint64_t setFileSize,
608 IOBufferMemoryDescriptor * ioBuffer,
609 IOPolledFileIOVars ** fileVars, OSData ** fileExtents,
610 OSData ** imagePath, uint8_t * volumeCryptKey)
3a60a9f5 611{
316670eb 612 IOReturn err = kIOReturnSuccess;
3a60a9f5
A		 613 IOPolledFileIOVars * vars;
614 _OpenFileContext ctx;
615 OSData * extentsData;
616 OSNumber * num;
6d2010ae
A		 617 IOService * part = 0;
618 OSString * keyUUID = 0;
619 OSString * keyStoreUUID = 0;
620 dev_t block_dev;
3a60a9f5
A		 621 dev_t hibernate_image_dev;
622 uint64_t maxiobytes;
316670eb
A		 623 AbsoluteTime startTime, endTime;
624 uint64_t nsec;
625
626 vars = IONew(IOPolledFileIOVars, 1);
627 if (!vars) return (kIOReturnNoMemory);
628 bzero(vars, sizeof(*vars));
3a60a9f5 629
3a60a9f5
A		 630 do
631 {
632 HIBLOG("sizeof(IOHibernateImageHeader) == %ld\n", sizeof(IOHibernateImageHeader));
633 if (sizeof(IOHibernateImageHeader) != 512)
634 continue;
635
636 vars->io = false;
637 vars->buffer = (uint8_t *) ioBuffer->getBytesNoCopy();
638 vars->bufferHalf = 0;
639 vars->bufferOffset = 0;
640 vars->bufferSize = ioBuffer->getLength() >> 1;
641
642 extentsData = OSData::withCapacity(32);
316670eb 643 ctx.extents = extentsData;
3a60a9f5 644 ctx.size = 0;
316670eb 645 clock_get_uptime(&startTime);
3a60a9f5
A		 646 vars->fileRef = kern_open_file_for_direct_io(filename,
647 &file_extent_callback, &ctx,
db609669
A		 648 setFileSize,
649 // write file:
650 0, (caddr_t) gIOHibernateCurrentHeader,
651 sizeof(IOHibernateImageHeader),
652 // results
6d2010ae 653 &block_dev,
3a60a9f5
A		 654 &hibernate_image_dev,
655 &vars->block0,
0b4c1975 656 &maxiobytes,
db609669 657 &vars->flags);
316670eb
A		 658#if 0
659 uint32_t msDelay = (131071 & random());
660 HIBLOG("sleep %d\n", msDelay);
661 IOSleep(msDelay);
662#endif
663 clock_get_uptime(&endTime);
664 SUB_ABSOLUTETIME(&endTime, &startTime);
665 absolutetime_to_nanoseconds(endTime, &nsec);
666
667 if (!vars->fileRef) err = kIOReturnNoSpace;
668
669 IOLockLock(gFSLock);
670 if (kFSOpening != gFSState) err = kIOReturnTimeout;
671 IOLockUnlock(gFSLock);
672
673 HIBLOG("kern_open_file_for_direct_io(%d) took %qd ms\n", err, nsec / 1000000ULL);
674 if (kIOReturnSuccess != err) break;
0b4c1975
A		 675
676 if (kIOHibernateModeSSDInvert & gIOHibernateMode)
6d2010ae 677 vars->flags ^= kIOHibernateOptionSSD;
0b4c1975
A		 678
679 HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx ssd %d\n", filename, ctx.size,
6d2010ae 680 vars->block0, maxiobytes, kIOHibernateOptionSSD & vars->flags);
3a60a9f5
A		 681 if (ctx.size < 1*1024*1024) // check against image size estimate!
682 {
683 err = kIOReturnNoSpace;
684 break;
685 }
686
db609669 687 vars->fileSize = ctx.size;
3a60a9f5
A		 688 if (maxiobytes < vars->bufferSize)
689 vars->bufferSize = maxiobytes;
690
691 vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
2d21ac55 692
6d2010ae
A		 693 part = IOCopyMediaForDev(block_dev);
694 if (!part)
695 break;
696
697 err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false,
698 (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
699 if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID)
700 {
701// IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
702 uuid_t volumeKeyUUID;
703 aks_volume_key_t vek;
704 static IOService * sKeyStore;
705 static const OSSymbol * sAKSGetKey;
706
707 if (!sAKSGetKey)
708 sAKSGetKey = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY);
709 if (!sKeyStore)
710 sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
711 if (sKeyStore)
712 err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
713 else
714 err = kIOReturnNoResources;
715 if (kIOReturnSuccess == err)
716 err = sKeyStore->callPlatformFunction(sAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
717 if (kIOReturnSuccess != err)
718 IOLog("volume key err 0x%x\n", err);
719 else
720 {
721 size_t bytes = (kIOHibernateAESKeySize / 8);
722 if (vek.key.keybytecount < bytes)
723 bytes = vek.key.keybytecount;
724 bcopy(&vek.key.keybytes[0], volumeCryptKey, bytes);
725 }
726 bzero(&vek, sizeof(vek));
727 }
728 part->release();
729
730 part = IOCopyMediaForDev(hibernate_image_dev);
731 if (!part)
732 break;
733
3a60a9f5
A		 734 IORegistryEntry * next;
735 IORegistryEntry * child;
736 OSData * data;
737
3a60a9f5
A		 738 vars->pollers = OSArray::withCapacity(4);
739 if (!vars->pollers)
740 break;
741
742 vars->blockSize = 512;
743 next = part;
744 do
745 {
746 IOPolledInterface * poller;
c0fea474
A		 747 OSObject * obj;
748
749 obj = next->getProperty(kIOPolledInterfaceSupportKey);
750 if (kOSBooleanFalse == obj)
751 {
752 vars->pollers->flushCollection();
753 break;
754 }
755 else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
3a60a9f5
A		 756 vars->pollers->setObject(poller);
757 if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey))))
758 vars->blockSize = num->unsigned32BitValue();
759 child = next;
760 }
761 while ((next = child->getParentEntry(gIOServicePlane))
762 && child->isParent(next, gIOServicePlane, true));
763
764 HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
6d2010ae 765 major(hibernate_image_dev), minor(hibernate_image_dev), (long)vars->blockSize, vars->pollers->getCount());
3a60a9f5
A		 766 if (vars->pollers->getCount() < kIOHibernateMinPollersNeeded)
767 continue;
768
769 err = IOHibernatePollerProbe(vars, (IOService *) part);
770 if (kIOReturnSuccess != err)
771 break;
772
773 err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer);
774 if (kIOReturnSuccess != err)
775 break;
776
777 *fileVars = vars;
778 *fileExtents = extentsData;
779
780 // make imagePath
3a60a9f5 781
0c530ab8 782 if ((extentsData->getLength() >= sizeof(IOPolledFileExtent)))
3a60a9f5 783 {
6d2010ae 784 char str2[24 + sizeof(uuid_string_t) + 2];
0c530ab8 785
b0d623f7 786#if defined(__i386__) || defined(__x86_64__)
0c530ab8
A		 787 if (!gIOCreateEFIDevicePathSymbol)
788 gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
789
6d2010ae
A		 790 if (keyUUID)
791 snprintf(str2, sizeof(str2), "%qx:%s",
792 vars->extentMap[0].start, keyUUID->getCStringNoCopy());
793 else
794 snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
0c530ab8
A		 795
796 err = IOService::getPlatform()->callPlatformFunction(
797 gIOCreateEFIDevicePathSymbol, false,
6d2010ae
A		 798 (void *) part, (void *) str2,
799 (void *) (uintptr_t) true, (void *) &data);
0c530ab8
A		 800#else
801 char str1[256];
802 int len = sizeof(str1);
803
804 if (!part->getPath(str1, &len, gIODTPlane))
805 err = kIOReturnNotFound;
806 else
807 {
2d21ac55 808 snprintf(str2, sizeof(str2), ",%qx", vars->extentMap[0].start);
0c530ab8
A		 809 // (strip the plane name)
810 char * tail = strchr(str1, ':');
811 if (!tail)
812 tail = str1 - 1;
813 data = OSData::withBytes(tail + 1, strlen(tail + 1));
814 data->appendBytes(str2, strlen(str2));
815 }
816#endif
817 if (kIOReturnSuccess == err)
3a60a9f5 818 *imagePath = data;
0c530ab8
A		 819 else
820 HIBLOG("error 0x%x getting path\n", err);
3a60a9f5
A		 821 }
822 }
823 while (false);
824
825 if (kIOReturnSuccess != err)
826 {
827 HIBLOG("error 0x%x opening hibernation file\n", err);
828 if (vars->fileRef)
b0d623f7 829 {
7ddcb079 830 kern_close_file_for_direct_io(vars->fileRef, 0, 0, 0, 0, 0);
316670eb 831 vars->fileRef = NULL;
b0d623f7 832 }
3a60a9f5
A		 833 }
834
835 if (part)
836 part->release();
837
838 return (err);
839}
840
841IOReturn
842IOPolledFileClose( IOPolledFileIOVars * vars )
843{
844 if (vars->pollers)
845 {
846 IOHibernatePollerClose(vars, kIOPolledPostflightState);
847 vars->pollers->release();
848 }
849
3a60a9f5
A		 850 bzero(vars, sizeof(IOPolledFileIOVars));
851
852 return (kIOReturnSuccess);
853}
854
855static IOReturn
856IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
857{
858 IOPolledFileExtent * extentMap;
859
860 extentMap = vars->extentMap;
861
862 vars->position = position;
863
864 while (position >= extentMap->length)
865 {
866 position -= extentMap->length;
867 extentMap++;
868 }
869
870 vars->currentExtent = extentMap;
871 vars->extentRemaining = extentMap->length - position;
872 vars->extentPosition = vars->position - position;
873
874 if (vars->bufferSize <= vars->extentRemaining)
875 vars->bufferLimit = vars->bufferSize;
876 else
877 vars->bufferLimit = vars->extentRemaining;
878
879 return (kIOReturnSuccess);
880}
881
882static IOReturn
883IOPolledFileWrite(IOPolledFileIOVars * vars,
884 const uint8_t * bytes, IOByteCount size,
885 hibernate_cryptvars_t * cryptvars)
886{
887 IOReturn err = kIOReturnSuccess;
888 IOByteCount copy;
889 bool flush = false;
890
891 do
892 {
893 if (!bytes && !size)
894 {
895 // seek to end of block & flush
896 size = vars->position & (vars->blockSize - 1);
897 if (size)
898 size = vars->blockSize - size;
899 flush = true;
900 // use some garbage for the fill
901 bytes = vars->buffer + vars->bufferOffset;
902 }
903
904 copy = vars->bufferLimit - vars->bufferOffset;
905 if (copy > size)
906 copy = size;
907 else
908 flush = true;
909
910 if (bytes)
911 {
912 bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
913 bytes += copy;
914 }
915 else
916 bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
917
918 size -= copy;
919 vars->bufferOffset += copy;
920 vars->position += copy;
921
922 if (flush && vars->bufferOffset)
923 {
924 uint64_t offset = (vars->position - vars->bufferOffset
925 - vars->extentPosition + vars->currentExtent->start);
926 uint32_t length = (vars->bufferOffset);
927
2d21ac55 928#if CRYPTO
0b4c1975
A		 929 if (cryptvars && vars->encryptStart
930 && (vars->position > vars->encryptStart)
931 && ((vars->position - length) < vars->encryptEnd))
3a60a9f5 932 {
6d2010ae
A		 933 AbsoluteTime startTime, endTime;
934
7ddcb079 935 uint64_t encryptLen, encryptStart;
3a60a9f5
A		 936 encryptLen = vars->position - vars->encryptStart;
937 if (encryptLen > length)
938 encryptLen = length;
939 encryptStart = length - encryptLen;
0b4c1975
A		 940 if (vars->position > vars->encryptEnd)
941 encryptLen -= (vars->position - vars->encryptEnd);
942
6d2010ae
A		 943 clock_get_uptime(&startTime);
944
3a60a9f5
A		 945 // encrypt the buffer
946 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
947 &cryptvars->aes_iv[0],
948 encryptLen / AES_BLOCK_SIZE,
949 vars->buffer + vars->bufferHalf + encryptStart,
950 &cryptvars->ctx.encrypt);
6d2010ae
A		 951
952 clock_get_uptime(&endTime);
953 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
954 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
955 vars->cryptBytes += encryptLen;
956
3a60a9f5
A		 957 // save initial vector for following encrypts
958 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
959 &cryptvars->aes_iv[0],
960 AES_BLOCK_SIZE);
961 }
2d21ac55 962#endif /* CRYPTO */
3a60a9f5
A		 963
964 if (vars->io)
965 {
2d21ac55 966 err = IOHibernatePollerIODone(vars, true);
3a60a9f5
A		 967 if (kIOReturnSuccess != err)
968 break;
969 }
970
971if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
972//if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
973
974 err = IOHibernatePollerIO(vars, kIOPolledWrite, vars->bufferHalf, offset, length);
975 if (kIOReturnSuccess != err)
976 break;
977 vars->io = true;
978
979 vars->extentRemaining -= vars->bufferOffset;
980 if (!vars->extentRemaining)
981 {
982 vars->currentExtent++;
983 vars->extentRemaining = vars->currentExtent->length;
984 vars->extentPosition = vars->position;
985 if (!vars->extentRemaining)
986 {
987 err = kIOReturnOverrun;
988 break;
989 }
990 }
991
992 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
993 vars->bufferOffset = 0;
994 if (vars->bufferSize <= vars->extentRemaining)
995 vars->bufferLimit = vars->bufferSize;
996 else
997 vars->bufferLimit = vars->extentRemaining;
998
999 flush = false;
1000 }
1001 }
1002 while (size);
1003
1004 return (err);
1005}
1006
1007static IOReturn
1008IOPolledFileRead(IOPolledFileIOVars * vars,
1009 uint8_t * bytes, IOByteCount size,
1010 hibernate_cryptvars_t * cryptvars)
1011{
1012 IOReturn err = kIOReturnSuccess;
1013 IOByteCount copy;
1014
1015// bytesWritten += size;
1016
1017 do
1018 {
1019 copy = vars->bufferLimit - vars->bufferOffset;
1020 if (copy > size)
1021 copy = size;
1022
1023 if (bytes)
1024 {
1025 bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
1026 bytes += copy;
1027 }
1028 size -= copy;
1029 vars->bufferOffset += copy;
1030// vars->position += copy;
1031
6d2010ae 1032 if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd))
3a60a9f5
A		 1033 {
1034 if (vars->io)
1035 {
2d21ac55 1036 err = IOHibernatePollerIODone(vars, false);
3a60a9f5
A		 1037 if (kIOReturnSuccess != err)
1038 break;
1039 }
1040 else
1041 cryptvars = 0;
1042
1043if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
1044
6d2010ae 1045 vars->position += vars->lastRead;
3a60a9f5 1046 vars->extentRemaining -= vars->lastRead;
6d2010ae 1047 vars->bufferLimit = vars->lastRead;
3a60a9f5
A		 1048
1049 if (!vars->extentRemaining)
1050 {
1051 vars->currentExtent++;
1052 vars->extentRemaining = vars->currentExtent->length;
1053 vars->extentPosition = vars->position;
1054 if (!vars->extentRemaining)
1055 {
1056 err = kIOReturnOverrun;
1057 break;
1058 }
1059 }
1060
36401178
A		 1061 uint64_t length;
1062 uint64_t lastReadLength = vars->lastRead;
3a60a9f5
A		 1063 uint64_t offset = (vars->position
1064 - vars->extentPosition + vars->currentExtent->start);
36401178
A		 1065 if (vars->extentRemaining <= vars->bufferSize)
1066 length = vars->extentRemaining;
1067 else
1068 length = vars->bufferSize;
6d2010ae
A		 1069 if ((length + vars->position) > vars->readEnd)
1070 length = vars->readEnd - vars->position;
3a60a9f5 1071
6d2010ae
A		 1072 vars->lastRead = length;
1073 if (length)
1074 {
3a60a9f5 1075//if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
6d2010ae
A		 1076 err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
1077 if (kIOReturnSuccess != err)
1078 break;
1079 vars->io = true;
1080 }
3a60a9f5
A		 1081
1082 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1083 vars->bufferOffset = 0;
1084
2d21ac55 1085#if CRYPTO
3a60a9f5
A		 1086 if (cryptvars)
1087 {
1088 uint8_t thisVector[AES_BLOCK_SIZE];
6d2010ae
A		 1089 AbsoluteTime startTime, endTime;
1090
3a60a9f5
A		 1091 // save initial vector for following decrypts
1092 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
36401178 1093 bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
3a60a9f5 1094 &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
6d2010ae 1095
3a60a9f5 1096 // decrypt the buffer
6d2010ae
A		 1097 clock_get_uptime(&startTime);
1098
3a60a9f5
A		 1099 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
1100 &thisVector[0],
36401178 1101 lastReadLength / AES_BLOCK_SIZE,
3a60a9f5
A		 1102 vars->buffer + vars->bufferHalf,
1103 &cryptvars->ctx.decrypt);
6d2010ae
A		 1104
1105 clock_get_uptime(&endTime);
1106 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
1107 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
1108 vars->cryptBytes += lastReadLength;
3a60a9f5 1109 }
b0d623f7 1110#endif /* CRYPTO */
3a60a9f5
A		 1111 }
1112 }
1113 while (size);
1114
1115 return (err);
1116}
1117
1118/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
99c3a104 1119
3a60a9f5
A		 1120IOReturn
1121IOHibernateSystemSleep(void)
1122{
1123 IOReturn err;
1124 OSData * data;
1125 OSObject * obj;
1126 OSString * str;
db609669 1127 OSNumber * num;
0b4c1975 1128 bool dsSSD;
316670eb 1129 IOHibernateVars * vars;
3a60a9f5
A		 1130
1131 gIOHibernateState = kIOHibernateStateInactive;
1132
316670eb
A		 1133 if (!gIOChosenEntry)
1134 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1135
b0d623f7
A		 1136 gIOHibernateDebugFlags = 0;
1137 if (kIOLogHibernate & gIOKitDebug)
1138 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
1139
0b4c1975
A		 1140 if (IOService::getPMRootDomain()->getHibernateSettings(
1141 &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime))
6d2010ae 1142 {
3a60a9f5
A		 1143 if (kIOHibernateModeSleep & gIOHibernateMode)
1144 // default to discard clean for safe sleep
1145 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
1146 | kIOHibernateModeDiscardCleanActive);
6d2010ae 1147 }
3a60a9f5 1148
3a60a9f5
A		 1149 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
1150 {
1151 if ((str = OSDynamicCast(OSString, obj)))
2d21ac55
A		 1152 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
1153 sizeof(gIOHibernateFilename));
3a60a9f5
A		 1154 obj->release();
1155 }
1156
1157 if (!gIOHibernateMode || !gIOHibernateFilename[0])
1158 return (kIOReturnUnsupported);
1159
1160 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
1161
316670eb
A		 1162 vars = IONew(IOHibernateVars, 1);
1163 if (!vars) return (kIOReturnNoMemory);
1164 bzero(vars, sizeof(*vars));
1165
1166 IOLockLock(gFSLock);
1167 if (kFSIdle != gFSState)
1168 {
1169 HIBLOG("hibernate file busy\n");
1170 IOLockUnlock(gFSLock);
1171 IODelete(vars, IOHibernateVars, 1);
1172 return (kIOReturnBusy);
1173 }
1174 gFSState = kFSOpening;
1175 IOLockUnlock(gFSLock);
2d21ac55 1176
3a60a9f5
A		 1177 do
1178 {
0c530ab8
A		 1179 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1180 4 * page_size, page_size);
1181 vars->ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1182 2 * kDefaultIOSize, page_size);
3a60a9f5 1183
6d2010ae
A		 1184 vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1185 ptoa_64(gIOHibernateHandoffPageCount), page_size);
1186
1187 if (!vars->srcBuffer || !vars->ioBuffer || !vars->handoffBuffer)
3a60a9f5
A		 1188 {
1189 err = kIOReturnNoMemory;
1190 break;
1191 }
1192
db609669
A		 1193 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey)))
1194 {
1195 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMinSize = num->unsigned64BitValue();
1196 obj->release();
1197 }
1198 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey)))
1199 {
1200 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMaxSize = num->unsigned64BitValue();
1201 obj->release();
1202 }
1203
1204 boolean_t encryptedswap = true;
1205 uint32_t pageCount;
1206 AbsoluteTime startTime, endTime;
1207 uint64_t nsec;
1208
1209 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1210 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
6d2010ae 1211 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
db609669
A		 1212
1213 dsSSD = (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey));
1214 clock_get_uptime(&startTime);
1215 err = hibernate_setup(gIOHibernateCurrentHeader,
1216 gIOHibernateFreeRatio, gIOHibernateFreeTime,
1217 dsSSD,
1218 &vars->page_list, &vars->page_list_wired, &vars->page_list_pal);
1219 clock_get_uptime(&endTime);
1220 SUB_ABSOLUTETIME(&endTime, &startTime);
1221 absolutetime_to_nanoseconds(endTime, &nsec);
1222 HIBLOG("hibernate_setup(%d) took %qd ms\n", err, nsec / 1000000ULL);
1223
1224 uint64_t setFileSize = 0;
1225
1226 if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode))
1227 {
1228 hibernate_page_list_setall(vars->page_list,
1229 vars->page_list_wired,
1230 vars->page_list_pal,
1231 true /* preflight */,
1232 &pageCount);
1233 PE_Video consoleInfo;
1234 bzero(&consoleInfo, sizeof(consoleInfo));
1235 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1236
1237 // estimate: 5% increase in pages compressed
1238 // screen preview 2 images compressed 50%
1239 setFileSize = ((ptoa_64((105 * pageCount) / 100) * gIOHibernateCompression) >> 8)
1240 + vars->page_list->list_size
1241 + (consoleInfo.v_width * consoleInfo.v_height * 4);
1242
1243 HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
1244 pageCount, (100ULL * gIOHibernateCompression) >> 8,
1245 setFileSize, vars->fileMinSize);
1246
1247 if (!(kIOHibernateModeFileResize & gIOHibernateMode)
1248 && (setFileSize < vars->fileMinSize))
1249 {
1250 setFileSize = vars->fileMinSize;
1251 }
1252 }
1253
1254 // open & invalidate the image file
1255
1256 err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, vars->ioBuffer,
6d2010ae
A		 1257 &vars->fileVars, &vars->fileExtents, &data,
1258 &vars->volumeCryptKey[0]);
3a60a9f5
A		 1259 if (KERN_SUCCESS != err)
1260 {
1261 HIBLOG("IOPolledFileOpen(%x)\n", err);
1262 break;
1263 }
3a60a9f5 1264
6d2010ae 1265 dsSSD = ((0 != (kIOHibernateOptionSSD & vars->fileVars->flags))
0b4c1975 1266 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
0b4c1975
A		 1267 if (dsSSD)
1268 {
1269 gIOHibernateCurrentHeader->options |=
1270 kIOHibernateOptionSSD
1271 | kIOHibernateOptionColor;
6d2010ae
A		 1272
1273#if defined(__i386__) || defined(__x86_64__)
1274 if (!uuid_is_null(vars->volumeCryptKey) &&
1275 (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey)))
1276 {
1277 uintptr_t smcVars[2];
1278 smcVars[0] = sizeof(vars->volumeCryptKey);
316670eb 1279 smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
6d2010ae
A		 1280
1281 IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
1282 bzero(smcVars, sizeof(smcVars));
1283 }
1284#endif
0b4c1975
A		 1285 }
1286 else
1287 {
1288 gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
1289 }
1290
0b4c1975 1291
3a60a9f5 1292 if (KERN_SUCCESS != err)
3a60a9f5 1293 break;
3a60a9f5 1294
6d2010ae 1295 if (encryptedswap || !uuid_is_null(vars->volumeCryptKey))
3a60a9f5
A		 1296 gIOHibernateMode ^= kIOHibernateModeEncrypt;
1297
0b4c1975
A		 1298 if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1299 {
1300 vars->videoAllocSize = kVideoMapSize;
1301 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
1302 vars->videoMapping = 0;
1303 }
3a60a9f5
A		 1304
1305 // generate crypt keys
1306 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
1307 vars->wiredCryptKey[i] = random();
1308 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
1309 vars->cryptKey[i] = random();
1310
1311 // set nvram
1312
1313 IORegistryEntry * regEntry;
1314 if (!gIOOptionsEntry)
1315 {
1316 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
1317 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
1318 if (regEntry && !gIOOptionsEntry)
1319 regEntry->release();
1320 }
3a60a9f5
A		 1321
1322 if (gIOOptionsEntry)
1323 {
1324 const OSSymbol * sym;
3a60a9f5
A		 1325
1326 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1327 if (sym)
1328 {
1329 gIOOptionsEntry->setProperty(sym, data);
1330 sym->release();
1331 }
1332 data->release();
1333
b0d623f7 1334#if defined(__i386__) || defined(__x86_64__)
0c530ab8
A		 1335 struct AppleRTCHibernateVars
1336 {
1337 uint8_t signature[4];
1338 uint32_t revision;
1339 uint8_t booterSignature[20];
1340 uint8_t wiredCryptKey[16];
1341 };
1342 AppleRTCHibernateVars rtcVars;
1343
1344 rtcVars.signature[0] = 'A';
1345 rtcVars.signature[1] = 'A';
1346 rtcVars.signature[2] = 'P';
1347 rtcVars.signature[3] = 'L';
1348 rtcVars.revision = 1;
1349 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
1350 if (gIOHibernateBootSignature[0])
1351 {
1352 char c;
1353 uint8_t value = 0;
1354 for (uint32_t i = 0;
1355 (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
1356 i++)
1357 {
1358 if (c >= 'a')
1359 c -= 'a' - 10;
1360 else if (c >= 'A')
1361 c -= 'A' - 10;
1362 else if (c >= '0')
1363 c -= '0';
1364 else
1365 continue;
1366 value = (value << 4) | c;
1367 if (i & 1)
1368 rtcVars.booterSignature[i >> 1] = value;
1369 }
1370 }
1371 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
1372 if (data)
2d21ac55 1373 {
060df5ea
A		 1374 if (!gIOHibernateRTCVariablesKey)
1375 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1376 if (gIOHibernateRTCVariablesKey)
1377 IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
1378
1379 if( gIOOptionsEntry )
1380 {
1381 if( gIOHibernateMode & kIOHibernateModeSwitch )
1382 {
1383 const OSSymbol *sym;
1384 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSwitchVarsKey);
1385 if( sym )
2d21ac55 1386 {
060df5ea
A		 1387 gIOOptionsEntry->setProperty(sym, data); /* intentional insecure backup of rtc boot vars */
1388 sym->release();
2d21ac55 1389 }
060df5ea
A		 1390 }
1391 }
2d21ac55 1392
060df5ea 1393 data->release();
0c530ab8
A		 1394 }
1395 if (gIOChosenEntry)
1396 {
1397 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
1398 if (data)
1399 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
060df5ea
A		 1400 {
1401 // set BootNext
1402
1403 if (!gIOHibernateBoot0082Data)
1404 {
1405 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
1406 if (data)
1407 {
1408 // AppleNVRAM_EFI_LOAD_OPTION
1409 struct {
1410 uint32_t Attributes;
1411 uint16_t FilePathLength;
1412 uint16_t Desc;
1413 } loadOptionHeader;
1414 loadOptionHeader.Attributes = 1;
1415 loadOptionHeader.FilePathLength = data->getLength();
1416 loadOptionHeader.Desc = 0;
1417 gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
1418 if (gIOHibernateBoot0082Data)
1419 {
1420 gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
1421 gIOHibernateBoot0082Data->appendBytes(data);
1422 }
1423 }
1424 }
1425 if (!gIOHibernateBoot0082Key)
1426 gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
1427 if (!gIOHibernateBootNextKey)
1428 gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
1429 if (!gIOHibernateBootNextData)
1430 {
1431 uint16_t bits = 0x0082;
1432 gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
1433 }
1434 if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData)
1435 {
1436 gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
1437 gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
1438 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
1439 }
1440 }
0c530ab8 1441 }
b0d623f7 1442#else /* !i386 && !x86_64 */
3a60a9f5
A		 1443 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1444 {
1445 data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1446 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
1447 if (sym && data)
1448 gIOOptionsEntry->setProperty(sym, data);
1449 if (sym)
1450 sym->release();
1451 if (data)
1452 data->release();
2d21ac55 1453 if (false && gIOHibernateBootSignature[0])
3a60a9f5
A		 1454 {
1455 data = OSData::withCapacity(16);
1456 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
1457 if (sym && data)
1458 {
1459 char c;
0c530ab8 1460 uint8_t value = 0;
3a60a9f5
A		 1461 for (uint32_t i = 0; (c = gIOHibernateBootSignature[i]); i++)
1462 {
1463 if (c >= 'a')
1464 c -= 'a' - 10;
1465 else if (c >= 'A')
1466 c -= 'A' - 10;
1467 else if (c >= '0')
1468 c -= '0';
1469 else
1470 continue;
1471 value = (value << 4) | c;
1472 if (i & 1)
1473 data->appendBytes(&value, sizeof(value));
1474 }
1475 gIOOptionsEntry->setProperty(sym, data);
1476 }
1477 if (sym)
1478 sym->release();
1479 if (data)
1480 data->release();
1481 }
1482 }
3a60a9f5
A		 1483 if (!vars->haveFastBoot)
1484 {
1485 // set boot volume to zero
1486 IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
1487 if (platform && (kIOReturnSuccess == platform->readXPRAM(kXPRamAudioVolume,
1488 &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume))))
1489 {
1490 uint8_t newVolume;
1491 newVolume = vars->saveBootAudioVolume & 0xf8;
1492 platform->writeXPRAM(kXPRamAudioVolume,
1493 &newVolume, sizeof(newVolume));
1494 }
1495 }
b0d623f7 1496#endif /* !i386 && !x86_64 */
3a60a9f5
A		 1497 }
1498 // --
1499
316670eb
A		 1500 }
1501 while (false);
1502
1503 IOLockLock(gFSLock);
1504 if ((kIOReturnSuccess == err) && (kFSOpening == gFSState))
1505 {
1506 gFSState = kFSOpened;
1507 gIOHibernateVars = *vars;
1508 gFileVars = *vars->fileVars;
1509 gIOHibernateVars.fileVars = &gFileVars;
1510 gIOHibernateFileRef = gFileVars.fileRef;
3a60a9f5
A		 1511 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
1512 gIOHibernateState = kIOHibernateStateHibernating;
1513 }
316670eb
A		 1514 else
1515 {
1516 HIBLOG("hibernate file close due timeout\n");
1517 if (vars->fileVars && vars->fileVars->fileRef) kern_close_file_for_direct_io(vars->fileVars->fileRef, 0, 0, 0, 0, 0);
1518 IOHibernateDone(vars);
1519 gFSState = kFSIdle;
1520 }
1521 IOLockUnlock(gFSLock);
1522
1523 if (vars->fileVars) IODelete(vars->fileVars, IOPolledFileIOVars, 1);
1524 IODelete(vars, IOHibernateVars, 1);
3a60a9f5
A		 1525
1526 return (err);
1527}
1528
1529/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1530
2d21ac55
A		 1531DECLARE_IOHIBERNATEPROGRESSALPHA
1532
1533static void
1534ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
1535{
1536 uint32_t rowBytes, pixelShift;
1537 uint32_t x, y;
1538 int32_t blob;
1539 uint32_t alpha, in, color, result;
1540 uint8_t * out;
1541 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1542
1543 rowBytes = display->rowBytes;
1544 pixelShift = display->depth >> 4;
1545 if (pixelShift < 1) return;
1546
1547 screen += ((display->width
1548 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1549 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1550
1551 for (y = 0; y < kIOHibernateProgressHeight; y++)
1552 {
1553 out = screen + y * rowBytes;
1554 for (blob = 0; blob < kIOHibernateProgressCount; blob++)
1555 {
1556 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
1557 for (x = 0; x < kIOHibernateProgressWidth; x++)
1558 {
1559 alpha = gIOHibernateProgressAlpha[y][x];
1560 result = color;
1561 if (alpha)
1562 {
1563 if (0xff != alpha)
1564 {
1565 if (1 == pixelShift)
1566 {
1567 in = *((uint16_t *)out) & 0x1f; // 16
1568 in = (in << 3) | (in >> 2);
1569 }
1570 else
1571 in = *((uint32_t *)out) & 0xff; // 32
1572 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
1573 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
1574 }
1575 if (1 == pixelShift)
1576 {
1577 result >>= 3;
1578 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1579 }
1580 else
1581 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1582 }
1583 out += (1 << pixelShift);
1584 }
1585 out += (kIOHibernateProgressSpacing << pixelShift);
1586 }
1587 }
1588}
1589
1590
1591static void
1592ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
1593{
1594 uint32_t rowBytes, pixelShift;
1595 uint32_t x, y;
1596 int32_t blob, lastBlob;
1597 uint32_t alpha, in, color, result;
1598 uint8_t * out;
1599 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1600
1601 pixelShift = display->depth >> 4;
1602 if (pixelShift < 1)
1603 return;
1604
1605 rowBytes = display->rowBytes;
1606
1607 screen += ((display->width
1608 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1609 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1610
1611 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
1612
1613 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
1614
1615 for (y = 0; y < kIOHibernateProgressHeight; y++)
1616 {
1617 out = screen + y * rowBytes;
1618 for (blob = firstBlob; blob <= lastBlob; blob++)
1619 {
1620 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
1621 for (x = 0; x < kIOHibernateProgressWidth; x++)
1622 {
1623 alpha = gIOHibernateProgressAlpha[y][x];
1624 result = color;
1625 if (alpha)
1626 {
1627 if (0xff != alpha)
1628 {
1629 in = display->progressSaveUnder[blob][saveindex[blob]++];
1630 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
1631 }
1632 if (1 == pixelShift)
1633 {
1634 result >>= 3;
1635 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1636 }
1637 else
1638 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1639 }
1640 out += (1 << pixelShift);
1641 }
1642 out += (kIOHibernateProgressSpacing << pixelShift);
1643 }
1644 }
1645}
1646
1647/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1648
316670eb
A		 1649IOReturn
1650IOHibernateIOKitSleep(void)
1651{
1652 IOReturn ret = kIOReturnSuccess;
1653 IOLockLock(gFSLock);
1654 if (kFSOpening == gFSState)
1655 {
1656 gFSState = kFSTimedOut;
1657 HIBLOG("hibernate file open timed out\n");
1658 ret = kIOReturnTimeout;
1659 }
1660 IOLockUnlock(gFSLock);
1661 return (ret);
1662}
1663
1664/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1665
3a60a9f5
A		 1666IOReturn
1667IOHibernateSystemHasSlept(void)
1668{
316670eb 1669 IOReturn ret = kIOReturnSuccess;
3a60a9f5 1670 IOHibernateVars * vars = &gIOHibernateVars;
316670eb 1671 OSObject * obj = 0;
2d21ac55
A		 1672 OSData * data;
1673
316670eb
A		 1674 IOLockLock(gFSLock);
1675 if ((kFSOpened != gFSState) && gIOHibernateMode)
1676 {
1677 ret = kIOReturnTimeout;
1678 }
1679 IOLockUnlock(gFSLock);
1680 if (kIOReturnSuccess != ret) return (ret);
1681
1682 if (gIOHibernateMode) obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
2d21ac55
A		 1683 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1684 if (obj && !vars->previewBuffer)
1685 obj->release();
3a60a9f5 1686
2d21ac55
A		 1687 vars->consoleMapping = NULL;
1688 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
3a60a9f5 1689 {
2d21ac55
A		 1690 vars->previewBuffer->release();
1691 vars->previewBuffer = 0;
1692 }
3a60a9f5 1693
0b4c1975
A		 1694 if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1695 && vars->previewBuffer
1696 && (data = OSDynamicCast(OSData,
2d21ac55
A		 1697 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
1698 {
1699 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
b0d623f7 1700 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
2d21ac55
A		 1701
1702 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1703
1704 if (kIOHibernatePreviewUpdates & flags)
1705 {
1706 PE_Video consoleInfo;
1707 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1708
1709 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1710
1711 graphicsInfo->width = consoleInfo.v_width;
1712 graphicsInfo->height = consoleInfo.v_height;
1713 graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
1714 graphicsInfo->depth = consoleInfo.v_depth;
1715 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1716
1717 HIBPRINT("video %p %d %d %d\n",
0b4c1975
A		 1718 vars->consoleMapping, graphicsInfo->depth,
1719 graphicsInfo->width, graphicsInfo->height);
2d21ac55
A		 1720 if (vars->consoleMapping)
1721 ProgressInit(graphicsInfo, vars->consoleMapping,
1722 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1723 }
3a60a9f5 1724 }
2d21ac55 1725
3a60a9f5
A		 1726 if (gIOOptionsEntry)
1727 gIOOptionsEntry->sync();
1728
316670eb 1729 return (ret);
3a60a9f5
A		 1730}
1731
1732/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1733
6d2010ae
A		 1734static DeviceTreeNode *
1735MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry)
1736{
1737 DeviceTreeNodeProperty * prop;
1738 DeviceTreeNode * child;
1739 IORegistryEntry * childRegEntry;
1740 const char * nameProp;
1741 unsigned int propLen, idx;
1742
1743 prop = (DeviceTreeNodeProperty *) (entry + 1);
1744 for (idx = 0; idx < entry->nProperties; idx++)
1745 {
1746 if (regEntry && (0 != strcmp("name", prop->name)))
1747 {
1748 regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
1749// HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
1750 }
1751 prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
1752 }
1753
1754 child = (DeviceTreeNode *) prop;
1755 for (idx = 0; idx < entry->nChildren; idx++)
1756 {
1757 if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen))
1758 panic("no name");
1759 childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
1760// HIBPRINT("%s == %p\n", nameProp, childRegEntry);
1761 child = MergeDeviceTree(child, childRegEntry);
1762 }
1763 return (child);
1764}
1765
3a60a9f5
A		 1766IOReturn
1767IOHibernateSystemWake(void)
1768{
316670eb
A		 1769 if (kFSOpened == gFSState)
1770 {
1771 IOHibernateDone(&gIOHibernateVars);
1772 }
1773 else
1774 {
1775 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1776 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1777 }
1778 return (kIOReturnSuccess);
1779}
3a60a9f5 1780
316670eb
A		 1781static IOReturn
1782IOHibernateDone(IOHibernateVars * vars)
1783{
99c3a104 1784 hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
3a60a9f5
A		 1785
1786 if (vars->videoMapping)
1787 {
1788 if (vars->videoMapSize)
1789 // remove mappings
1790 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1791 if (vars->videoAllocSize)
1792 // dealloc range
b0d623f7 1793 kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
3a60a9f5
A		 1794 }
1795
1796 if (vars->previewBuffer)
1797 {
1798 vars->previewBuffer->release();
1799 vars->previewBuffer = 0;
1800 }
1801
0b4c1975
A		 1802 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1803 {
1804 IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
1805 gIOHibernateCurrentHeader->options, 32);
1806 }
1807 else
1808 {
1809 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1810 }
1811
1812 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1813 && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus))
1814 {
1815 IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
1816 &gIOHibernateGraphicsInfo->gfxStatus,
1817 sizeof(gIOHibernateGraphicsInfo->gfxStatus));
1818 }
1819 else
1820 {
1821 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1822 }
1823
1824
3a60a9f5
A		 1825 if (vars->fileVars)
1826 {
1827 IOPolledFileClose(vars->fileVars);
1828 }
1829
1830 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1831
b0d623f7 1832#if defined(__i386__) || defined(__x86_64__)
060df5ea 1833 IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
6d2010ae 1834 IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
2d21ac55
A		 1835
1836 /*
1837 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1838 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1839 */
1840 if (gIOOptionsEntry) {
2d21ac55 1841
060df5ea
A		 1842 if (gIOHibernateRTCVariablesKey) {
1843 if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
1844 gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
1845 }
1846 }
1847
1848 if (gIOHibernateBootNextKey)
1849 {
1850 if (gIOHibernateBootNextSave)
1851 {
1852 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
1853 gIOHibernateBootNextSave->release();
1854 gIOHibernateBootNextSave = NULL;
2d21ac55 1855 }
060df5ea
A		 1856 else
1857 gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
1858 }
1859 gIOOptionsEntry->sync();
2d21ac55 1860 }
0c530ab8 1861#endif
3a60a9f5
A		 1862
1863 if (vars->srcBuffer)
1864 vars->srcBuffer->release();
1865 if (vars->ioBuffer)
1866 vars->ioBuffer->release();
6d2010ae 1867 bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
99c3a104 1868 if (vars->handoffBuffer)
6d2010ae 1869 {
99c3a104 1870 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
6d2010ae 1871 {
99c3a104
A		 1872 IOHibernateHandoff * handoff;
1873 bool done = false;
1874 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
1875 !done;
1876 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
6d2010ae 1877 {
99c3a104
A		 1878 HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
1879 uint8_t * data = &handoff->data[0];
1880 switch (handoff->type)
1881 {
1882 case kIOHibernateHandoffTypeEnd:
1883 done = true;
1884 break;
6d2010ae 1885
99c3a104
A		 1886 case kIOHibernateHandoffTypeDeviceTree:
1887 MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
1888 break;
1889
1890 case kIOHibernateHandoffTypeKeyStore:
6d2010ae 1891#if defined(__i386__) || defined(__x86_64__)
6d2010ae 1892 {
99c3a104
A		 1893 IOBufferMemoryDescriptor *
1894 md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
1895 if (md)
1896 {
1897 IOSetKeyStoreData(md);
1898 }
6d2010ae 1899 }
6d2010ae 1900#endif
99c3a104
A		 1901 break;
1902
1903 default:
1904 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
1905 break;
1906 }
1907 }
6d2010ae
A		 1908 }
1909 vars->handoffBuffer->release();
1910 }
3a60a9f5
A		 1911 if (vars->fileExtents)
1912 vars->fileExtents->release();
1913
1914 bzero(vars, sizeof(*vars));
1915
1916// gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1917
1918 return (kIOReturnSuccess);
1919}
1920
1921IOReturn
1922IOHibernateSystemPostWake(void)
1923{
316670eb
A		 1924 struct kern_direct_file_io_ref_t * fileRef;
1925
1926 if (kFSOpened == gFSState)
3a60a9f5 1927 {
6d2010ae 1928 // invalidate & close the image file
0c530ab8 1929 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
316670eb
A		 1930 if ((fileRef = gIOHibernateFileRef))
1931 {
1932 gIOHibernateFileRef = 0;
1933 kern_close_file_for_direct_io(fileRef,
6d2010ae 1934 0, (caddr_t) gIOHibernateCurrentHeader,
7ddcb079
A		 1935 sizeof(IOHibernateImageHeader),
1936 sizeof(IOHibernateImageHeader),
1937 gIOHibernateCurrentHeader->imageSize);
316670eb
A		 1938 }
1939 gFSState = kFSIdle;
3a60a9f5
A		 1940 }
1941 return (kIOReturnSuccess);
1942}
1943
316670eb
A		 1944bool IOHibernateWasScreenLocked(void)
1945{
1946 bool ret = false;
1947 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry)
1948 {
1949 OSData *
1950 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
1951 if (data) switch (*((uint32_t *)data->getBytesNoCopy()))
1952 {
1953 case kIOScreenLockLocked:
1954 case kIOScreenLockFileVaultDialog:
1955 ret = true;
1956 break;
1957 case kIOScreenLockNoLock:
1958 case kIOScreenLockUnlocked:
1959 default:
1960 ret = false;
1961 break;
1962 }
1963 }
1964 return (ret);
1965}
1966
3a60a9f5
A		 1967/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1968
2d21ac55 1969SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
6d2010ae 1970 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2d21ac55
A		 1971 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
1972SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
6d2010ae 1973 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2d21ac55
A		 1974 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
1975SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
6d2010ae 1976 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2d21ac55
A		 1977 &gIOHibernateMode, 0, "");
1978
3a60a9f5
A		 1979void
1980IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1981{
1982 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
1983 if (data)
1984 {
1985 rootDomain->setProperty(kIOHibernateStateKey, data);
1986 data->release();
1987 }
1988
2d21ac55 1989 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))
3a60a9f5
A		 1990 gIOHibernateMode = kIOHibernateModeOn;
1991 else
1992 gIOHibernateFilename[0] = 0;
1993
3a60a9f5 1994 sysctl_register_oid(&sysctl__kern_hibernatefile);
3a60a9f5 1995 sysctl_register_oid(&sysctl__kern_bootsignature);
3a60a9f5 1996 sysctl_register_oid(&sysctl__kern_hibernatemode);
316670eb
A		 1997
1998 gFSLock = IOLockAlloc();
3a60a9f5
A		 1999}
2000
2d21ac55 2001
3a60a9f5
A		 2002/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2003
2004static void
2005hibernate_setup_for_wake(void)
2006{
3a60a9f5
A		 2007}
2008
2009/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2010
0c530ab8
A		 2011#define C_ASSERT(e) typedef char __C_ASSERT__[(e) ? 1 : -1]
2012
0b4c1975
A		 2013static bool
2014no_encrypt_page(vm_offset_t ppnum)
2015{
2016 if (pmap_is_noencrypt((ppnum_t)ppnum) == TRUE)
2017 {
2018 return true;
2019 }
2020 return false;
2021}
2022
2023uint32_t wired_pages_encrypted = 0;
2024uint32_t dirty_pages_encrypted = 0;
2025uint32_t wired_pages_clear = 0;
2026
6d2010ae
A		 2027static void
2028hibernate_pal_callback(void *vars_arg, vm_offset_t addr)
2029{
2030 IOHibernateVars *vars = (IOHibernateVars *)vars_arg;
2031 /* Make sure it's not in either of the save lists */
2032 hibernate_set_page_state(vars->page_list, vars->page_list_wired, atop_64(addr), 1, kIOHibernatePageStateFree);
2033
2034 /* Set it in the bitmap of pages owned by the PAL */
2035 hibernate_page_bitset(vars->page_list_pal, TRUE, atop_64(addr));
2036}
2037
2038static struct hibernate_cryptvars_t *local_cryptvars;
2039
2040extern "C" int
2041hibernate_pal_write(void *buffer, size_t size)
2042{
2043 IOHibernateVars * vars = &gIOHibernateVars;
2044
2045 IOReturn err = IOPolledFileWrite(vars->fileVars, (const uint8_t *)buffer, size, local_cryptvars);
2046 if (kIOReturnSuccess != err) {
2047 kprintf("epic hibernate fail! %d\n", err);
2048 return err;
2049 }
2050
2051 return 0;
2052}
2053
2054
2d21ac55 2055extern "C" uint32_t
3a60a9f5
A		 2056hibernate_write_image(void)
2057{
2058 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
2059 IOHibernateVars * vars = &gIOHibernateVars;
2060 IOPolledFileExtent * fileExtents;
2061
0c530ab8
A		 2062 C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
2063
3a60a9f5
A		 2064 uint32_t pageCount, pagesDone;
2065 IOReturn err;
0b4c1975
A		 2066 vm_offset_t ppnum, page;
2067 IOItemCount count;
3a60a9f5
A		 2068 uint8_t * src;
2069 uint8_t * data;
2070 IOByteCount pageCompressedSize;
2071 uint64_t compressedSize, uncompressedSize;
2072 uint64_t image1Size = 0;
2073 uint32_t bitmap_size;
0b4c1975 2074 bool iterDone, pollerOpen, needEncrypt;
3a60a9f5
A		 2075 uint32_t restore1Sum, sum, sum1, sum2;
2076 uint32_t tag;
2077 uint32_t pageType;
2078 uint32_t pageAndCount[2];
6d2010ae
A		 2079 addr64_t phys64;
2080 IOByteCount segLen;
3a60a9f5
A		 2081
2082 AbsoluteTime startTime, endTime;
6d2010ae 2083 AbsoluteTime allTime, compTime;
0b4c1975 2084 uint64_t compBytes;
3a60a9f5
A		 2085 uint64_t nsec;
2086 uint32_t lastProgressStamp = 0;
2087 uint32_t progressStamp;
2d21ac55 2088 uint32_t blob, lastBlob = (uint32_t) -1L;
3a60a9f5
A		 2089
2090 hibernate_cryptvars_t _cryptvars;
2091 hibernate_cryptvars_t * cryptvars = 0;
2092
0b4c1975
A		 2093 wired_pages_encrypted = 0;
2094 dirty_pages_encrypted = 0;
2095 wired_pages_clear = 0;
2096
3a60a9f5
A		 2097 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
2098 return (false /* sleep */ );
2099
0b4c1975
A		 2100 if (kIOHibernateModeSleep & gIOHibernateMode)
2101 kdebug_enable = save_kdebug_enable;
2102
2103 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START, 0, 0, 0, 0, 0);
6d2010ae 2104 IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
0b4c1975 2105
3a60a9f5
A		 2106 restore1Sum = sum1 = sum2 = 0;
2107
6d2010ae
A		 2108 hibernate_pal_prepare();
2109
2d21ac55 2110#if CRYPTO
3a60a9f5
A		 2111 // encryption data. "iv" is the "initial vector".
2112 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2113 {
2114 static const unsigned char first_iv[AES_BLOCK_SIZE]
2115 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
2116 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
2117
2118 cryptvars = &gIOHibernateCryptWakeContext;
2119 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
2120 aes_encrypt_key(vars->cryptKey,
2121 kIOHibernateAESKeySize,
2122 &cryptvars->ctx.encrypt);
2123 aes_decrypt_key(vars->cryptKey,
2124 kIOHibernateAESKeySize,
2125 &cryptvars->ctx.decrypt);
2126
2127 cryptvars = &_cryptvars;
2128 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
6d2010ae
A		 2129 for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++)
2130 vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
2131 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
3a60a9f5
A		 2132 aes_encrypt_key(vars->wiredCryptKey,
2133 kIOHibernateAESKeySize,
2134 &cryptvars->ctx.encrypt);
2135
2136 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
2137 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
2138 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
6d2010ae
A		 2139
2140 local_cryptvars = cryptvars;
3a60a9f5 2141 }
2d21ac55 2142#endif /* CRYPTO */
3a60a9f5
A		 2143
2144 hibernate_setup_for_wake();
2145
2146 hibernate_page_list_setall(vars->page_list,
2147 vars->page_list_wired,
db609669
A		 2148 vars->page_list_pal,
2149 false /* !preflight */,
3a60a9f5
A		 2150 &pageCount);
2151
2152 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
2153
2154 fileExtents = (IOPolledFileExtent *) vars->fileExtents->getBytesNoCopy();
2155
2156#if 0
2157 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
2158 for (page = 0; page < count; page++)
2159 {
2160 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
2161 fileExtents[page].start, fileExtents[page].length,
2162 fileExtents[page].start + fileExtents[page].length);
2163 }
2164#endif
2165
0b4c1975 2166 needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
3a60a9f5 2167 AbsoluteTime_to_scalar(&compTime) = 0;
6d2010ae 2168 compBytes = 0;
3a60a9f5
A		 2169
2170 clock_get_uptime(&allTime);
b0d623f7
A		 2171 IOService::getPMRootDomain()->pmStatsRecordEvent(
2172 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
3a60a9f5
A		 2173
2174 do
2175 {
2176 compressedSize = 0;
2177 uncompressedSize = 0;
3a60a9f5
A		 2178
2179 IOPolledFileSeek(vars->fileVars, sizeof(IOHibernateImageHeader));
2180
2181 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
2182 ml_get_interrupts_enabled());
2183 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledBeforeSleepState, vars->ioBuffer);
2184 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
2185 pollerOpen = (kIOReturnSuccess == err);
2186 if (!pollerOpen)
2187 break;
2188
2189 // copy file block extent list if larger than header
2190
2191 count = vars->fileExtents->getLength();
2192 if (count > sizeof(header->fileExtentMap))
2193 {
2194 count -= sizeof(header->fileExtentMap);
2195 err = IOPolledFileWrite(vars->fileVars,
2196 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
2197 if (kIOReturnSuccess != err)
2198 break;
2199 }
2200
b0d623f7
A		 2201 uintptr_t hibernateBase;
2202 uintptr_t hibernateEnd;
2203
6d2010ae 2204 hibernateBase = HIB_BASE; /* Defined in PAL headers */
b0d623f7 2205
316670eb 2206 hibernateEnd = (segHIBB + segSizeHIB);
6d2010ae 2207
3a60a9f5 2208 // copy out restore1 code
6d2010ae
A		 2209
2210 for (count = 0;
2211 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2212 count += segLen)
2213 {
2214 for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++)
2215 {
2216 gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
2217 }
2218 }
2219
2220 page = atop_32(kvtophys(hibernateBase));
2221 count = atop_32(round_page(hibernateEnd) - hibernateBase);
2222 header->restore1CodePhysPage = page;
2223 header->restore1CodeVirt = hibernateBase;
3a60a9f5 2224 header->restore1PageCount = count;
b0d623f7
A		 2225 header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
2226 header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
3a60a9f5 2227
316670eb 2228 // sum __HIB seg, with zeros for the stack
b0d623f7 2229 src = (uint8_t *) trunc_page(hibernateBase);
3a60a9f5
A		 2230 for (page = 0; page < count; page++)
2231 {
2232 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
6d2010ae 2233 restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
3a60a9f5 2234 else
0b4c1975 2235 restore1Sum += 0x00000000;
3a60a9f5
A		 2236 src += page_size;
2237 }
2238 sum1 = restore1Sum;
2239
316670eb 2240 // write the __HIB seg, with zeros for the stack
3a60a9f5 2241
b0d623f7
A		 2242 src = (uint8_t *) trunc_page(hibernateBase);
2243 count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
3a60a9f5
A		 2244 if (count)
2245 {
2246 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
2247 if (kIOReturnSuccess != err)
2248 break;
2249 }
2250 err = IOPolledFileWrite(vars->fileVars,
2251 (uint8_t *) 0,
2252 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
2253 cryptvars);
2254 if (kIOReturnSuccess != err)
2255 break;
2256 src = &gIOHibernateRestoreStackEnd[0];
b0d623f7 2257 count = round_page(hibernateEnd) - ((uintptr_t) src);
3a60a9f5
A		 2258 if (count)
2259 {
2260 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
2261 if (kIOReturnSuccess != err)
2262 break;
2263 }
2264
316670eb
A		 2265 vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
2266 vars->fileVars->encryptEnd = UINT64_MAX;
2267 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
2268
3a60a9f5
A		 2269 // write the preview buffer
2270
2d21ac55 2271 if (vars->previewBuffer)
3a60a9f5
A		 2272 {
2273 ppnum = 0;
2274 count = 0;
2275 do
2276 {
b0d623f7 2277 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
3a60a9f5
A		 2278 pageAndCount[0] = atop_64(phys64);
2279 pageAndCount[1] = atop_32(segLen);
2280 err = IOPolledFileWrite(vars->fileVars,
2281 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
2282 cryptvars);
2283 if (kIOReturnSuccess != err)
2284 break;
2285 count += segLen;
2286 ppnum += sizeof(pageAndCount);
2287 }
2288 while (phys64);
2289 if (kIOReturnSuccess != err)
2290 break;
2291
b0d623f7 2292 src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
316670eb
A		 2293
2294 ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
2295
2d21ac55 2296 count = vars->previewBuffer->getLength();
3a60a9f5
A		 2297
2298 header->previewPageListSize = ppnum;
2299 header->previewSize = count + ppnum;
2300
2301 for (page = 0; page < count; page += page_size)
0b4c1975
A		 2302 {
2303 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
2304 sum1 += hibernate_sum_page(src + page, atop_64(phys64));
2305 }
3a60a9f5
A		 2306 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
2307 if (kIOReturnSuccess != err)
2308 break;
2309 }
2310
2311 // mark areas for no save
2312
2313 for (count = 0;
b0d623f7 2314 (phys64 = vars->ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
3a60a9f5
A		 2315 count += segLen)
2316 {
2317 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2318 atop_64(phys64), atop_32(segLen),
2319 kIOHibernatePageStateFree);
2320 pageCount -= atop_32(segLen);
2321 }
2322
2323 for (count = 0;
b0d623f7 2324 (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
3a60a9f5
A		 2325 count += segLen)
2326 {
2327 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2328 atop_64(phys64), atop_32(segLen),
2329 kIOHibernatePageStateFree);
2330 pageCount -= atop_32(segLen);
2331 }
2332
2333 // copy out bitmap of pages available for trashing during restore
2334
2335 bitmap_size = vars->page_list_wired->list_size;
2336 src = (uint8_t *) vars->page_list_wired;
2337 err = IOPolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
2338 if (kIOReturnSuccess != err)
2339 break;
2340
2341 // mark more areas for no save, but these are not available
2342 // for trashing during restore
0c530ab8
A		 2343
2344 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
3a60a9f5 2345
6d2010ae
A		 2346
2347 page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
2348 count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
3a60a9f5
A		 2349 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2350 page, count,
2351 kIOHibernatePageStateFree);
2352 pageCount -= count;
3a60a9f5
A		 2353
2354 if (vars->previewBuffer) for (count = 0;
b0d623f7 2355 (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
3a60a9f5
A		 2356 count += segLen)
2357 {
2358 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2359 atop_64(phys64), atop_32(segLen),
2360 kIOHibernatePageStateFree);
2361 pageCount -= atop_32(segLen);
2362 }
2363
6d2010ae
A		 2364 for (count = 0;
2365 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2366 count += segLen)
2367 {
2368 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2369 atop_64(phys64), atop_32(segLen),
2370 kIOHibernatePageStateFree);
2371 pageCount -= atop_32(segLen);
2372 }
2373
2374 (void)hibernate_pal_callback;
2375
3a60a9f5
A		 2376 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2377
6d2010ae
A		 2378 pagesDone = 0;
2379 lastBlob = 0;
3a60a9f5 2380
db609669
A		 2381 HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
2382 bitmap_size, header->previewSize,
2383 pageCount, vars->fileVars->position);
3a60a9f5 2384
0b4c1975
A		 2385 enum
2386 // pageType
2387 {
2388 kWired = 0x02,
2389 kEncrypt = 0x01,
2390 kWiredEncrypt = kWired | kEncrypt,
2391 kWiredClear = kWired,
2392 kUnwiredEncrypt = kEncrypt
2393 };
2394
2395 for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)
3a60a9f5 2396 {
316670eb
A		 2397 if (kUnwiredEncrypt == pageType)
2398 {
2399 // start unwired image
7ddcb079 2400 vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
0b4c1975
A		 2401 vars->fileVars->encryptEnd = UINT64_MAX;
2402 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
316670eb
A		 2403 bcopy(&cryptvars->aes_iv[0],
2404 &gIOHibernateCryptWakeContext.aes_iv[0],
2405 sizeof(cryptvars->aes_iv));
2406 cryptvars = &gIOHibernateCryptWakeContext;
0b4c1975
A		 2407 }
2408 for (iterDone = false, ppnum = 0; !iterDone; )
2409 {
2410 count = hibernate_page_list_iterate((kWired & pageType)
2411 ? vars->page_list_wired : vars->page_list,
2412 &ppnum);
2413// kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
2414 iterDone = !count;
3a60a9f5 2415
0b4c1975 2416 if (count && (kWired & pageType) && needEncrypt)
3a60a9f5 2417 {
0b4c1975
A		 2418 uint32_t checkIndex;
2419 for (checkIndex = 0;
2420 (checkIndex < count)
2421 && (((kEncrypt & pageType) == 0) == no_encrypt_page(ppnum + checkIndex));
2422 checkIndex++)
2423 {}
2424 if (!checkIndex)
2425 {
2426 ppnum++;
2427 continue;
2428 }
2429 count = checkIndex;
3a60a9f5
A		 2430 }
2431
0b4c1975
A		 2432 switch (pageType)
2433 {
2434 case kWiredEncrypt: wired_pages_encrypted += count; break;
2435 case kWiredClear: wired_pages_clear += count; break;
2436 case kUnwiredEncrypt: dirty_pages_encrypted += count; break;
2437 }
2438
2439 if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */}
3a60a9f5 2440 else
3a60a9f5 2441 {
0b4c1975
A		 2442 pageAndCount[0] = ppnum;
2443 pageAndCount[1] = count;
2444 err = IOPolledFileWrite(vars->fileVars,
2445 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
2446 cryptvars);
2447 if (kIOReturnSuccess != err)
2448 break;
3a60a9f5 2449 }
0b4c1975
A		 2450
2451 for (page = ppnum; page < (ppnum + count); page++)
3a60a9f5 2452 {
0b4c1975
A		 2453 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
2454 if (err)
2455 {
2456 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
2457 break;
2458 }
2459
2460 sum = hibernate_sum_page(src, page);
2461 if (kWired & pageType)
2462 sum1 += sum;
2463 else
2464 sum2 += sum;
2465
2466 clock_get_uptime(&startTime);
2467
2468 pageCompressedSize = WKdm_compress ((WK_word*) src, (WK_word*) (src + page_size), PAGE_SIZE_IN_WORDS);
2469
3a60a9f5 2470 clock_get_uptime(&endTime);
0b4c1975
A		 2471 ADD_ABSOLUTETIME(&compTime, &endTime);
2472 SUB_ABSOLUTETIME(&compTime, &startTime);
2473 compBytes += page_size;
2474
2475 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2476 pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
2477
2478 if (pageCompressedSize > page_size)
3a60a9f5 2479 {
0b4c1975
A		 2480// HIBLOG("------------lose: %d\n", pageCompressedSize);
2481 pageCompressedSize = page_size;
2482 }
db609669 2483
0b4c1975
A		 2484 if (pageCompressedSize != page_size)
2485 data = (src + page_size);
2486 else
2487 data = src;
2488
2489 tag = pageCompressedSize | kIOHibernateTagSignature;
2490 err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
2491 if (kIOReturnSuccess != err)
2492 break;
2493
2494 err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
2495 if (kIOReturnSuccess != err)
2496 break;
2497
2498 compressedSize += pageCompressedSize;
2499 if (pageCompressedSize)
2500 uncompressedSize += page_size;
2501 pagesDone++;
2502
2503 if (vars->consoleMapping && (0 == (1023 & pagesDone)))
2504 {
2505 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
2506 if (blob != lastBlob)
2507 {
2508 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
2509 lastBlob = blob;
2510 }
2511 }
2512 if (0 == (8191 & pagesDone))
2513 {
2514 clock_get_uptime(&endTime);
2515 SUB_ABSOLUTETIME(&endTime, &allTime);
2516 absolutetime_to_nanoseconds(endTime, &nsec);
2517 progressStamp = nsec / 750000000ULL;
2518 if (progressStamp != lastProgressStamp)
2519 {
2520 lastProgressStamp = progressStamp;
2521 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
2522 }
3a60a9f5
A		 2523 }
2524 }
0b4c1975
A		 2525 if (kIOReturnSuccess != err)
2526 break;
2527 ppnum = page;
3a60a9f5 2528 }
0b4c1975 2529
3a60a9f5
A		 2530 if (kIOReturnSuccess != err)
2531 break;
0b4c1975
A		 2532
2533 if ((kEncrypt & pageType))
3a60a9f5 2534 {
7ddcb079 2535 vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
0b4c1975
A		 2536 HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
2537 }
2538
2539 if (kWiredEncrypt != pageType)
2540 {
2541 // end of image1/2 - fill to next block
3a60a9f5
A		 2542 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2543 if (kIOReturnSuccess != err)
2544 break;
0b4c1975
A		 2545 }
2546 if (kWiredClear == pageType)
2547 {
7ddcb079
A		 2548 // enlarge wired image for test
2549// err = IOPolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
2550
0b4c1975
A		 2551 // end wired image
2552 header->encryptStart = vars->fileVars->encryptStart;
2553 header->encryptEnd = vars->fileVars->encryptEnd;
3a60a9f5 2554 image1Size = vars->fileVars->position;
7ddcb079 2555 HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
0b4c1975 2556 image1Size, header->encryptStart, header->encryptEnd);
3a60a9f5
A		 2557 }
2558 }
3a60a9f5
A		 2559 if (kIOReturnSuccess != err)
2560 break;
2561
2562 // Header:
2563
2564 header->imageSize = vars->fileVars->position;
2565 header->image1Size = image1Size;
2566 header->bitmapSize = bitmap_size;
2567 header->pageCount = pageCount;
3a60a9f5
A		 2568
2569 header->restore1Sum = restore1Sum;
2570 header->image1Sum = sum1;
2571 header->image2Sum = sum2;
316670eb 2572 header->sleepTime = gIOLastSleepTime.tv_sec;
db609669
A		 2573
2574 header->compression = (compressedSize << 8) / uncompressedSize;
2575 gIOHibernateCompression = header->compression;
3a60a9f5
A		 2576
2577 count = vars->fileExtents->getLength();
2578 if (count > sizeof(header->fileExtentMap))
2579 {
2580 header->fileExtentMapSize = count;
2581 count = sizeof(header->fileExtentMap);
2582 }
2583 else
2584 header->fileExtentMapSize = sizeof(header->fileExtentMap);
2585 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
0b4c1975 2586
6d2010ae
A		 2587 header->deviceBase = vars->fileVars->block0;
2588
3a60a9f5
A		 2589 IOPolledFileSeek(vars->fileVars, 0);
2590 err = IOPolledFileWrite(vars->fileVars,
2591 (uint8_t *) header, sizeof(IOHibernateImageHeader),
2592 cryptvars);
2593 if (kIOReturnSuccess != err)
2594 break;
2595 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2596 if (kIOReturnSuccess != err)
2597 break;
2d21ac55 2598 err = IOHibernatePollerIODone(vars->fileVars, true);
3a60a9f5
A		 2599 if (kIOReturnSuccess != err)
2600 break;
2601 }
2602 while (false);
2603
2604 clock_get_uptime(&endTime);
b0d623f7
A		 2605
2606 IOService::getPMRootDomain()->pmStatsRecordEvent(
2607 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
2608
3a60a9f5
A		 2609 SUB_ABSOLUTETIME(&endTime, &allTime);
2610 absolutetime_to_nanoseconds(endTime, &nsec);
2611 HIBLOG("all time: %qd ms, ",
2612 nsec / 1000000ULL);
2613
2614 absolutetime_to_nanoseconds(compTime, &nsec);
6d2010ae
A		 2615 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2616 compBytes,
2617 nsec / 1000000ULL,
2618 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
3a60a9f5 2619
6d2010ae
A		 2620 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2621 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
2622 vars->fileVars->cryptBytes,
2623 nsec / 1000000ULL,
2624 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
3a60a9f5 2625
db609669
A		 2626 HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
2627 header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
3a60a9f5 2628 uncompressedSize, atop_32(uncompressedSize), compressedSize,
8f6c56a5 2629 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
3a60a9f5
A		 2630 sum1, sum2);
2631
0b4c1975
A		 2632 HIBLOG("wired_pages_encrypted %d, wired_pages_clear %d, dirty_pages_encrypted %d\n",
2633 wired_pages_encrypted, wired_pages_clear, dirty_pages_encrypted);
2634
c910b4d9
A		 2635 if (vars->fileVars->io)
2636 (void) IOHibernatePollerIODone(vars->fileVars, false);
2637
3a60a9f5
A		 2638 if (pollerOpen)
2639 IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
2640
2d21ac55
A		 2641 if (vars->consoleMapping)
2642 ProgressUpdate(gIOHibernateGraphicsInfo,
2643 vars->consoleMapping, 0, kIOHibernateProgressCount);
2644
3a60a9f5
A		 2645 HIBLOG("hibernate_write_image done(%x)\n", err);
2646
2647 // should we come back via regular wake, set the state in memory.
2648 gIOHibernateState = kIOHibernateStateInactive;
2649
0b4c1975
A		 2650 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END,
2651 wired_pages_encrypted, wired_pages_clear, dirty_pages_encrypted, 0, 0);
2652
2d21ac55
A		 2653 if (kIOReturnSuccess == err)
2654 {
2655 if (kIOHibernateModeSleep & gIOHibernateMode)
2656 {
2657 return (kIOHibernatePostWriteSleep);
2658 }
2659 else if(kIOHibernateModeRestart & gIOHibernateMode)
2660 {
2661 return (kIOHibernatePostWriteRestart);
2662 }
2663 else
2664 {
2665 /* by default, power down */
2666 return (kIOHibernatePostWriteHalt);
2667 }
2668 }
2669 else if (kIOReturnAborted == err)
2670 {
2671 return (kIOHibernatePostWriteWake);
2672 }
3a60a9f5 2673 else
3a60a9f5 2674 {
2d21ac55
A		 2675 /* on error, sleep */
2676 return (kIOHibernatePostWriteSleep);
3a60a9f5
A		 2677 }
2678}
2679
2680/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2681
2682extern "C" void
2683hibernate_machine_init(void)
2684{
2685 IOReturn err;
2686 uint32_t sum;
2687 uint32_t pagesDone;
0b4c1975 2688 uint32_t pagesRead = 0;
6d2010ae 2689 AbsoluteTime startTime, compTime;
3a60a9f5 2690 AbsoluteTime allTime, endTime;
6d2010ae 2691 uint64_t compBytes;
3a60a9f5
A		 2692 uint64_t nsec;
2693 uint32_t lastProgressStamp = 0;
2694 uint32_t progressStamp;
3a60a9f5
A		 2695 hibernate_cryptvars_t * cryptvars = 0;
2696
2697 IOHibernateVars * vars = &gIOHibernateVars;
2698
2699 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
2700 return;
2701
3a60a9f5
A		 2702 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2703 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2704
2705 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, image1Size %qx, conflictCount %d, nextFree %x\n",
2706 gIOHibernateState, pagesDone, sum, gIOHibernateCurrentHeader->image1Size,
2707 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
2708
2709 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
2710 {
2711 HIBLOG("regular wake\n");
2712 return;
2713 }
2714
2715 HIBPRINT("diag %x %x %x %x\n",
2716 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
6d2010ae 2717 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
3a60a9f5 2718
0c530ab8
A		 2719 if ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)
2720 hibernate_page_list_discard(vars->page_list);
2721
6d2010ae
A		 2722 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
2723
2724 if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount)
2725 panic("handoff overflow");
2726
2727 IOHibernateHandoff * handoff;
2728 bool done = false;
2729 bool foundCryptData = false;
2730
2731 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
2732 !done;
2733 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
2734 {
2735// HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
2736 uint8_t * data = &handoff->data[0];
2737 switch (handoff->type)
2738 {
2739 case kIOHibernateHandoffTypeEnd:
2740 done = true;
2741 break;
2742
2743 case kIOHibernateHandoffTypeGraphicsInfo:
2744 bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
2745 break;
2746
2747 case kIOHibernateHandoffTypeCryptVars:
2748 if (cryptvars)
2749 {
2750 hibernate_cryptwakevars_t *
2751 wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
2752 bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
2753 }
2754 foundCryptData = true;
2755 bzero(data, handoff->bytecount);
2756 break;
2757
2758 case kIOHibernateHandoffTypeMemoryMap:
2759 hibernate_newruntime_map(data, handoff->bytecount,
2760 gIOHibernateCurrentHeader->systemTableOffset);
2761 break;
2762
2763 case kIOHibernateHandoffTypeDeviceTree:
2764 {
2765// DTEntry chosen = NULL;
2766// HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen));
2767 }
2768 break;
2769
2770 default:
2771 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
2772 break;
2773 }
2774 }
2775 if (cryptvars && !foundCryptData)
2776 panic("hibernate handoff");
2777
316670eb
A		 2778 HIBPRINT("video %x %d %d %d status %x\n",
2779 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2780 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
2781
2782 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)
3a60a9f5
A		 2783 {
2784 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2785 * gIOHibernateGraphicsInfo->rowBytes);
2786 IOMapPages(kernel_map,
2787 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2788 vars->videoMapSize, kIOMapInhibitCache );
2789 }
2790
316670eb
A		 2791 if (vars->videoMapSize)
2792 ProgressUpdate(gIOHibernateGraphicsInfo,
2793 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2794
0c530ab8 2795 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
3a60a9f5
A		 2796 uint32_t decoOffset;
2797
2798 clock_get_uptime(&allTime);
6d2010ae
A		 2799 AbsoluteTime_to_scalar(&compTime) = 0;
2800 compBytes = 0;
3a60a9f5
A		 2801
2802 HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2803 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0);
2804 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
2805
3a60a9f5
A		 2806 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2807
3a60a9f5
A		 2808 // kick off the read ahead
2809 vars->fileVars->io = false;
2810 vars->fileVars->bufferHalf = 0;
2811 vars->fileVars->bufferLimit = 0;
2812 vars->fileVars->lastRead = 0;
6d2010ae 2813 vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
3a60a9f5 2814 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
6d2010ae
A		 2815 vars->fileVars->cryptBytes = 0;
2816 AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
3a60a9f5 2817
6d2010ae 2818 err = IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
3a60a9f5
A		 2819 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2820 // --
2821
2822 HIBLOG("hibernate_machine_init reading\n");
2823
2824 uint32_t * header = (uint32_t *) src;
2825 sum = 0;
2826
6d2010ae 2827 while (kIOReturnSuccess == err)
3a60a9f5
A		 2828 {
2829 unsigned int count;
2830 unsigned int page;
2831 uint32_t tag;
2832 vm_offset_t ppnum, compressedSize;
2833
36401178
A		 2834 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2835 if (kIOReturnSuccess != err)
2836 break;
3a60a9f5
A		 2837
2838 ppnum = header[0];
2839 count = header[1];
2840
2841// HIBPRINT("(%x, %x)\n", ppnum, count);
2842
2843 if (!count)
2844 break;
2845
2846 for (page = 0; page < count; page++)
2847 {
36401178
A		 2848 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2849 if (kIOReturnSuccess != err)
2850 break;
3a60a9f5
A		 2851
2852 compressedSize = kIOHibernateTagLength & tag;
36401178
A		 2853 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength))
2854 {
2855 err = kIOReturnIPCError;
2856 break;
2857 }
2858
3a60a9f5
A		 2859 if (!compressedSize)
2860 {
2861 ppnum++;
2862 pagesDone++;
2863 continue;
2864 }
2865
36401178
A		 2866 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2867 if (kIOReturnSuccess != err)
2868 break;
2869
2870 if (compressedSize < page_size)
3a60a9f5
A		 2871 {
2872 decoOffset = page_size;
6d2010ae
A		 2873
2874 clock_get_uptime(&startTime);
3a60a9f5 2875 WKdm_decompress((WK_word*) src, (WK_word*) (src + decoOffset), PAGE_SIZE_IN_WORDS);
6d2010ae
A		 2876 clock_get_uptime(&endTime);
2877 ADD_ABSOLUTETIME(&compTime, &endTime);
2878 SUB_ABSOLUTETIME(&compTime, &startTime);
2879
2880 compBytes += page_size;
3a60a9f5
A		 2881 }
2882 else
2883 decoOffset = 0;
2884
0b4c1975 2885 sum += hibernate_sum_page((src + decoOffset), ppnum);
3a60a9f5
A		 2886
2887 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2888 if (err)
36401178 2889 {
b0d623f7 2890 HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
36401178
A		 2891 break;
2892 }
3a60a9f5
A		 2893
2894 ppnum++;
2895 pagesDone++;
0b4c1975 2896 pagesRead++;
3a60a9f5 2897
3a60a9f5
A		 2898 if (0 == (8191 & pagesDone))
2899 {
2900 clock_get_uptime(&endTime);
2901 SUB_ABSOLUTETIME(&endTime, &allTime);
2902 absolutetime_to_nanoseconds(endTime, &nsec);
2903 progressStamp = nsec / 750000000ULL;
2904 if (progressStamp != lastProgressStamp)
2905 {
2906 lastProgressStamp = progressStamp;
2907 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2908 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2909 }
2910 }
2911 }
2912 }
7ddcb079
A		 2913 if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages))
2914 err = kIOReturnLockedRead;
3a60a9f5 2915
36401178
A		 2916 if (kIOReturnSuccess != err)
2917 panic("Hibernate restore error %x", err);
2918
3a60a9f5 2919 gIOHibernateCurrentHeader->actualImage2Sum = sum;
db609669 2920 gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
3a60a9f5
A		 2921
2922 if (vars->fileVars->io)
2d21ac55 2923 (void) IOHibernatePollerIODone(vars->fileVars, false);
3a60a9f5
A		 2924
2925 err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
2926
3a60a9f5 2927 clock_get_uptime(&endTime);
b0d623f7
A		 2928
2929 IOService::getPMRootDomain()->pmStatsRecordEvent(
2930 kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
2931 IOService::getPMRootDomain()->pmStatsRecordEvent(
2932 kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
2933
3a60a9f5
A		 2934 SUB_ABSOLUTETIME(&endTime, &allTime);
2935 absolutetime_to_nanoseconds(endTime, &nsec);
2936
6d2010ae 2937 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %qd ms, ",
3a60a9f5 2938 pagesDone, sum, nsec / 1000000ULL);
6d2010ae
A		 2939
2940 absolutetime_to_nanoseconds(compTime, &nsec);
2941 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2942 compBytes,
2943 nsec / 1000000ULL,
2944 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2945
2946 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2947 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
2948 vars->fileVars->cryptBytes,
2949 nsec / 1000000ULL,
2950 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2951
2952 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_NONE, pagesRead, pagesDone, 0, 0, 0);
3a60a9f5
A		 2953}
2954
2955/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
99c3a104
A		 2956
2957void IOHibernateSystemRestart(void)
2958{
2959 static uint8_t noteStore[32] __attribute__((aligned(32)));
2960 IORegistryEntry * regEntry;
2961 const OSSymbol * sym;
2962 OSData * noteProp;
2963 OSData * data;
2964 uintptr_t * smcVars;
2965 uint8_t * smcBytes;
2966 size_t len;
2967 addr64_t element;
2968
2969 data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
2970 if (!data) return;
2971
2972 smcVars = (typeof(smcVars)) data->getBytesNoCopy();
2973 smcBytes = (typeof(smcBytes)) smcVars[1];
2974 len = smcVars[0];
2975 if (len > sizeof(noteStore)) len = sizeof(noteStore);
2976 noteProp = OSData::withCapacity(3 * sizeof(element));
2977 if (!noteProp) return;
2978 element = len;
2979 noteProp->appendBytes(&element, sizeof(element));
2980 element = crc32(0, smcBytes, len);
2981 noteProp->appendBytes(&element, sizeof(element));
2982
2983 bcopy(smcBytes, noteStore, len);
2984 element = (addr64_t) &noteStore[0];
2985 element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
2986 noteProp->appendBytes(&element, sizeof(element));
2987
2988 if (!gIOOptionsEntry)
2989 {
2990 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
2991 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
2992 if (regEntry && !gIOOptionsEntry)
2993 regEntry->release();
2994 }
2995
2996 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
2997 if (gIOOptionsEntry && sym) gIOOptionsEntry->setProperty(sym, noteProp);
2998 if (noteProp) noteProp->release();
2999 if (sym) sym->release();
3000}
3001
3002
3003
mirror of XNU