]> git.saurik.com Git - apple/xnu.git/blobdiff - iokit/Kernel/IOHibernateIO.cpp
xnu-6153.11.26.tar.gz
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
index 9105cdbb9f362b67c627f0349fe9b335f810fc6a..3b696609a311b7cbc26f6efe96a33a40f2eb6e9b 100644 (file)
@@ -1,8 +1,8 @@
 /*
- * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2016 Apple Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
- * 
+ *
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
- * 
+ *
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
- * 
+ *
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
- * 
+ *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
-
 /*
-
-Sleep:
-
-- PMRootDomain calls IOHibernateSystemSleep() before system sleep
-(devices awake, normal execution context)
-- IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level, 
-  grabs its extents and searches for a polling driver willing to work with that IOMedia.
-  The BSD code makes an ioctl to the storage driver to get the partition base offset to
-  the disk, and other ioctls to get the transfer constraints 
-  If successful, the file is written to make sure its initially not bootable (in case of
-  later failure) and nvram set to point to the first block of the file. (Has to be done
-  here so blocking is possible in nvram support).
-  hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
-  page out any pages it wants to (currently zero, but probably some percentage of memory).
-  Its assumed just allocating pages will cause the VM system to naturally select the best
-  pages for eviction. It also copies processor flags needed for the restore path and sets
-  a flag in the boot processor proc info.
-  gIOHibernateState = kIOHibernateStateHibernating.
-- Regular sleep progresses - some drivers may inspect the root domain property 
-  kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
-  as usual but leaves motherboard I/O on.
-- Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
-  at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
-  all ppc RC bits out of the hash table and caches into the mapping structures.
-- hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
-  hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
-  All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
-  by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
-  The image header and block list are written. The header includes the second file extent so
-  only the header block is needed to read the file, regardless of filesystem.
-  The kernel section "__HIB" is written uncompressed to the image. This section of code and data 
-  (only) is used to decompress the image during wake/boot.
-  Some additional pages are removed from the bitmaps - the buffers used for hibernation.
-  The bitmaps are written to the image.
-  More areas are removed from the bitmaps (after they have been written to the image) - the 
 section "__HIB" pages and interrupt stack.
-  Each wired page is compressed and written and then each non-wired page. Compression and 
-  disk writes are in parallel.
-  The image header is written to the start of the file and the polling driver closed.
-  The machine powers down (or sleeps).
-  
-Boot/Wake:
-
-- BootX sees the boot-image nvram variable containing the device and block number of the image,
-  reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
-- BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
-  in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
-  that is in the kernel's __HIB section.
-- BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
-  hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off, 
-  only code & data in that section is safe to call since all the other wired pages are still 
-  compressed in the image.
-- hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
-  It uses the bitmaps to work out which pages can be uncompressed from the image to their final
-  location directly, and copies those that can't to interim free pages. When the image has been
-  completed, the copies are uncompressed, overwriting the wired image pages.
-  hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
-  is used to get pages into place for 64bit.
-- the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
-  changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
-  are removed from the software strutures, and the hash table is reinitialized. 
-- After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
-  of memory, using the polled mode driver, before other threads can run or any devices are turned on.
-  This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
-  for the remaining non wired pages. 
-- The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
-  (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
-  is closed via bsd.
-
-Polled Mode I/O:
-
-IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
-registry, specifying an object of calls IOPolledInterface.
-
-Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
-partition) that the image is going to live, looking for polled interface properties. If it finds
-one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
-interfaces found are kept in an ordered list.
-
-There is an Open/Close pair of calls made to each of the interfaces at various stages since there are 
-few different contexts things happen in:
-
-- there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
-up and running) and after wake - this is safe to allocate memory and do anything. The device
-ignores sleep requests from that point since its a waste of time if it goes to sleep and
-immediately wakes back up for the image write.
-
-- there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
-immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
-from the low level bits (motherboard I/O etc). There is only one thread running. The close can be 
-used to flush and set the disk to sleep.
-
-- there is an Open/Close (AfterSleep) pair made around the image read operations that happen
-immediately after sleep. These can't block or allocate memory. This is happening after the platform
-expert has woken the low level bits of the system, but most of the I/O system has not. There is only
-one thread running.
-
-For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
-(prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
-the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
-Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
-that is called for the hardware to check for events, and complete the I/O via the callback.
-The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
-to restrict I/O ops.
-*/
+ *
+ *  Sleep:
+ *
+ *  - PMRootDomain calls IOHibernateSystemSleep() before system sleep
+ *  (devices awake, normal execution context)
+ *  - IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
 grabs its extents and searches for a polling driver willing to work with that IOMedia.
 The BSD code makes an ioctl to the storage driver to get the partition base offset to
+ *  the disk, and other ioctls to get the transfer constraints
 If successful, the file is written to make sure its initially not bootable (in case of
 later failure) and nvram set to point to the first block of the file. (Has to be done
 here so blocking is possible in nvram support).
 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
 page out any pages it wants to (currently zero, but probably some percentage of memory).
 Its assumed just allocating pages will cause the VM system to naturally select the best
 pages for eviction. It also copies processor flags needed for the restore path and sets
 a flag in the boot processor proc info.
 gIOHibernateState = kIOHibernateStateHibernating.
+ *  - Regular sleep progresses - some drivers may inspect the root domain property
 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
 as usual but leaves motherboard I/O on.
+ *  - Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
 all ppc RC bits out of the hash table and caches into the mapping structures.
+ *  - hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
 The image header and block list are written. The header includes the second file extent so
 only the header block is needed to read the file, regardless of filesystem.
+ *  The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
 (only) is used to decompress the image during wake/boot.
 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
 The bitmaps are written to the image.
+ *  More areas are removed from the bitmaps (after they have been written to the image) - the
*  segment "__HIB" pages and interrupt stack.
+ *  Each wired page is compressed and written and then each non-wired page. Compression and
 disk writes are in parallel.
 The image header is written to the start of the file and the polling driver closed.
 The machine powers down (or sleeps).
+ *
+ *  Boot/Wake:
+ *
+ *  - BootX sees the boot-image nvram variable containing the device and block number of the image,
 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
+ *  - BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
 that is in the kernel's __HIB section.
+ *  - BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
+ *  hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
+ *  only code & data in that section is safe to call since all the other wired pages are still
 compressed in the image.
+ *  - hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
 location directly, and copies those that can't to interim free pages. When the image has been
 completed, the copies are uncompressed, overwriting the wired image pages.
 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
 is used to get pages into place for 64bit.
+ *  - the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
+ *  are removed from the software strutures, and the hash table is reinitialized.
+ *  - After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
+ *  for the remaining non wired pages.
+ *  - The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
 is closed via bsd.
+ *
+ *  Polled Mode I/O:
+ *
+ *  IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
+ *  registry, specifying an object of calls IOPolledInterface.
+ *
+ *  Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
+ *  partition) that the image is going to live, looking for polled interface properties. If it finds
+ *  one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
+ *  interfaces found are kept in an ordered list.
+ *
+ *  There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
+ *  few different contexts things happen in:
+ *
+ *  - there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
+ *  up and running) and after wake - this is safe to allocate memory and do anything. The device
+ *  ignores sleep requests from that point since its a waste of time if it goes to sleep and
+ *  immediately wakes back up for the image write.
+ *
+ *  - there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
+ *  immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
+ *  from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
+ *  used to flush and set the disk to sleep.
+ *
+ *  - there is an Open/Close (AfterSleep) pair made around the image read operations that happen
+ *  immediately after sleep. These can't block or allocate memory. This is happening after the platform
+ *  expert has woken the low level bits of the system, but most of the I/O system has not. There is only
+ *  one thread running.
+ *
+ *  For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
+ *  (prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
+ *  the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
+ *  Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
+ *  that is called for the hardware to check for events, and complete the I/O via the callback.
+ *  The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
+ *  to restrict I/O ops.
+ */
 
 #include <sys/systm.h>
 
@@ -147,85 +146,104 @@ to restrict I/O ops.
 #include <IOKit/IOMessage.h>
 #include <IOKit/IODeviceTreeSupport.h>
 #include <IOKit/IOBSD.h>
+#include <IOKit/IOKitKeysPrivate.h>
 #include "RootDomainUserClient.h"
 #include <IOKit/pwr_mgt/IOPowerConnection.h>
 #include "IOPMPowerStateQueue.h"
 #include <IOKit/IOBufferMemoryDescriptor.h>
-#include <crypto/aes.h>
+#include <IOKit/AppleKeyStoreInterface.h>
+#include <libkern/crypto/aes.h>
 
 #include <sys/uio.h>
 #include <sys/conf.h>
 #include <sys/stat.h>
 #include <sys/fcntl.h>                       // (FWRITE, ...)
 #include <sys/sysctl.h>
+#include <sys/kdebug.h>
+#include <stdint.h>
 
 #include <IOKit/IOHibernatePrivate.h>
 #include <IOKit/IOPolledInterface.h>
 #include <IOKit/IONVRAM.h>
 #include "IOHibernateInternal.h"
-#include "WKdm.h"
+#include <vm/WKdm_new.h>
+#include <vm/vm_protos.h>
 #include "IOKitKernelInternal.h"
+#include <pexpert/device_tree.h>
 
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+#include <machine/pal_routines.h>
+#include <machine/pal_hibernate.h>
+#include <i386/tsc.h>
+#include <i386/cpuid.h>
+#include <san/kasan.h>
 
-OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
-
-OSMetaClassDefineReservedUnused(IOPolledInterface, 0);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
+extern "C" addr64_t             kvtophys(vm_offset_t va);
+extern "C" ppnum_t              pmap_find_phys(pmap_t pmap, addr64_t va);
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
-extern uint32_t                gIOHibernateState;
-uint32_t                       gIOHibernateMode;
-static char                    gIOHibernateBootSignature[256+1];
-static char                    gIOHibernateFilename[MAXPATHLEN+1];
-static uint32_t                        gIOHibernateFreeRatio = 0;              // free page target (percent)
-uint32_t                       gIOHibernateFreeTime  = 0*1000; // max time to spend freeing pages (ms)
-
-static IODTNVRAM *             gIOOptionsEntry;
-static IORegistryEntry *       gIOChosenEntry;
-#ifdef __i386__
-static const OSSymbol *         gIOCreateEFIDevicePathSymbol;
-#endif
+#define DISABLE_TRIM            0
+#define TRIM_DELAY              25000
+
+extern unsigned int             save_kdebug_enable;
+extern uint32_t                 gIOHibernateState;
+uint32_t                        gIOHibernateMode;
+static char                     gIOHibernateBootSignature[256 + 1];
+static char                     gIOHibernateFilename[MAXPATHLEN + 1];
+
+static uuid_string_t            gIOHibernateBridgeBootSessionUUIDString;
+
+static uint32_t                 gIOHibernateFreeRatio = 0;       // free page target (percent)
+uint32_t                        gIOHibernateFreeTime  = 0 * 1000;  // max time to spend freeing pages (ms)
+static uint64_t                 gIOHibernateCompression = 0x80;  // default compression 50%
+boolean_t                       gIOHibernateStandbyDisabled;
+
+static IODTNVRAM *              gIOOptionsEntry;
+static IORegistryEntry *        gIOChosenEntry;
+
+static const OSSymbol *         gIOHibernateBootImageKey;
+static const OSSymbol *         gIOHibernateBootSignatureKey;
+static const OSSymbol *         gIOBridgeBootSessionUUIDKey;
+
+#if defined(__i386__) || defined(__x86_64__)
+
+static const OSSymbol *         gIOHibernateRTCVariablesKey;
+static const OSSymbol *         gIOHibernateBoot0082Key;
+static const OSSymbol *         gIOHibernateBootNextKey;
+static OSData *                 gIOHibernateBoot0082Data;
+static OSData *                 gIOHibernateBootNextData;
+static OSObject *               gIOHibernateBootNextSave;
+
+#endif /* defined(__i386__) || defined(__x86_64__) */
+
+static IOLock *                           gFSLock;
+uint32_t                           gFSState;
+static thread_call_t                      gIOHibernateTrimCalloutEntry;
+static IOPolledFileIOVars                 gFileVars;
+static IOHibernateVars                    gIOHibernateVars;
+static IOPolledFileCryptVars              gIOHibernateCryptWakeContext;
+static hibernate_graphics_t               _hibernateGraphics;
+static hibernate_graphics_t *             gIOHibernateGraphicsInfo = &_hibernateGraphics;
+static hibernate_statistics_t             _hibernateStats;
+static hibernate_statistics_t *           gIOHibernateStats = &_hibernateStats;
+
+enum{
+       kFSIdle      = 0,
+       kFSOpening   = 2,
+       kFSOpened    = 3,
+       kFSTimedOut  = 4,
+       kFSTrimDelay = 5
+};
 
-static IOPolledFileIOVars                gFileVars;
-static IOHibernateVars                   gIOHibernateVars;
-static struct kern_direct_file_io_ref_t * gIOHibernateFileRef;
-static hibernate_cryptvars_t             gIOHibernateCryptWakeContext;
+static IOReturn IOHibernateDone(IOHibernateVars * vars);
+static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature);
+static void     IOSetBootImageNVRAM(OSData * data);
+static void     IOHibernateSystemPostWakeTrim(void * p1, void * p2);
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
-enum { kXPRamAudioVolume = 8 };
 enum { kDefaultIOSize = 128 * 1024 };
-enum { kVideoMapSize  = 32 * 1024 * 1024 };
-
-#ifndef kIOMediaPreferredBlockSizeKey
-#define kIOMediaPreferredBlockSizeKey  "Preferred Block Size"
-#endif
-
-#ifndef kIOBootPathKey 
-#define kIOBootPathKey                 "bootpath"
-#endif
-#ifndef kIOSelectedBootDeviceKey       
-#define kIOSelectedBootDeviceKey       "boot-device"
-#endif
-
-
-enum { kIOHibernateMinPollersNeeded = 2 };
+enum { kVideoMapSize  = 80 * 1024 * 1024 };
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
@@ -233,1095 +251,642 @@ enum { kIOHibernateMinPollersNeeded = 2 };
 
 static IOReturn
 IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
-                                   IOByteCount offset, addr64_t bytes, IOByteCount length)
+    IOByteCount offset, addr64_t bytes, IOByteCount length)
 {
-    addr64_t srcAddr = bytes;
-    IOByteCount remaining;
+       addr64_t srcAddr = bytes;
+       IOByteCount remaining;
 
-    remaining = length = min(length, md->getLength() - offset);
-    while (remaining) {        // (process another target segment?)
-        addr64_t    dstAddr64;
-        IOByteCount dstLen;
+       remaining = length = min(length, md->getLength() - offset);
+       while (remaining) { // (process another target segment?)
+               addr64_t    dstAddr64;
+               IOByteCount dstLen;
 
-        dstAddr64 = md->getPhysicalSegment64(offset, &dstLen);
-        if (!dstAddr64)
-            break;
+               dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
+               if (!dstAddr64) {
+                       break;
+               }
 
-        // Clip segment length to remaining
-        if (dstLen > remaining)
-            dstLen = remaining;
+               // Clip segment length to remaining
+               if (dstLen > remaining) {
+                       dstLen = remaining;
+               }
 
 #if 1
-       bcopy_phys(srcAddr, dstAddr64, dstLen);
+               bcopy_phys(srcAddr, dstAddr64, dstLen);
 #else
-        copypv(srcAddr, dstAddr64, dstLen,
-                            cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
+               copypv(srcAddr, dstAddr64, dstLen,
+                   cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
 #endif
-        srcAddr   += dstLen;
-        offset    += dstLen;
-        remaining -= dstLen;
-    }
+               srcAddr   += dstLen;
+               offset    += dstLen;
+               remaining -= dstLen;
+       }
 
-    assert(!remaining);
+       assert(!remaining);
 
-    return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
+       return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
 }
 
 // copy from MD to phys addr
 
 static IOReturn
 IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
-                                IOByteCount offset, addr64_t bytes, IOByteCount length)
+    IOByteCount offset, addr64_t bytes, IOByteCount length)
 {
-    addr64_t dstAddr = bytes;
-    IOByteCount remaining;
+       addr64_t dstAddr = bytes;
+       IOByteCount remaining;
 
-    remaining = length = min(length, md->getLength() - offset);
-    while (remaining) {        // (process another target segment?)
-        addr64_t    srcAddr64;
-        IOByteCount dstLen;
+       remaining = length = min(length, md->getLength() - offset);
+       while (remaining) { // (process another target segment?)
+               addr64_t    srcAddr64;
+               IOByteCount dstLen;
 
-        srcAddr64 = md->getPhysicalSegment64(offset, &dstLen);
-        if (!srcAddr64)
-            break;
+               srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
+               if (!srcAddr64) {
+                       break;
+               }
 
-        // Clip segment length to remaining
-        if (dstLen > remaining)
-            dstLen = remaining;
+               // Clip segment length to remaining
+               if (dstLen > remaining) {
+                       dstLen = remaining;
+               }
 
 #if 1
-       bcopy_phys(srcAddr64, dstAddr, dstLen);
+               bcopy_phys(srcAddr64, dstAddr, dstLen);
 #else
-        copypv(srcAddr, dstAddr64, dstLen,
-                            cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
+               copypv(srcAddr, dstAddr64, dstLen,
+                   cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
 #endif
-        dstAddr    += dstLen;
-        offset     += dstLen;
-        remaining  -= dstLen;
-    }
+               dstAddr    += dstLen;
+               offset     += dstLen;
+               remaining  -= dstLen;
+       }
 
-    assert(!remaining);
+       assert(!remaining);
 
-    return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
+       return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 void
 hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
-                               vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
+    vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
 {
-    count += ppnum;
-    switch (kind)
-    {
-      case kIOHibernatePageStateUnwiredSave:
-       // unwired save
-       for (; ppnum < count; ppnum++)
-       {
-           hibernate_page_bitset(page_list,       FALSE, ppnum);
-           hibernate_page_bitset(page_list_wired, TRUE,  ppnum);
-       }
-       break;
-      case kIOHibernatePageStateWiredSave:
-       // wired save
-       for (; ppnum < count; ppnum++)
-       {
-           hibernate_page_bitset(page_list,       FALSE, ppnum);
-           hibernate_page_bitset(page_list_wired, FALSE, ppnum);
-       }
-       break;
-      case kIOHibernatePageStateFree:
-       // free page
-       for (; ppnum < count; ppnum++)
-       {
-           hibernate_page_bitset(page_list,       TRUE, ppnum);
-           hibernate_page_bitset(page_list_wired, TRUE, ppnum);
-       }
-       break;
-      default:
-       panic("hibernate_set_page_state");
-    }
+       count += ppnum;
+       switch (kind) {
+       case kIOHibernatePageStateUnwiredSave:
+               // unwired save
+               for (; ppnum < count; ppnum++) {
+                       hibernate_page_bitset(page_list, FALSE, ppnum);
+                       hibernate_page_bitset(page_list_wired, TRUE, ppnum);
+               }
+               break;
+       case kIOHibernatePageStateWiredSave:
+               // wired save
+               for (; ppnum < count; ppnum++) {
+                       hibernate_page_bitset(page_list, FALSE, ppnum);
+                       hibernate_page_bitset(page_list_wired, FALSE, ppnum);
+               }
+               break;
+       case kIOHibernatePageStateFree:
+               // free page
+               for (; ppnum < count; ppnum++) {
+                       hibernate_page_bitset(page_list, TRUE, ppnum);
+                       hibernate_page_bitset(page_list_wired, TRUE, ppnum);
+               }
+               break;
+       default:
+               panic("hibernate_set_page_state");
+       }
 }
 
 static vm_offset_t
 hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
 {
-    uint32_t            page = *pPage;
-    uint32_t            count;
-    hibernate_bitmap_t * bitmap;
-
-    while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
-    {
-       count = hibernate_page_bitmap_count(bitmap, TRUE, page);
-       if (!count)
-           break;
-       page += count;
-       if (page <= bitmap->last_page)
-           break;
-    }
-
-    *pPage = page;
-    if (bitmap)
-       count = hibernate_page_bitmap_count(bitmap, FALSE, page);
-    else
-       count = 0;
-
-    return (count);
-}
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+       uint32_t             page = *pPage;
+       uint32_t             count;
+       hibernate_bitmap_t * bitmap;
+
+       while ((bitmap = hibernate_page_bitmap_pin(list, &page))) {
+               count = hibernate_page_bitmap_count(bitmap, TRUE, page);
+               if (!count) {
+                       break;
+               }
+               page += count;
+               if (page <= bitmap->last_page) {
+                       break;
+               }
+       }
 
-static IOReturn
-IOHibernatePollerProbe(IOPolledFileIOVars * vars, IOService * target)
-{
-    IOReturn            err = kIOReturnError;
-    int32_t            idx;
-    IOPolledInterface * poller;
-
-    for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
-    {
-        poller = (IOPolledInterface *) vars->pollers->getObject(idx);
-        err = poller->probe(target);
-        if (err)
-        {
-            HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
-            break;
-        }
-    }
-
-    return (err);
-}
+       *pPage = page;
+       if (bitmap) {
+               count = hibernate_page_bitmap_count(bitmap, FALSE, page);
+       } else {
+               count = 0;
+       }
 
-static IOReturn
-IOHibernatePollerOpen(IOPolledFileIOVars * vars, uint32_t state, IOMemoryDescriptor * md)
-{
-    IOReturn            err = kIOReturnError;
-    int32_t            idx;
-    IOPolledInterface * poller;
-
-    for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
-    {
-        poller = (IOPolledInterface *) vars->pollers->getObject(idx);
-        err = poller->open(state, md);
-        if (err)
-        {
-            HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
-            break;
-        }
-    }
-
-    return (err);
+       return count;
 }
 
-static IOReturn
-IOHibernatePollerClose(IOPolledFileIOVars * vars, uint32_t state)
-{
-    IOReturn            err = kIOReturnError;
-    int32_t            idx;
-    IOPolledInterface * poller;
-
-    for (idx = 0;
-         (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
-         idx++)
-    {
-        err = poller->close(state);
-        if (err)
-            HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
-    }
-
-    return (err);
-}
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
-static void
-IOHibernatePollerIOComplete(void *   target,
-                            void *   parameter,
-                            IOReturn status,
-                            UInt64   actualByteCount)
+IOReturn
+IOHibernateSystemSleep(void)
 {
-    IOPolledFileIOVars * vars = (IOPolledFileIOVars *) parameter;
+       IOReturn   err;
+       OSData *   nvramData;
+       OSObject * obj;
+       OSString * str;
+       OSNumber * num;
+       bool       dsSSD, vmflush, swapPinned;
+       IOHibernateVars * vars;
+       uint64_t   setFileSize = 0;
+
+       gIOHibernateState = kIOHibernateStateInactive;
+
+       gIOHibernateDebugFlags = 0;
+       if (kIOLogHibernate & gIOKitDebug) {
+               gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
+       }
 
-    vars->ioStatus = status;
-}
+       if (IOService::getPMRootDomain()->getHibernateSettings(
+                   &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime)) {
+               if (kIOHibernateModeSleep & gIOHibernateMode) {
+                       // default to discard clean for safe sleep
+                       gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
+                           | kIOHibernateModeDiscardCleanActive);
+               }
+       }
 
-static IOReturn
-IOHibernatePollerIO(IOPolledFileIOVars * vars, 
-                    uint32_t operation, uint32_t bufferOffset, 
-                   uint64_t deviceOffset, uint64_t length)
-{
+       if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey))) {
+               if ((str = OSDynamicCast(OSString, obj))) {
+                       strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
+                           sizeof(gIOHibernateFilename));
+               }
+               obj->release();
+       }
 
-    IOReturn            err = kIOReturnError;
-    IOPolledInterface * poller;
-    IOPolledCompletion  completion;
+       if (!gIOHibernateMode || !gIOHibernateFilename[0]) {
+               return kIOReturnUnsupported;
+       }
 
-    completion.target    = 0;
-    completion.action    = &IOHibernatePollerIOComplete;
-    completion.parameter = vars;
+       HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
 
-    vars->ioStatus = -1;
+       vars = IONew(IOHibernateVars, 1);
+       if (!vars) {
+               return kIOReturnNoMemory;
+       }
+       bzero(vars, sizeof(*vars));
 
-    poller = (IOPolledInterface *) vars->pollers->getObject(0);
-    err = poller->startIO(operation, bufferOffset, deviceOffset + vars->block0, length, completion);
-    if (err)
-        HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
+       IOLockLock(gFSLock);
+       if (!gIOHibernateTrimCalloutEntry) {
+               gIOHibernateTrimCalloutEntry = thread_call_allocate(&IOHibernateSystemPostWakeTrim, &gFSLock);
+       }
+       IOHibernateSystemPostWakeTrim(NULL, NULL);
+       thread_call_cancel(gIOHibernateTrimCalloutEntry);
+       if (kFSIdle != gFSState) {
+               HIBLOG("hibernate file busy\n");
+               IOLockUnlock(gFSLock);
+               IODelete(vars, IOHibernateVars, 1);
+               return kIOReturnBusy;
+       }
+       gFSState = kFSOpening;
+       IOLockUnlock(gFSLock);
 
-    return (err);
-}
+       swapPinned = false;
+       do{
+               vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
+                   2 * page_size + WKdm_SCRATCH_BUF_SIZE_INTERNAL, page_size);
 
-static IOReturn
-IOHibernatePollerIODone(IOPolledFileIOVars * vars, bool abortable)
-{
-    IOReturn            err = kIOReturnSuccess;
-    int32_t            idx = 0;
-    IOPolledInterface * poller;
-
-    while (-1 == vars->ioStatus)
-    {
-        for (idx = 0; 
-           (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
-             idx++)
-        {
-           IOReturn newErr;
-            newErr = poller->checkForWork();
-           if ((newErr == kIOReturnAborted) && !abortable)
-               newErr = kIOReturnSuccess;
-           if (kIOReturnSuccess == err)
-               err = newErr;
-        }
-    }
-
-    if (err)
-    {
-       HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
-    }
-    else 
-    {
-       err = vars->ioStatus;
-       if (kIOReturnSuccess != err)
-           HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
-    }
-
-    return (err);
-}
+               vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
+                   ptoa_64(gIOHibernateHandoffPageCount), page_size);
 
-IOReturn
-IOPolledInterface::checkAllForWork(void)
-{
-    IOReturn            err = kIOReturnNotReady;
-    int32_t            idx;
-    IOPolledInterface * poller;
+               if (!vars->srcBuffer || !vars->handoffBuffer) {
+                       err = kIOReturnNoMemory;
+                       break;
+               }
 
-    IOHibernateVars * vars  = &gIOHibernateVars;
+               if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey))) {
+                       if ((num = OSDynamicCast(OSNumber, obj))) {
+                               vars->fileMinSize = num->unsigned64BitValue();
+                       }
+                       obj->release();
+               }
+               if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey))) {
+                       if ((num = OSDynamicCast(OSNumber, obj))) {
+                               vars->fileMaxSize = num->unsigned64BitValue();
+                       }
+                       obj->release();
+               }
 
-    if (!vars->fileVars || !vars->fileVars->pollers)
-       return (err);
+               boolean_t encryptedswap = true;
+               uint32_t pageCount;
+               AbsoluteTime startTime, endTime;
+               uint64_t nsec;
+
+               bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
+               gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
+               gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
+
+               vmflush = ((kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
+               err = hibernate_alloc_page_lists(&vars->page_list,
+                   &vars->page_list_wired,
+                   &vars->page_list_pal);
+               if (KERN_SUCCESS != err) {
+                       HIBLOG("%s err, hibernate_alloc_page_lists return 0x%x\n", __FUNCTION__, err);
+                       break;
+               }
 
-    for (idx = 0;
-            (poller = (IOPolledInterface *) vars->fileVars->pollers->getObject(idx));
-            idx++)
-    {
-        err = poller->checkForWork();
-        if (err)
-            HIBLOG("IOPolledInterface::checkAllForWork[%d] 0x%x\n", idx, err);
-    }
+               err = hibernate_pin_swap(TRUE);
+               if (KERN_SUCCESS != err) {
+                       HIBLOG("%s error, hibernate_pin_swap return 0x%x\n", __FUNCTION__, err);
+                       break;
+               }
+               swapPinned = true;
+
+               if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode)) {
+                       hibernate_page_list_setall(vars->page_list,
+                           vars->page_list_wired,
+                           vars->page_list_pal,
+                           true /* preflight */,
+                           vmflush /* discard */,
+                           &pageCount);
+                       PE_Video consoleInfo;
+                       bzero(&consoleInfo, sizeof(consoleInfo));
+                       IOService::getPlatform()->getConsoleInfo(&consoleInfo);
+
+                       // estimate: 6% increase in pages compressed
+                       // screen preview 2 images compressed 0%
+                       setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
+                           + vars->page_list->list_size
+                           + (consoleInfo.v_width * consoleInfo.v_height * 8);
+                       enum { setFileRound = 1024 * 1024ULL };
+                       setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
+
+                       HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
+                           pageCount, (100ULL * gIOHibernateCompression) >> 8,
+                           setFileSize, vars->fileMinSize);
+
+                       if (!(kIOHibernateModeFileResize & gIOHibernateMode)
+                           && (setFileSize < vars->fileMinSize)) {
+                               setFileSize = vars->fileMinSize;
+                       }
+               }
 
-    return (err);
-}
+               vars->volumeCryptKeySize = sizeof(vars->volumeCryptKey);
+               err = IOPolledFileOpen(gIOHibernateFilename,
+                   (kIOPolledFileCreate | kIOPolledFileHibernate),
+                   setFileSize, 0,
+                   gIOHibernateCurrentHeader, sizeof(gIOHibernateCurrentHeader),
+                   &vars->fileVars, &nvramData,
+                   &vars->volumeCryptKey[0], &vars->volumeCryptKeySize);
+
+               if (KERN_SUCCESS != err) {
+                       IOLockLock(gFSLock);
+                       if (kFSOpening != gFSState) {
+                               err = kIOReturnTimeout;
+                       }
+                       IOLockUnlock(gFSLock);
+               }
 
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+               if (KERN_SUCCESS != err) {
+                       HIBLOG("IOPolledFileOpen(%x)\n", err);
+                       break;
+               }
 
-struct _OpenFileContext
-{
-    OSData * extents;
-    uint64_t size;
-};
+               // write extents for debug data usage in EFI
+               IOWriteExtentsToFile(vars->fileVars, kIOHibernateHeaderOpenSignature);
 
-static void
-file_extent_callback(void * ref, uint64_t start, uint64_t length)
-{
-    _OpenFileContext * ctx = (_OpenFileContext *) ref;
-    IOPolledFileExtent extent;
+               err = IOPolledFilePollersSetup(vars->fileVars, kIOPolledPreflightState);
+               if (KERN_SUCCESS != err) {
+                       break;
+               }
 
-    extent.start  = start;
-    extent.length = length;
+               clock_get_uptime(&startTime);
+               err = hibernate_setup(gIOHibernateCurrentHeader,
+                   vmflush,
+                   vars->page_list, vars->page_list_wired, vars->page_list_pal);
+               clock_get_uptime(&endTime);
+               SUB_ABSOLUTETIME(&endTime, &startTime);
+               absolutetime_to_nanoseconds(endTime, &nsec);
 
-    ctx->extents->appendBytes(&extent, sizeof(extent));
-    ctx->size += length;
-}
+               boolean_t haveSwapPin, hibFileSSD;
+               haveSwapPin = vm_swap_files_pinned();
 
-IOReturn
-IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
-                           IOPolledFileIOVars ** fileVars, OSData ** fileExtents,
-                           OSData ** imagePath)
-{
-    IOReturn                   err = kIOReturnError;
-    IOPolledFileIOVars *       vars;
-    _OpenFileContext           ctx;
-    OSData *                   extentsData;
-    OSNumber *                 num;
-    IORegistryEntry *          part = 0;
-    OSDictionary *             matching;
-    OSIterator *               iter;
-    dev_t                      hibernate_image_dev;
-    uint64_t                   maxiobytes;
-
-    vars = &gFileVars;
-    do
-    {
-       HIBLOG("sizeof(IOHibernateImageHeader) == %ld\n", sizeof(IOHibernateImageHeader));
-       if (sizeof(IOHibernateImageHeader) != 512)
-           continue;
-    
-       vars->io           = false;
-       vars->buffer       = (uint8_t *) ioBuffer->getBytesNoCopy();
-       vars->bufferHalf   = 0;
-       vars->bufferOffset = 0;
-       vars->bufferSize   = ioBuffer->getLength() >> 1;
-    
-       extentsData = OSData::withCapacity(32);
-    
-       ctx.extents = extentsData;
-       ctx.size    = 0;
-       vars->fileRef = kern_open_file_for_direct_io(filename, 
-                                                   &file_extent_callback, &ctx, 
-                                                   &hibernate_image_dev,
-                                                    &vars->block0,
-                                                    &maxiobytes);
-       if (!vars->fileRef)
-       {
-           err = kIOReturnNoSpace;
-           break;
-       }
-       HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx\n", filename, ctx.size, 
-                    vars->block0, maxiobytes);
-       if (ctx.size < 1*1024*1024)             // check against image size estimate!
-       {
-           err = kIOReturnNoSpace;
-           break;
-       }
-
-        if (maxiobytes < vars->bufferSize)
-            vars->bufferSize = maxiobytes;
-    
-       vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
-    
-       matching = IOService::serviceMatching("IOMedia");
-       num = OSNumber::withNumber(major(hibernate_image_dev), 32);
-       matching->setObject(kIOBSDMajorKey, num);
-       num->release();
-       num = OSNumber::withNumber(minor(hibernate_image_dev), 32);
-       matching->setObject(kIOBSDMinorKey, num);
-       num->release();
-       iter = IOService::getMatchingServices(matching);
-       matching->release();
-       if (iter)
-       {
-           part = (IORegistryEntry *) iter->getNextObject();
-           part->retain();
-           iter->release();
-       }
-       if (!part)
-           break;
-
-       int minor, major;
-       IORegistryEntry * next;
-       IORegistryEntry * child;
-       OSData * data;
+               hibFileSSD = (kIOPolledFileSSD & vars->fileVars->flags);
 
-       num = (OSNumber *) part->getProperty(kIOBSDMajorKey);
-       if (!num)
-           break;
-       major = num->unsigned32BitValue();
-       num = (OSNumber *) part->getProperty(kIOBSDMinorKey);
-       if (!num)
-           break;
-       minor = num->unsigned32BitValue();
-
-       hibernate_image_dev = makedev(major, minor);
-
-        vars->pollers = OSArray::withCapacity(4);
-       if (!vars->pollers)
-           break;
-
-       vars->blockSize = 512;
-       next = part;
-       do
-       {
-            IOPolledInterface * poller;
-           OSObject *          obj;
-
-           obj = next->getProperty(kIOPolledInterfaceSupportKey);
-           if (kOSBooleanFalse == obj)
-           {
-               vars->pollers->flushCollection();
-               break;
-           }
-            else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
-                vars->pollers->setObject(poller);
-           if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey))))
-               vars->blockSize = num->unsigned32BitValue();
-            child = next;
-       }
-       while ((next = child->getParentEntry(gIOServicePlane)) 
-                && child->isParent(next, gIOServicePlane, true));
-
-       HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
-                   major, minor, vars->blockSize, vars->pollers->getCount());
-       if (vars->pollers->getCount() < kIOHibernateMinPollersNeeded)
-           continue;
-
-       err = IOHibernatePollerProbe(vars, (IOService *) part);
-       if (kIOReturnSuccess != err)
-           break;
-
-       err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer);
-       if (kIOReturnSuccess != err)
-           break;
-
-       *fileVars    = vars;
-       *fileExtents = extentsData;
-    
-       // make imagePath
-
-       if ((extentsData->getLength() >= sizeof(IOPolledFileExtent)))
-       {
-           char str2[24];
-
-#if __i386__
-           if (!gIOCreateEFIDevicePathSymbol)
-               gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
-
-           snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
-
-           err = IOService::getPlatform()->callPlatformFunction(
-                                               gIOCreateEFIDevicePathSymbol, false,
-                                               (void *) part, (void *) str2, (void *) true,
-                                               (void *) &data);
-#else
-           char str1[256];
-           int len = sizeof(str1);
-
-           if (!part->getPath(str1, &len, gIODTPlane))
-               err = kIOReturnNotFound;
-           else
-           {
-               snprintf(str2, sizeof(str2), ",%qx", vars->extentMap[0].start);
-               // (strip the plane name)
-               char * tail = strchr(str1, ':');
-               if (!tail)
-                   tail = str1 - 1;
-               data = OSData::withBytes(tail + 1, strlen(tail + 1));
-               data->appendBytes(str2, strlen(str2));
-           }
-#endif
-       if (kIOReturnSuccess == err)
-           *imagePath = data;
-       else
-           HIBLOG("error 0x%x getting path\n", err);
-       }
-    }
-    while (false);
+               HIBLOG("hibernate_setup(%d) took %qd ms, swapPin(%d) ssd(%d)\n",
+                   err, nsec / 1000000ULL,
+                   haveSwapPin, hibFileSSD);
+               if (KERN_SUCCESS != err) {
+                       break;
+               }
 
-    if (kIOReturnSuccess != err)
-    {
-        HIBLOG("error 0x%x opening hibernation file\n", err);
-       if (vars->fileRef)
-           kern_close_file_for_direct_io(vars->fileRef);
-    }
+               gIOHibernateStandbyDisabled = ((!haveSwapPin || !hibFileSSD));
 
-    if (part)
-       part->release();
+               dsSSD = ((0 != (kIOPolledFileSSD & vars->fileVars->flags))
+                   && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
 
-    return (err);
-}
+               if (dsSSD) {
+                       gIOHibernateCurrentHeader->options |= kIOHibernateOptionSSD | kIOHibernateOptionColor;
+               } else {
+                       gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
+               }
 
-IOReturn
-IOPolledFileClose( IOPolledFileIOVars * vars )
-{
-    if (vars->pollers)
-    {
-       IOHibernatePollerClose(vars, kIOPolledPostflightState);
-        vars->pollers->release();
-    }
 
-    gIOHibernateFileRef = vars->fileRef;
+#if defined(__i386__) || defined(__x86_64__)
+               if (vars->volumeCryptKeySize &&
+                   (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey))) {
+                       uintptr_t smcVars[2];
+                       smcVars[0] = vars->volumeCryptKeySize;
+                       smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
 
-    bzero(vars, sizeof(IOPolledFileIOVars));
+                       IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
+                       bzero(smcVars, sizeof(smcVars));
+               }
+#endif
 
-    return (kIOReturnSuccess);
-}
 
-static IOReturn
-IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
-{
-    IOPolledFileExtent * extentMap;
+               if (encryptedswap || vars->volumeCryptKeySize) {
+                       gIOHibernateMode ^= kIOHibernateModeEncrypt;
+               }
 
-    extentMap = vars->extentMap;
+               if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options) {
+                       vars->videoAllocSize = kVideoMapSize;
+                       if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize, VM_KERN_MEMORY_IOKIT)) {
+                               vars->videoMapping = 0;
+                       }
+               }
 
-    vars->position = position;
+               // generate crypt keys
+               for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++) {
+                       vars->wiredCryptKey[i] = random();
+               }
+               for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++) {
+                       vars->cryptKey[i] = random();
+               }
 
-    while (position >= extentMap->length)
-    {
-       position -= extentMap->length;
-       extentMap++;
-    }
+               // set nvram
 
-    vars->currentExtent   = extentMap;
-    vars->extentRemaining = extentMap->length - position;
-    vars->extentPosition  = vars->position - position;
+               IOSetBootImageNVRAM(nvramData);
+               nvramData->release();
 
-    if (vars->bufferSize <= vars->extentRemaining)
-       vars->bufferLimit = vars->bufferSize;
-    else
-       vars->bufferLimit = vars->extentRemaining;
+#if defined(__i386__) || defined(__x86_64__)
+               {
+                       struct AppleRTCHibernateVars {
+                               uint8_t     signature[4];
+                               uint32_t    revision;
+                               uint8_t     booterSignature[20];
+                               uint8_t     wiredCryptKey[16];
+                       };
+                       AppleRTCHibernateVars rtcVars;
+                       OSData * data;
+
+                       rtcVars.signature[0] = 'A';
+                       rtcVars.signature[1] = 'A';
+                       rtcVars.signature[2] = 'P';
+                       rtcVars.signature[3] = 'L';
+                       rtcVars.revision     = 1;
+                       bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
+
+                       if (gIOChosenEntry
+                           && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOHibernateBootSignatureKey)))
+                           && (sizeof(rtcVars.booterSignature) <= data->getLength())) {
+                               bcopy(data->getBytesNoCopy(), &rtcVars.booterSignature[0], sizeof(rtcVars.booterSignature));
+                       } else if (gIOHibernateBootSignature[0]) {
+                               char c;
+                               uint8_t value = 0;
+                               uint32_t in, out, digits;
+                               for (in = out = digits = 0;
+                                   (c = gIOHibernateBootSignature[in]) && (in < sizeof(gIOHibernateBootSignature));
+                                   in++) {
+                                       if ((c >= 'a') && (c <= 'f')) {
+                                               c -= 'a' - 10;
+                                       } else if ((c >= 'A') && (c <= 'F')) {
+                                               c -= 'A' - 10;
+                                       } else if ((c >= '0') && (c <= '9')) {
+                                               c -= '0';
+                                       } else {
+                                               if (c == '=') {
+                                                       out = digits = value = 0;
+                                               }
+                                               continue;
+                                       }
+                                       value = (value << 4) | c;
+                                       if (digits & 1) {
+                                               rtcVars.booterSignature[out++] = value;
+                                               if (out >= sizeof(rtcVars.booterSignature)) {
+                                                       break;
+                                               }
+                                       }
+                                       digits++;
+                               }
+                       }
+#if DEBUG || DEVELOPMENT
+                       if (kIOLogHibernate & gIOKitDebug) {
+                               IOKitKernelLogBuffer("H> rtc:",
+                                   &rtcVars, sizeof(rtcVars), &kprintf);
+                       }
+#endif /* DEBUG || DEVELOPMENT */
+
+                       data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
+                       if (data) {
+                               if (gIOHibernateRTCVariablesKey) {
+                                       IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
+                               }
+                               data->release();
+                       }
+                       if (gIOChosenEntry && gIOOptionsEntry) {
+                               data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
+                               if (data) {
+                                       gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
+                               }
+                               // set BootNext
+                               if (!gIOHibernateBoot0082Data) {
+                                       OSData * fileData = NULL;
+                                       data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
+                                       if (data && data->getLength() >= 4) {
+                                               fileData = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-file-path"));
+                                       }
+                                       if (data) {
+                                               // AppleNVRAM_EFI_LOAD_OPTION
+                                               struct {
+                                                       uint32_t Attributes;
+                                                       uint16_t FilePathLength;
+                                                       uint16_t Desc;
+                                               } loadOptionHeader;
+                                               loadOptionHeader.Attributes     = 1;
+                                               loadOptionHeader.FilePathLength = data->getLength();
+                                               loadOptionHeader.Desc           = 0;
+                                               if (fileData) {
+                                                       loadOptionHeader.FilePathLength -= 4;
+                                                       loadOptionHeader.FilePathLength += fileData->getLength();
+                                               }
+                                               gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
+                                               if (gIOHibernateBoot0082Data) {
+                                                       gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
+                                                       if (fileData) {
+                                                               gIOHibernateBoot0082Data->appendBytes(data->getBytesNoCopy(), data->getLength() - 4);
+                                                               gIOHibernateBoot0082Data->appendBytes(fileData);
+                                                       } else {
+                                                               gIOHibernateBoot0082Data->appendBytes(data);
+                                                       }
+                                               }
+                                       }
+                               }
+                               if (!gIOHibernateBootNextData) {
+                                       uint16_t bits = 0x0082;
+                                       gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
+                               }
+
+#if DEBUG || DEVELOPMENT
+                               if (kIOLogHibernate & gIOKitDebug) {
+                                       IOKitKernelLogBuffer("H> bootnext:",
+                                           gIOHibernateBoot0082Data->getBytesNoCopy(), gIOHibernateBoot0082Data->getLength(), &kprintf);
+                               }
+#endif /* DEBUG || DEVELOPMENT */
+                               if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData) {
+                                       gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
+                                       gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
+                                       gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
+                               }
+                               // BootNext
+                       }
+               }
+#endif /* !i386 && !x86_64 */
+       }while (false);
 
-    return (kIOReturnSuccess);
-}
+       if (swapPinned) {
+               hibernate_pin_swap(FALSE);
+       }
 
-static IOReturn
-IOPolledFileWrite(IOPolledFileIOVars * vars,
-                    const uint8_t * bytes, IOByteCount size,
-                    hibernate_cryptvars_t * cryptvars)
-{
-    IOReturn    err = kIOReturnSuccess;
-    IOByteCount copy;
-    bool       flush = false;
-
-    do
-    {
-       if (!bytes && !size)
-       {
-           // seek to end of block & flush
-           size = vars->position & (vars->blockSize - 1);
-           if (size)
-               size = vars->blockSize - size;
-           flush = true;
-            // use some garbage for the fill
-            bytes = vars->buffer + vars->bufferOffset;
-       }
-
-       copy = vars->bufferLimit - vars->bufferOffset;
-       if (copy > size)
-           copy = size;
-       else
-           flush = true;
-
-       if (bytes)
-       {
-           bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
-           bytes += copy;
-       }
-        else
-           bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
-        
-       size -= copy;
-       vars->bufferOffset += copy;
-       vars->position += copy;
-
-       if (flush && vars->bufferOffset)
-       {
-           uint64_t offset = (vars->position - vars->bufferOffset 
-                               - vars->extentPosition + vars->currentExtent->start);
-           uint32_t length = (vars->bufferOffset);
+       IOLockLock(gFSLock);
+       if ((kIOReturnSuccess == err) && (kFSOpening != gFSState)) {
+               HIBLOG("hibernate file close due timeout\n");
+               err = kIOReturnTimeout;
+       }
+       if (kIOReturnSuccess == err) {
+               gFSState = kFSOpened;
+               gIOHibernateVars = *vars;
+               gFileVars = *vars->fileVars;
+               gFileVars.allocated = false;
+               gIOHibernateVars.fileVars = &gFileVars;
+               gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
+               gIOHibernateState = kIOHibernateStateHibernating;
+
+#if DEBUG || DEVELOPMENT
+               if (kIOLogHibernate & gIOKitDebug) {
+                       OSData * data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
+                       if (data) {
+                               uintptr_t * smcVars = (typeof(smcVars))data->getBytesNoCopy();
+                               IOKitKernelLogBuffer("H> smc:",
+                                   (const void *)smcVars[1], smcVars[0], &kprintf);
+                       }
+               }
+#endif /* DEBUG || DEVELOPMENT */
+       } else {
+               IOPolledFileIOVars * fileVars = vars->fileVars;
+               IOHibernateDone(vars);
+               IOPolledFileClose(&fileVars,
+#if DISABLE_TRIM
+                   0, NULL, 0, 0, 0);
+#else
+                   0, NULL, 0, sizeof(IOHibernateImageHeader), setFileSize);
+#endif
+               gFSState = kFSIdle;
+       }
+       IOLockUnlock(gFSLock);
 
-#if CRYPTO
-            if (cryptvars && vars->encryptStart && (vars->position > vars->encryptStart))
-            {
-                uint32_t encryptLen, encryptStart;
-                encryptLen = vars->position - vars->encryptStart;
-                if (encryptLen > length)
-                    encryptLen = length;
-                encryptStart = length - encryptLen;
-                
-                // encrypt the buffer
-                aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
-                                &cryptvars->aes_iv[0],
-                                encryptLen / AES_BLOCK_SIZE,
-                                vars->buffer + vars->bufferHalf + encryptStart,
-                                &cryptvars->ctx.encrypt);
-                // save initial vector for following encrypts
-                bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
-                        &cryptvars->aes_iv[0],
-                        AES_BLOCK_SIZE);
-            }
-#endif /* CRYPTO */
+       if (vars->fileVars) {
+               IODelete(vars->fileVars, IOPolledFileIOVars, 1);
+       }
+       IODelete(vars, IOHibernateVars, 1);
 
-           if (vars->io)
-            {
-               err = IOHibernatePollerIODone(vars, true);
-                if (kIOReturnSuccess != err)
-                    break;
-            }
-
-if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
-//if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
-
-           err = IOHibernatePollerIO(vars, kIOPolledWrite, vars->bufferHalf, offset, length);
-            if (kIOReturnSuccess != err)
-                break;
-           vars->io = true;
-
-           vars->extentRemaining -= vars->bufferOffset;
-           if (!vars->extentRemaining)
-           {
-               vars->currentExtent++;
-               vars->extentRemaining = vars->currentExtent->length;
-               vars->extentPosition  = vars->position;
-                if (!vars->extentRemaining)
-                {
-                    err = kIOReturnOverrun;
-                    break;
-                }
-           }
-
-           vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
-           vars->bufferOffset = 0;
-           if (vars->bufferSize <= vars->extentRemaining)
-               vars->bufferLimit = vars->bufferSize;
-           else
-               vars->bufferLimit = vars->extentRemaining;
-
-           flush = false;
-       }
-    }
-    while (size);
-
-    return (err);
+       return err;
 }
 
-static IOReturn
-IOPolledFileRead(IOPolledFileIOVars * vars,
-                    uint8_t * bytes, IOByteCount size,
-                    hibernate_cryptvars_t * cryptvars)
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+static void
+IOSetBootImageNVRAM(OSData * data)
 {
-    IOReturn    err = kIOReturnSuccess;
-    IOByteCount copy;
-
-//    bytesWritten += size;
-
-    do
-    {
-       copy = vars->bufferLimit - vars->bufferOffset;
-       if (copy > size)
-           copy = size;
-
-       if (bytes)
-       {
-           bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
-           bytes += copy;
-       }
-       size -= copy;
-       vars->bufferOffset += copy;
-//     vars->position += copy;
-
-       if (vars->bufferOffset == vars->bufferLimit)
-       {
-           if (vars->io)
-            {
-               err = IOHibernatePollerIODone(vars, false);
-                if (kIOReturnSuccess != err)
-                    break;
-            }
-            else
-                cryptvars = 0;
-
-if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
-
-           vars->position += vars->lastRead;
-           vars->extentRemaining -= vars->lastRead;
-           vars->bufferLimit = vars->lastRead;
-
-           if (!vars->extentRemaining)
-           {
-               vars->currentExtent++;
-               vars->extentRemaining = vars->currentExtent->length;
-               vars->extentPosition  = vars->position;
-                if (!vars->extentRemaining)
-                {
-                    err = kIOReturnOverrun;
-                    break;
-                }
-           }
-
-           uint64_t length;
-           uint64_t lastReadLength = vars->lastRead;
-           uint64_t offset = (vars->position 
-                               - vars->extentPosition + vars->currentExtent->start);
-           if (vars->extentRemaining <= vars->bufferSize)
-               length = vars->extentRemaining;
-           else
-               length = vars->bufferSize;
-           vars->lastRead = length;
-
-//if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
-
-           err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
-            if (kIOReturnSuccess != err)
-                break;
-           vars->io = true;
-
-           vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
-           vars->bufferOffset = 0;
+       IORegistryEntry * regEntry;
 
-#if CRYPTO
-            if (cryptvars)
-            {
-                uint8_t thisVector[AES_BLOCK_SIZE];
-                // save initial vector for following decrypts
-                bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
-                bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE, 
-                        &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
-                // decrypt the buffer
-                aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
-                                &thisVector[0],
-                                lastReadLength / AES_BLOCK_SIZE,
-                                vars->buffer + vars->bufferHalf,
-                                &cryptvars->ctx.decrypt);
-            }
-#endif CRYPTO
-       }
-    }
-    while (size);
-
-    return (err);
+       if (!gIOOptionsEntry) {
+               regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
+               gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
+               if (regEntry && !gIOOptionsEntry) {
+                       regEntry->release();
+               }
+       }
+       if (gIOOptionsEntry && gIOHibernateBootImageKey) {
+               if (data) {
+                       gIOOptionsEntry->setProperty(gIOHibernateBootImageKey, data);
+#if DEBUG || DEVELOPMENT
+                       if (kIOLogHibernate & gIOKitDebug) {
+                               IOKitKernelLogBuffer("H> boot-image:",
+                                   data->getBytesNoCopy(), data->getLength(), &kprintf);
+                       }
+#endif /* DEBUG || DEVELOPMENT */
+               } else {
+                       gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
+                       gIOOptionsEntry->sync();
+               }
+       }
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-               
-IOReturn
-IOHibernateSystemSleep(void)
+/*
+ * Writes header to disk with signature, block size and file extents data.
+ * If there are more than 2 extents, then they are written on second block.
+ */
+static IOReturn
+IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature)
 {
-    IOReturn   err;
-    OSData *   data;
-    OSObject * obj;
-    OSString * str;
-    OSNumber * num;
-    OSDictionary *sleepOverrideOptions;
-
-    IOHibernateVars * vars  = &gIOHibernateVars;
-
-    if (vars->fileVars && vars->fileVars->fileRef)
-       // already on the way down
-       return (kIOReturnSuccess);
-
-    gIOHibernateState = kIOHibernateStateInactive;
-
-    /* The invocation of IOPMSleepSystemWithOptions() may override
-     * existing hibernation settings.
-     */
-    sleepOverrideOptions = (OSDictionary *)OSDynamicCast( OSDictionary,
-        IOService::getPMRootDomain()->copyProperty(kRootDomainSleepOptionsKey));
-
-
-    /* Hibernate mode overriden by sleep otions ? */
-    obj = NULL;
-    
-    if (sleepOverrideOptions) {
-        obj = sleepOverrideOptions->getObject(kIOHibernateModeKey);
-        if (obj) obj->retain();
-    }
-
-    if(!obj) {
-        obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateModeKey);
-    }
-
-    if (obj && (num = OSDynamicCast(OSNumber, obj)) )
-    {
-           gIOHibernateMode = num->unsigned32BitValue();
-        if (kIOHibernateModeSleep & gIOHibernateMode)
-            // default to discard clean for safe sleep
-            gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive 
-                                | kIOHibernateModeDiscardCleanActive);
-    }
-
-    if (obj) obj->release();
-    
-    /* Hibernate free rotio overriden by sleep options ? */
-    obj = NULL;
-    
-    if (sleepOverrideOptions) {
-        obj = sleepOverrideOptions->getObject(kIOHibernateFreeRatioKey);
-        if (obj) obj->retain();
-    }
-    
-    if(!obj) {
-        obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeRatioKey);
-    }
-    if (obj && (num = OSDynamicCast(OSNumber, obj)))
-    {
-           gIOHibernateFreeRatio = num->unsigned32BitValue();
-    }
-    if (obj) obj->release();
-    
-    if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeTimeKey)))
-    {
-       if ((num = OSDynamicCast(OSNumber, obj)))
-           gIOHibernateFreeTime = num->unsigned32BitValue();
-       obj->release();
-    }
-    if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
-    {
-       if ((str = OSDynamicCast(OSString, obj)))
-           strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
-                           sizeof(gIOHibernateFilename));
-       obj->release();
-    }
-
-    if (sleepOverrideOptions)
-        sleepOverrideOptions->release();
-
-    if (!gIOHibernateMode || !gIOHibernateFilename[0])
-       return (kIOReturnUnsupported);
-
-    HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
-
-
-    do
-    {
-        vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
-                                   4 * page_size, page_size);
-        vars->ioBuffer  = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, 
-                                   2 * kDefaultIOSize, page_size);
-
-        if (!vars->srcBuffer || !vars->ioBuffer)
-        {
-            err = kIOReturnNoMemory;
-            break;
-        }
-
-        err = IOPolledFileOpen(gIOHibernateFilename, vars->ioBuffer,
-                                &vars->fileVars, &vars->fileExtents, &data);
-        if (KERN_SUCCESS != err)
-        {
-           HIBLOG("IOPolledFileOpen(%x)\n", err);
-            break;
-        }
-       if (vars->fileVars->fileRef)
-       {
-           // invalidate the image file
-           gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
-           int err = kern_write_file(vars->fileVars->fileRef, 0,
-                                       (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
-            if (KERN_SUCCESS != err)
-                HIBLOG("kern_write_file(%d)\n", err);
-       }
-
-       bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
-
-        boolean_t encryptedswap;
-        err = hibernate_setup(gIOHibernateCurrentHeader, 
-                                gIOHibernateFreeRatio, gIOHibernateFreeTime,
-                                &vars->page_list, &vars->page_list_wired, &encryptedswap);
-        if (KERN_SUCCESS != err)
-        {
-           HIBLOG("hibernate_setup(%d)\n", err);
-            break;
-        }
-
-        if (encryptedswap)
-            gIOHibernateMode ^= kIOHibernateModeEncrypt; 
-
-        vars->videoAllocSize = kVideoMapSize;
-        if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
-            vars->videoMapping = 0;
-
-       // generate crypt keys
-        for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
-            vars->wiredCryptKey[i] = random();
-        for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
-            vars->cryptKey[i] = random();
-
-       // set nvram
-
-        IORegistryEntry * regEntry;
-        if (!gIOOptionsEntry)
-        {
-            regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
-            gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
-            if (regEntry && !gIOOptionsEntry)
-                regEntry->release();
-        }
-        if (!gIOChosenEntry)
-            gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
-
-       if (gIOOptionsEntry)
-       {
-            const OSSymbol *  sym;
-
-            sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
-            if (sym)
-            {
-                gIOOptionsEntry->setProperty(sym, data);
-                sym->release();
-            }
-            data->release();
-
-#ifdef __ppc__
-            size_t           len;
-            char              valueString[16];
-
-           vars->saveBootDevice = gIOOptionsEntry->copyProperty(kIOSelectedBootDeviceKey);
-            if (gIOChosenEntry)
-            {
-               OSData * bootDevice = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOBootPathKey));
-               if (bootDevice)
-               {
-                   sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
-                   OSString * str2 = OSString::withCStringNoCopy((const char *) bootDevice->getBytesNoCopy());
-                   if (sym && str2)
-                       gIOOptionsEntry->setProperty(sym, str2);
-                   if (sym)
-                       sym->release();
-                   if (str2)
-                       str2->release();
-               }
-                data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMemorySignatureKey));
-                if (data)
-                {
-                    vars->haveFastBoot = true;
-
-                    len = sprintf(valueString, "0x%lx", *((UInt32 *)data->getBytesNoCopy()));
-                    data = OSData::withBytes(valueString, len + 1);
-                    sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
-                    if (sym && data)
-                        gIOOptionsEntry->setProperty(sym, data);
-                    if (sym)
-                        sym->release();
-                    if (data)
-                        data->release();
-                }
-                data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
-                if (data)
-                    gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
-            }
-#endif /* __ppc__ */
-#ifdef __i386__
-           struct AppleRTCHibernateVars
-           {
-               uint8_t     signature[4];
-               uint32_t    revision;
-               uint8_t     booterSignature[20];
-               uint8_t     wiredCryptKey[16];
-           };
-           AppleRTCHibernateVars rtcVars;
-
-           rtcVars.signature[0] = 'A';
-           rtcVars.signature[1] = 'A';
-           rtcVars.signature[2] = 'P';
-           rtcVars.signature[3] = 'L';
-           rtcVars.revision     = 1;
-           bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
-           if (gIOHibernateBootSignature[0])
-           {
-               char c;
-               uint8_t value = 0;
-               for (uint32_t i = 0;
-                   (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
-                   i++)
-               {
-                   if (c >= 'a')
-                       c -= 'a' - 10;
-                   else if (c >= 'A')
-                       c -= 'A' - 10;
-                   else if (c >= '0')
-                       c -= '0';
-                   else
-                       continue;
-                   value = (value << 4) | c;
-                   if (i & 1)
-                       rtcVars.booterSignature[i >> 1] = value;
-               }
-           }
-           data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
-           if (data)
-           { 
-                       IOService::getPMRootDomain()->setProperty(kIOHibernateRTCVariablesKey, data);
-               
-                       if( gIOOptionsEntry )
-                       {
-                               if( gIOHibernateMode & kIOHibernateModeSwitch )
-                               {
-                                       const OSSymbol *sym;
-                                       sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSwitchVarsKey);
-                                       if( sym )
-                                       {
-                                               gIOOptionsEntry->setProperty(sym, data); /* intentional insecure backup of rtc boot vars */
-                                               sym->release();
-                                       }
-                               }       
-                       }
-
-                       data->release();
-           }
-            if (gIOChosenEntry)
-            {
-                data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
-                if (data)
-                    gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
-            }
-#else /* !__i386__ */
-            if (kIOHibernateModeEncrypt & gIOHibernateMode)
-            {
-                data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
-                sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
-                if (sym && data)
-                    gIOOptionsEntry->setProperty(sym, data);
-                if (sym)
-                    sym->release();
-                if (data)
-                    data->release();
-                if (false && gIOHibernateBootSignature[0])
-                {
-                    data = OSData::withCapacity(16);
-                    sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
-                    if (sym && data)
-                    {
-                        char c;
-                        uint8_t value = 0;
-                        for (uint32_t i = 0; (c = gIOHibernateBootSignature[i]); i++)
-                        {
-                            if (c >= 'a')
-                                c -= 'a' - 10;
-                            else if (c >= 'A')
-                                c -= 'A' - 10;
-                            else if (c >= '0')
-                                c -= '0';
-                            else
-                                continue;
-                            value = (value << 4) | c;
-                            if (i & 1)
-                                data->appendBytes(&value, sizeof(value));
-                        }
-                        gIOOptionsEntry->setProperty(sym, data);
-                    }
-                    if (sym)
-                        sym->release();
-                    if (data)
-                        data->release();
-                }
-            }
-            if (!vars->haveFastBoot)
-            {
-                // set boot volume to zero
-                IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
-                if (platform && (kIOReturnSuccess == platform->readXPRAM(kXPRamAudioVolume, 
-                                            &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume))))
-                {
-                    uint8_t newVolume;
-                    newVolume = vars->saveBootAudioVolume & 0xf8;
-                    platform->writeXPRAM(kXPRamAudioVolume, 
-                                            &newVolume, sizeof(newVolume));
-                }
-            }
-#endif /* !__i386__ */
+       IOHibernateImageHeader hdr;
+       IOItemCount            count;
+       IOReturn               err = kIOReturnSuccess;
+       int                    rc;
+       IOPolledFileExtent *   fileExtents;
+
+       fileExtents = (typeof(fileExtents))vars->fileExtents->getBytesNoCopy();
+
+       memset(&hdr, 0, sizeof(IOHibernateImageHeader));
+       count = vars->fileExtents->getLength();
+       if (count > sizeof(hdr.fileExtentMap)) {
+               hdr.fileExtentMapSize = count;
+               count = sizeof(hdr.fileExtentMap);
+       } else {
+               hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap);
        }
-       // --
 
-       gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
-       gIOHibernateState = kIOHibernateStateHibernating;
-    }
-    while (false);
+       bcopy(fileExtents, &hdr.fileExtentMap[0], count);
+
+       // copy file block extent list if larger than header
+       if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap)) {
+               count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap);
+               rc = kern_write_file(vars->fileRef, vars->blockSize,
+                   (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)),
+                   count, IO_SKIP_ENCRYPTION);
+               if (rc != 0) {
+                       HIBLOG("kern_write_file returned %d\n", rc);
+                       err = kIOReturnIOError;
+                       goto exit;
+               }
+       }
+       hdr.signature = signature;
+       hdr.deviceBlockSize = vars->blockSize;
+
+       rc = kern_write_file(vars->fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION);
+       if (rc != 0) {
+               HIBLOG("kern_write_file returned %d\n", rc);
+               err = kIOReturnIOError;
+               goto exit;
+       }
 
-    return (err);
+exit:
+       return err;
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@@ -1331,115 +896,121 @@ DECLARE_IOHIBERNATEPROGRESSALPHA
 static void
 ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
 {
-    uint32_t   rowBytes, pixelShift;
-    uint32_t   x, y;
-    int32_t    blob;
-    uint32_t   alpha, in, color, result;
-    uint8_t *  out;
-    uint32_t   saveindex[kIOHibernateProgressCount] = { 0 };
-
-    rowBytes = display->rowBytes;
-    pixelShift = display->depth >> 4;
-    if (pixelShift < 1) return;
-    
-    screen += ((display->width
-                - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
-        + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
-    
-    for (y = 0; y < kIOHibernateProgressHeight; y++)
-    {
-        out = screen + y * rowBytes;
-        for (blob = 0; blob < kIOHibernateProgressCount; blob++)
-        {
-            color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
-            for (x = 0; x < kIOHibernateProgressWidth; x++)
-            {
-                alpha  = gIOHibernateProgressAlpha[y][x];
-                result = color;
-                if (alpha)
-                {
-                    if (0xff != alpha)
-                    {
-                        if (1 == pixelShift)
-                        {
-                            in = *((uint16_t *)out) & 0x1f;    // 16
-                            in = (in << 3) | (in >> 2);
-                        }
-                        else
-                            in = *((uint32_t *)out) & 0xff;    // 32
-                        saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
-                        result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
-                    }
-                    if (1 == pixelShift)
-                    {
-                        result >>= 3;
-                        *((uint16_t *)out) = (result << 10) | (result << 5) | result;  // 16
-                    }
-                    else
-                        *((uint32_t *)out) = (result << 16) | (result << 8) | result;  // 32
-                }
-                out += (1 << pixelShift);
-            }
-            out += (kIOHibernateProgressSpacing << pixelShift);
-        }
-    }
+       uint32_t    rowBytes, pixelShift;
+       uint32_t    x, y;
+       int32_t     blob;
+       uint32_t    alpha, in, color, result;
+       uint8_t *   out;
+       uint32_t    saveindex[kIOHibernateProgressCount] = { 0 };
+
+       rowBytes = display->rowBytes;
+       pixelShift = display->depth >> 4;
+       if (pixelShift < 1) {
+               return;
+       }
+
+       screen += ((display->width
+           - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
+           + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
+
+       for (y = 0; y < kIOHibernateProgressHeight; y++) {
+               out = screen + y * rowBytes;
+               for (blob = 0; blob < kIOHibernateProgressCount; blob++) {
+                       color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
+                       for (x = 0; x < kIOHibernateProgressWidth; x++) {
+                               alpha  = gIOHibernateProgressAlpha[y][x];
+                               result = color;
+                               if (alpha) {
+                                       if (0xff != alpha) {
+                                               if (1 == pixelShift) {
+                                                       in = *((uint16_t *)out) & 0x1f; // 16
+                                                       in = (in << 3) | (in >> 2);
+                                               } else {
+                                                       in = *((uint32_t *)out) & 0xff; // 32
+                                               }
+                                               saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
+                                               result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
+                                       }
+                                       if (1 == pixelShift) {
+                                               result >>= 3;
+                                               *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
+                                       } else {
+                                               *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
+                                       }
+                               }
+                               out += (1 << pixelShift);
+                       }
+                       out += (kIOHibernateProgressSpacing << pixelShift);
+               }
+       }
 }
 
 
 static void
 ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
 {
-    uint32_t  rowBytes, pixelShift;
-    uint32_t  x, y;
-    int32_t   blob, lastBlob;
-    uint32_t  alpha, in, color, result;
-    uint8_t * out;
-    uint32_t  saveindex[kIOHibernateProgressCount] = { 0 };
-
-    pixelShift = display->depth >> 4;
-    if (pixelShift < 1)
-        return;
-
-    rowBytes = display->rowBytes;
-
-    screen += ((display->width 
-            - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
-                + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
-
-    lastBlob  = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
-
-    screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
-
-    for (y = 0; y < kIOHibernateProgressHeight; y++)
-    {
-        out = screen + y * rowBytes;
-        for (blob = firstBlob; blob <= lastBlob; blob++)
-        {
-            color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
-            for (x = 0; x < kIOHibernateProgressWidth; x++)
-            {
-                alpha  = gIOHibernateProgressAlpha[y][x];
-                result = color;
-                if (alpha)
-                {
-                    if (0xff != alpha)
-                    {
-                        in = display->progressSaveUnder[blob][saveindex[blob]++];
-                        result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
-                    }
-                    if (1 == pixelShift)
-                    {
-                        result >>= 3;
-                        *((uint16_t *)out) = (result << 10) | (result << 5) | result;  // 16
-                    }
-                    else
-                        *((uint32_t *)out) = (result << 16) | (result << 8) | result;  // 32
-                }
-                out += (1 << pixelShift);
-            }
-            out += (kIOHibernateProgressSpacing << pixelShift);
-        }
-    }
+       uint32_t  rowBytes, pixelShift;
+       uint32_t  x, y;
+       int32_t   blob, lastBlob;
+       uint32_t  alpha, in, color, result;
+       uint8_t * out;
+       uint32_t  saveindex[kIOHibernateProgressCount] = { 0 };
+
+       pixelShift = display->depth >> 4;
+       if (pixelShift < 1) {
+               return;
+       }
+
+       rowBytes = display->rowBytes;
+
+       screen += ((display->width
+           - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
+           + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
+
+       lastBlob  = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
+
+       screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
+
+       for (y = 0; y < kIOHibernateProgressHeight; y++) {
+               out = screen + y * rowBytes;
+               for (blob = firstBlob; blob <= lastBlob; blob++) {
+                       color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
+                       for (x = 0; x < kIOHibernateProgressWidth; x++) {
+                               alpha  = gIOHibernateProgressAlpha[y][x];
+                               result = color;
+                               if (alpha) {
+                                       if (0xff != alpha) {
+                                               in = display->progressSaveUnder[blob][saveindex[blob]++];
+                                               result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
+                                       }
+                                       if (1 == pixelShift) {
+                                               result >>= 3;
+                                               *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
+                                       } else {
+                                               *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
+                                       }
+                               }
+                               out += (1 << pixelShift);
+                       }
+                       out += (kIOHibernateProgressSpacing << pixelShift);
+               }
+       }
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+IOReturn
+IOHibernateIOKitSleep(void)
+{
+       IOReturn ret = kIOReturnSuccess;
+       IOLockLock(gFSLock);
+       if (kFSOpening == gFSState) {
+               gFSState = kFSTimedOut;
+               HIBLOG("hibernate file open timed out\n");
+               ret = kIOReturnTimeout;
+       }
+       IOLockUnlock(gFSLock);
+       return ret;
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@@ -1447,976 +1018,1519 @@ ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBl
 IOReturn
 IOHibernateSystemHasSlept(void)
 {
-    IOHibernateVars * vars  = &gIOHibernateVars;
-    OSObject        * obj;
-    OSData          * data;
-
-    obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
-    vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
-    if (obj && !vars->previewBuffer)
-       obj->release();
-
-    vars->consoleMapping = NULL;
-    if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
-    {
-       vars->previewBuffer->release();
-       vars->previewBuffer = 0;
-    }
-
-    if (vars->previewBuffer && (data = OSDynamicCast(OSData, 
-       IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
-    {
-       UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
-       HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", flags);
-
-       IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
-
-       if (kIOHibernatePreviewUpdates & flags)
-       {
-           PE_Video           consoleInfo;
-           hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
-
-           IOService::getPlatform()->getConsoleInfo(&consoleInfo);
-
-           graphicsInfo->width    = consoleInfo.v_width;
-           graphicsInfo->height   = consoleInfo.v_height;
-           graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
-           graphicsInfo->depth    = consoleInfo.v_depth;
-           vars->consoleMapping   = (uint8_t *) consoleInfo.v_baseAddr;
-
-           HIBPRINT("video %p %d %d %d\n",
-                       vars->consoleMapping, gIOHibernateGraphicsInfo->depth, 
-                       gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height);
-           if (vars->consoleMapping)
-                       ProgressInit(graphicsInfo, vars->consoleMapping,
-                                       &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
-       }
-    }
-
-    if (gIOOptionsEntry)
-        gIOOptionsEntry->sync();
-
-    return (kIOReturnSuccess);
+       IOReturn          ret = kIOReturnSuccess;
+       IOHibernateVars * vars  = &gIOHibernateVars;
+       OSObject        * obj = NULL;
+       OSData          * data;
+
+       IOLockLock(gFSLock);
+       if ((kFSOpened != gFSState) && gIOHibernateMode) {
+               ret = kIOReturnTimeout;
+       }
+       IOLockUnlock(gFSLock);
+       if (kIOReturnSuccess != ret) {
+               return ret;
+       }
+
+       if (gIOHibernateMode) {
+               obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
+       }
+       vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
+       if (obj && !vars->previewBuffer) {
+               obj->release();
+       }
+
+       vars->consoleMapping = NULL;
+       if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare())) {
+               vars->previewBuffer->release();
+               vars->previewBuffer = NULL;
+       }
+
+       if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
+           && vars->previewBuffer
+           && (data = OSDynamicCast(OSData,
+           IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey)))) {
+               UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
+               HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
+
+               IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
+
+               if (kIOHibernatePreviewUpdates & flags) {
+                       PE_Video           consoleInfo;
+                       hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
+
+                       IOService::getPlatform()->getConsoleInfo(&consoleInfo);
+
+                       graphicsInfo->width    = consoleInfo.v_width;
+                       graphicsInfo->height   = consoleInfo.v_height;
+                       graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
+                       graphicsInfo->depth    = consoleInfo.v_depth;
+                       vars->consoleMapping   = (uint8_t *) consoleInfo.v_baseAddr;
+
+                       HIBPRINT("video %p %d %d %d\n",
+                           vars->consoleMapping, graphicsInfo->depth,
+                           graphicsInfo->width, graphicsInfo->height);
+                       if (vars->consoleMapping) {
+                               ProgressInit(graphicsInfo, vars->consoleMapping,
+                                   &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
+                       }
+               }
+       }
+
+       if (gIOOptionsEntry) {
+               gIOOptionsEntry->sync();
+       }
+
+       return ret;
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
+static DeviceTreeNode *
+MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry)
+{
+       DeviceTreeNodeProperty * prop;
+       DeviceTreeNode *         child;
+       IORegistryEntry *        childRegEntry;
+       const char *             nameProp;
+       unsigned int             propLen, idx;
+
+       prop = (DeviceTreeNodeProperty *) (entry + 1);
+       for (idx = 0; idx < entry->nProperties; idx++) {
+               if (regEntry && (0 != strcmp("name", prop->name))) {
+                       regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
+//         HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
+               }
+               prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
+       }
+
+       child = (DeviceTreeNode *) prop;
+       for (idx = 0; idx < entry->nChildren; idx++) {
+               if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen)) {
+                       panic("no name");
+               }
+               childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
+//     HIBPRINT("%s == %p\n", nameProp, childRegEntry);
+               child = MergeDeviceTree(child, childRegEntry);
+       }
+       return child;
+}
+
 IOReturn
 IOHibernateSystemWake(void)
 {
-    IOHibernateVars * vars  = &gIOHibernateVars;
-
-    hibernate_teardown(vars->page_list, vars->page_list_wired);
-
-    if (vars->videoMapping)
-    {
-        if (vars->videoMapSize)
-            // remove mappings
-            IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
-        if (vars->videoAllocSize)
-            // dealloc range
-            kmem_free(kernel_map, trunc_page_32(vars->videoMapping), vars->videoAllocSize);
-    }
-
-    if (vars->previewBuffer)
-    {
-        vars->previewBuffer->release();
-        vars->previewBuffer = 0;
-    }
-
-    if (vars->fileVars)
-    {
-       IOPolledFileClose(vars->fileVars);
-    }
-
-    // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
-
-#ifdef __ppc__
-    OSData * data = OSData::withCapacity(4);
-    if (gIOOptionsEntry && data)
-    {
-        const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
-        if (sym)
-        {
-            gIOOptionsEntry->setProperty(sym, data);
-            sym->release();
-        }
-        sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
-        if (sym)
-        {
-           if (vars->saveBootDevice)
-           {
-               gIOOptionsEntry->setProperty(sym, vars->saveBootDevice);
-               vars->saveBootDevice->release();
-           }
-            sym->release();
-        }
-        sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
-        if (sym)
-        {
-            gIOOptionsEntry->setProperty(sym, data);
-            sym->release();
-        }
-        sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
-        if (sym)
-        {
-            gIOOptionsEntry->removeProperty(sym);
-            sym->release();
-        }
-    }
-    if (data)
-        data->release();
-
-    if (gIOOptionsEntry)
-    {
-       if (!vars->haveFastBoot)
-       {
-           // reset boot audio volume
-           IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
-           if (platform)
-               platform->writeXPRAM(kXPRamAudioVolume, 
-                                       &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume));
-       }
-
-       // sync now to hardware if the booter has not
-       if (kIOHibernateStateInactive == gIOHibernateState)
-           gIOOptionsEntry->sync();
-       else
-           // just sync the variables in case a later panic syncs nvram (it won't sync variables)
-           gIOOptionsEntry->syncOFVariables();
-    }
-#endif
+       if (kFSOpened == gFSState) {
+               IOPolledFilePollersClose(gIOHibernateVars.fileVars, kIOPolledPostflightState);
+               IOHibernateDone(&gIOHibernateVars);
+       } else {
+               IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
+               IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
+       }
+       return kIOReturnSuccess;
+}
 
-#ifdef __i386__
-       IOService::getPMRootDomain()->removeProperty(kIOHibernateRTCVariablesKey);
+static IOReturn
+IOHibernateDone(IOHibernateVars * vars)
+{
+       IOReturn err;
+       OSData * data;
+
+       hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
+
+       if (vars->videoMapping) {
+               if (vars->videoMapSize) {
+                       // remove mappings
+                       IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
+               }
+               if (vars->videoAllocSize) {
+                       // dealloc range
+                       kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
+               }
+       }
+
+       if (vars->previewBuffer) {
+               vars->previewBuffer->release();
+               vars->previewBuffer = NULL;
+       }
+
+       if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
+               IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
+                   gIOHibernateCurrentHeader->options, 32);
+       } else {
+               IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
+       }
+
+       if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
+           && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus)) {
+               IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
+                   &gIOHibernateGraphicsInfo->gfxStatus,
+                   sizeof(gIOHibernateGraphicsInfo->gfxStatus));
+       } else {
+               IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
+       }
+
+       // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
+
+#if defined(__i386__) || defined(__x86_64__)
+       IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
+       IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
 
        /*
         * Hibernate variable is written to NVRAM on platforms in which RtcRam
         * is not backed by coin cell.  Remove Hibernate data from NVRAM.
         */
        if (gIOOptionsEntry) {
-               const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
+               if (gIOHibernateRTCVariablesKey) {
+                       if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
+                               gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
+                       }
+               }
 
-               if (sym) {
-                       if (gIOOptionsEntry->getProperty(sym)) {
-                               gIOOptionsEntry->removeProperty(sym);
-                               gIOOptionsEntry->sync();
+               if (gIOHibernateBootNextKey) {
+                       if (gIOHibernateBootNextSave) {
+                               gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
+                               gIOHibernateBootNextSave->release();
+                               gIOHibernateBootNextSave = NULL;
+                       } else {
+                               gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
                        }
-                       sym->release();
                }
+               if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) {
+                       gIOOptionsEntry->sync();
+               }
+       }
+#endif
+
+       if (vars->srcBuffer) {
+               vars->srcBuffer->release();
        }
+       bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
+       if (vars->handoffBuffer) {
+               if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
+                       IOHibernateHandoff * handoff;
+                       bool done = false;
+                       for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
+                           !done;
+                           handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount]) {
+                               HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
+                               uint8_t * data = &handoff->data[0];
+                               switch (handoff->type) {
+                               case kIOHibernateHandoffTypeEnd:
+                                       done = true;
+                                       break;
+
+                               case kIOHibernateHandoffTypeDeviceTree:
+                                       MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
+                                       break;
+
+                               case kIOHibernateHandoffTypeKeyStore:
+#if defined(__i386__) || defined(__x86_64__)
+                                       {
+                                               IOBufferMemoryDescriptor *
+                                                   md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
+                                               if (md) {
+                                                       IOSetKeyStoreData(md);
+                                               }
+                                       }
 #endif
+                                       break;
 
-    if (vars->srcBuffer)
-       vars->srcBuffer->release();
-    if (vars->ioBuffer)
-       vars->ioBuffer->release();
-    if (vars->fileExtents)
-       vars->fileExtents->release();
+                               default:
+                                       done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
+                                       break;
+                               }
+                       }
+#if defined(__i386__) || defined(__x86_64__)
+                       if (vars->volumeCryptKeySize) {
+                               IOBufferMemoryDescriptor *
+                                   bmd = IOBufferMemoryDescriptor::withBytes(&vars->volumeCryptKey[0],
+                                   vars->volumeCryptKeySize, kIODirectionOutIn);
+                               if (!bmd) {
+                                       panic("IOBufferMemoryDescriptor");
+                               }
+                               IOSetAPFSKeyStoreData(bmd);
+                               bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
+                       }
+#endif
+               }
+               vars->handoffBuffer->release();
+       }
+
+       if (gIOChosenEntry
+           && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey)))
+           && (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength())) {
+               bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0],
+                   sizeof(gIOHibernateBridgeBootSessionUUIDString));
+       }
 
-    bzero(vars, sizeof(*vars));
+       if (vars->hwEncrypt) {
+               err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, NULL, 0);
+               HIBLOG("IOPolledFilePollersSetEncryptionKey(0,%x)\n", err);
+       }
+
+       bzero(vars, sizeof(*vars));
 
 //    gIOHibernateState = kIOHibernateStateInactive;       // leave it for post wake code to see
 
-    return (kIOReturnSuccess);
+       return kIOReturnSuccess;
+}
+
+static void
+IOHibernateSystemPostWakeTrim(void * p1, void * p2)
+{
+       // invalidate & close the image file
+       if (p1) {
+               IOLockLock(gFSLock);
+       }
+       if (kFSTrimDelay == gFSState) {
+               IOPolledFileIOVars * vars = &gFileVars;
+               IOPolledFileClose(&vars,
+#if DISABLE_TRIM
+                   0, NULL, 0, 0, 0);
+#else
+                   0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader),
+                   sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize);
+#endif
+               gFSState = kFSIdle;
+       }
+       if (p1) {
+               IOLockUnlock(gFSLock);
+       }
 }
 
 IOReturn
-IOHibernateSystemPostWake(void)
+IOHibernateSystemPostWake(bool now)
 {
-    if (gIOHibernateFileRef)
-    {
-       // invalidate the image file
        gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
-       int err = kern_write_file(gIOHibernateFileRef, 0,
-                                   (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
-       if (KERN_SUCCESS != err)
-           HIBLOG("kern_write_file(%d)\n", err);
-
-       kern_close_file_for_direct_io(gIOHibernateFileRef);
-        gIOHibernateFileRef = 0;
-    }
-    return (kIOReturnSuccess);
+       IOSetBootImageNVRAM(NULL);
+
+       IOLockLock(gFSLock);
+       if (kFSTrimDelay == gFSState) {
+               thread_call_cancel(gIOHibernateTrimCalloutEntry);
+               IOHibernateSystemPostWakeTrim(NULL, NULL);
+       } else if (kFSOpened != gFSState) {
+               gFSState = kFSIdle;
+       } else {
+               gFSState = kFSTrimDelay;
+               if (now) {
+                       thread_call_cancel(gIOHibernateTrimCalloutEntry);
+                       IOHibernateSystemPostWakeTrim(NULL, NULL);
+               } else {
+                       AbsoluteTime deadline;
+                       clock_interval_to_deadline(TRIM_DELAY, kMillisecondScale, &deadline );
+                       thread_call_enter1_delayed(gIOHibernateTrimCalloutEntry, NULL, deadline);
+               }
+       }
+       IOLockUnlock(gFSLock);
+
+       return kIOReturnSuccess;
+}
+
+uint32_t
+IOHibernateWasScreenLocked(void)
+{
+       uint32_t ret = 0;
+       if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry) {
+               OSData *
+                   data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
+               if (data) {
+                       ret = ((uint32_t *)data->getBytesNoCopy())[0];
+                       gIOChosenEntry->setProperty(kIOBooterScreenLockStateKey, data);
+               }
+       } else {
+               gIOChosenEntry->removeProperty(kIOBooterScreenLockStateKey);
+       }
+
+       return ret;
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
-SYSCTL_STRING(_kern, OID_AUTO, hibernatefile, 
-               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, 
-               gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
-SYSCTL_STRING(_kern, OID_AUTO, bootsignature, 
-               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, 
-               gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
-SYSCTL_UINT(_kern, OID_AUTO, hibernatemode, 
-               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, 
-               &gIOHibernateMode, 0, "");
+SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
+    CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
+    gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
+SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
+    CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
+    gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
+    CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
+    &gIOHibernateMode, 0, "");
+SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
+    CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
+    &_hibernateStats, hibernate_statistics_t, "");
+SYSCTL_STRING(_kern_bridge, OID_AUTO, bootsessionuuid,
+    CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
+    gIOHibernateBridgeBootSessionUUIDString, sizeof(gIOHibernateBridgeBootSessionUUIDString), "");
+
+SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
+    CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+    &_hibernateStats.graphicsReadyTime, 0, "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
+    CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+    &_hibernateStats.wakeNotificationTime, 0, "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
+    CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+    &_hibernateStats.lockScreenReadyTime, 0, "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
+    CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+    &_hibernateStats.hidReadyTime, 0, "");
 
 void
 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
 {
-    OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
-    if (data)
-    {
-       rootDomain->setProperty(kIOHibernateStateKey, data);
-       data->release();
-    }
-
-    if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))
-       gIOHibernateMode = kIOHibernateModeOn;
-    else
-       gIOHibernateFilename[0] = 0;
-
-    sysctl_register_oid(&sysctl__kern_hibernatefile);
-    sysctl_register_oid(&sysctl__kern_bootsignature);
-    sysctl_register_oid(&sysctl__kern_hibernatemode);
-}
+       gIOHibernateBootImageKey     = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
+       gIOHibernateBootSignatureKey = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
+       gIOBridgeBootSessionUUIDKey  = OSSymbol::withCStringNoCopy(kIOBridgeBootSessionUUIDKey);
+
+#if defined(__i386__) || defined(__x86_64__)
+       gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
+       gIOHibernateBoot0082Key     = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
+       gIOHibernateBootNextKey     = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
+       gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
+#endif /* defined(__i386__) || defined(__x86_64__) */
+
+       OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
+       if (data) {
+               rootDomain->setProperty(kIOHibernateStateKey, data);
+               data->release();
+       }
+
+       if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename))) {
+               gIOHibernateMode = kIOHibernateModeOn;
+       } else {
+               gIOHibernateFilename[0] = 0;
+       }
+
+       sysctl_register_oid(&sysctl__kern_hibernatefile);
+       sysctl_register_oid(&sysctl__kern_bootsignature);
+       sysctl_register_oid(&sysctl__kern_hibernatemode);
+       sysctl_register_oid(&sysctl__kern_hibernatestatistics);
+       sysctl_register_oid(&sysctl__kern_hibernategraphicsready);
+       sysctl_register_oid(&sysctl__kern_hibernatewakenotification);
+       sysctl_register_oid(&sysctl__kern_hibernatelockscreenready);
+       sysctl_register_oid(&sysctl__kern_hibernatehidready);
+
+       gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
+
+       if (gIOChosenEntry
+           && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey)))
+           && (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength())) {
+               sysctl_register_oid(&sysctl__kern_bridge_bootsessionuuid);
+               bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0], sizeof(gIOHibernateBridgeBootSessionUUIDString));
+       }
 
+       gFSLock = IOLockAlloc();
+}
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
-static void
-hibernate_setup_for_wake(void)
+static IOReturn
+IOHibernatePolledFileWrite(IOPolledFileIOVars * vars,
+    const uint8_t * bytes, IOByteCount size,
+    IOPolledFileCryptVars * cryptvars)
 {
-#if __ppc__
-    // go slow (state needed for wake)
-    ml_set_processor_speed(1);
-#endif
+       IOReturn err;
+
+       err = IOPolledFileWrite(vars, bytes, size, cryptvars);
+       if ((kIOReturnSuccess == err) && hibernate_should_abort()) {
+               err = kIOReturnAborted;
+       }
+
+       return err;
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
-#define C_ASSERT(e) typedef char    __C_ASSERT__[(e) ? 1 : -1]
-
 extern "C" uint32_t
 hibernate_write_image(void)
 {
-    IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
-    IOHibernateVars *        vars  = &gIOHibernateVars;
-    IOPolledFileExtent *     fileExtents;
-
-    C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
-
-    uint32_t    pageCount, pagesDone;
-    IOReturn     err;
-    vm_offset_t  ppnum;
-    IOItemCount  page, count;
-    uint8_t *   src;
-    uint8_t *   data;
-    IOByteCount  pageCompressedSize;
-    uint64_t    compressedSize, uncompressedSize;
-    uint64_t    image1Size = 0;
-    uint32_t    bitmap_size;
-    bool        iterDone, pollerOpen, needEncryptStart;
-    uint32_t    restore1Sum, sum, sum1, sum2;
-    uint32_t    tag;
-    uint32_t    pageType;
-    uint32_t    pageAndCount[2];
-
-    AbsoluteTime startTime, endTime;
-    AbsoluteTime allTime, compTime, decoTime;
-    uint64_t     nsec;
-    uint32_t     lastProgressStamp = 0;
-    uint32_t     progressStamp;
-    uint32_t    blob, lastBlob = (uint32_t) -1L;
-
-    hibernate_cryptvars_t _cryptvars;
-    hibernate_cryptvars_t * cryptvars = 0;
-
-    if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
-        return (false /* sleep */ );
-
-    restore1Sum = sum1 = sum2 = 0;
+       IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
+       IOHibernateVars *        vars  = &gIOHibernateVars;
+       IOPolledFileExtent *     fileExtents;
+
+       _static_assert_1_arg(sizeof(IOHibernateImageHeader) == 512);
+
+       uint32_t     pageCount, pagesDone;
+       IOReturn     err;
+       vm_offset_t  ppnum, page;
+       IOItemCount  count;
+       uint8_t *    src;
+       uint8_t *    data;
+       uint8_t *    compressed;
+       uint8_t *    scratch;
+       IOByteCount  pageCompressedSize;
+       uint64_t     compressedSize, uncompressedSize;
+       uint64_t     image1Size = 0;
+       uint32_t     bitmap_size;
+       bool         iterDone, pollerOpen, needEncrypt;
+       uint32_t     restore1Sum, sum, sum1, sum2;
+       int          wkresult;
+       uint32_t     tag;
+       uint32_t     pageType;
+       uint32_t     pageAndCount[2];
+       addr64_t     phys64;
+       IOByteCount  segLen;
+       uintptr_t    hibernateBase;
+       uintptr_t    hibernateEnd;
+
+       AbsoluteTime startTime, endTime;
+       AbsoluteTime allTime, compTime;
+       uint64_t     compBytes;
+       uint64_t     nsec;
+       uint32_t     lastProgressStamp = 0;
+       uint32_t     progressStamp;
+       uint32_t     blob, lastBlob = (uint32_t) -1L;
+
+       uint32_t     wiredPagesEncrypted;
+       uint32_t     dirtyPagesEncrypted;
+       uint32_t     wiredPagesClear;
+       uint32_t     svPageCount;
+       uint32_t     zvPageCount;
+
+       IOPolledFileCryptVars _cryptvars;
+       IOPolledFileCryptVars * cryptvars = NULL;
+
+       wiredPagesEncrypted = 0;
+       dirtyPagesEncrypted = 0;
+       wiredPagesClear     = 0;
+       svPageCount         = 0;
+       zvPageCount         = 0;
+
+       if (!vars->fileVars
+           || !vars->fileVars->pollers
+           || !(kIOHibernateModeOn & gIOHibernateMode)) {
+               return kIOHibernatePostWriteSleep;
+       }
+
+       if (kIOHibernateModeSleep & gIOHibernateMode) {
+               kdebug_enable = save_kdebug_enable;
+       }
+
+       KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START);
+       IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
+
+       restore1Sum = sum1 = sum2 = 0;
 
 #if CRYPTO
-    // encryption data. "iv" is the "initial vector".
-    if (kIOHibernateModeEncrypt & gIOHibernateMode)
-    {
-        static const unsigned char first_iv[AES_BLOCK_SIZE]
-        = {  0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
-             0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
-    
-        cryptvars = &gIOHibernateCryptWakeContext;
-        bzero(cryptvars, sizeof(hibernate_cryptvars_t));
-        aes_encrypt_key(vars->cryptKey,
-                        kIOHibernateAESKeySize,
-                        &cryptvars->ctx.encrypt);
-        aes_decrypt_key(vars->cryptKey,
-                        kIOHibernateAESKeySize,
-                        &cryptvars->ctx.decrypt);
-
-        cryptvars = &_cryptvars;
-        bzero(cryptvars, sizeof(hibernate_cryptvars_t));
-        aes_encrypt_key(vars->wiredCryptKey,
-                        kIOHibernateAESKeySize,
-                        &cryptvars->ctx.encrypt);
-
-        bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
-        bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
-        bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
-        bzero(gIOHibernateCryptWakeVars, sizeof(hibernate_cryptwakevars_t));
-    }
-#endif /* CRYPTO */
+       // encryption data. "iv" is the "initial vector".
+       if (kIOHibernateModeEncrypt & gIOHibernateMode) {
+               static const unsigned char first_iv[AES_BLOCK_SIZE]
+                       = {  0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
+                            0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
+
+               cryptvars = &gIOHibernateCryptWakeContext;
+               bzero(cryptvars, sizeof(IOPolledFileCryptVars));
+               aes_encrypt_key(vars->cryptKey,
+                   kIOHibernateAESKeySize,
+                   &cryptvars->ctx.encrypt);
+               aes_decrypt_key(vars->cryptKey,
+                   kIOHibernateAESKeySize,
+                   &cryptvars->ctx.decrypt);
+
+               cryptvars = &_cryptvars;
+               bzero(cryptvars, sizeof(IOPolledFileCryptVars));
+               for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++) {
+                       vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
+               }
+               aes_encrypt_key(vars->wiredCryptKey,
+                   kIOHibernateAESKeySize,
+                   &cryptvars->ctx.encrypt);
 
-    hibernate_setup_for_wake();
+               bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
+               bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
+               bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
+       }
+#endif /* CRYPTO */
 
-    hibernate_page_list_setall(vars->page_list,
-                               vars->page_list_wired,
-                               &pageCount);
+       hibernate_page_list_setall(vars->page_list,
+           vars->page_list_wired,
+           vars->page_list_pal,
+           false /* !preflight */,
+           /* discard_all */
+           ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
+           && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
+           &pageCount);
 
-    HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
+       HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
 
-    fileExtents = (IOPolledFileExtent *) vars->fileExtents->getBytesNoCopy();
+       fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy();
 
 #if 0
-    count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
-    for (page = 0; page < count; page++)
-    {
-       HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page, 
-               fileExtents[page].start, fileExtents[page].length,
-               fileExtents[page].start + fileExtents[page].length);
-    }
+       count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
+       for (page = 0; page < count; page++) {
+               HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
+                   fileExtents[page].start, fileExtents[page].length,
+                   fileExtents[page].start + fileExtents[page].length);
+       }
+#endif
+
+       needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
+       AbsoluteTime_to_scalar(&compTime) = 0;
+       compBytes = 0;
+
+       clock_get_uptime(&allTime);
+       IOService::getPMRootDomain()->pmStatsRecordEvent(
+               kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
+       do{
+               compressedSize   = 0;
+               uncompressedSize = 0;
+               svPageCount      = 0;
+               zvPageCount      = 0;
+
+               IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
+
+               HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
+                   ml_get_interrupts_enabled());
+               err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState,
+                   // abortable if not low battery
+                   !IOService::getPMRootDomain()->mustHibernate());
+               HIBLOG("IOHibernatePollerOpen(%x)\n", err);
+               pollerOpen = (kIOReturnSuccess == err);
+               if (!pollerOpen) {
+                       break;
+               }
+
+               if (vars->volumeCryptKeySize) {
+                       err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
+                       HIBLOG("IOPolledFilePollersSetEncryptionKey(%x)\n", err);
+                       vars->hwEncrypt = (kIOReturnSuccess == err);
+                       bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
+                       if (vars->hwEncrypt) {
+                               header->options |= kIOHibernateOptionHWEncrypt;
+                       }
+               }
+
+               // copy file block extent list if larger than header
+
+               count = vars->fileVars->fileExtents->getLength();
+               if (count > sizeof(header->fileExtentMap)) {
+                       count -= sizeof(header->fileExtentMap);
+                       err = IOHibernatePolledFileWrite(vars->fileVars,
+                           ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
+                       if (kIOReturnSuccess != err) {
+                               break;
+                       }
+               }
+
+               hibernateBase = HIB_BASE; /* Defined in PAL headers */
+               hibernateEnd = (segHIBB + segSizeHIB);
+
+               // copy out restore1 code
+
+               for (count = 0;
+                   (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+                   count += segLen) {
+                       for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++) {
+                               gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
+                       }
+               }
+
+               page = atop_32(kvtophys(hibernateBase));
+               count = atop_32(round_page(hibernateEnd) - hibernateBase);
+               header->restore1CodePhysPage = page;
+               header->restore1CodeVirt = hibernateBase;
+               header->restore1PageCount = count;
+               header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint)      - hibernateBase;
+               header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
+
+               if (uuid_parse(&gIOHibernateBridgeBootSessionUUIDString[0], &header->bridgeBootSessionUUID[0])) {
+                       bzero(&header->bridgeBootSessionUUID[0], sizeof(header->bridgeBootSessionUUID));
+               }
+
+               // sum __HIB seg, with zeros for the stack
+               src = (uint8_t *) trunc_page(hibernateBase);
+               for (page = 0; page < count; page++) {
+                       if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0])) {
+                               restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
+                       } else {
+                               restore1Sum += 0x00000000;
+                       }
+                       src += page_size;
+               }
+               sum1 = restore1Sum;
+
+               // write the __HIB seg, with zeros for the stack
+
+               src = (uint8_t *) trunc_page(hibernateBase);
+               count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
+               if (count) {
+                       err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
+                       if (kIOReturnSuccess != err) {
+                               break;
+                       }
+               }
+               err = IOHibernatePolledFileWrite(vars->fileVars,
+                   (uint8_t *) NULL,
+                   &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
+                   cryptvars);
+               if (kIOReturnSuccess != err) {
+                       break;
+               }
+               src = &gIOHibernateRestoreStackEnd[0];
+               count = round_page(hibernateEnd) - ((uintptr_t) src);
+               if (count) {
+                       err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
+                       if (kIOReturnSuccess != err) {
+                               break;
+                       }
+               }
+
+               if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) {
+                       vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
+                       vars->fileVars->encryptEnd   = UINT64_MAX;
+                       HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
+               }
+
+               // write the preview buffer
+
+               if (vars->previewBuffer) {
+                       ppnum = 0;
+                       count = 0;
+                       do{
+                               phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
+                               pageAndCount[0] = atop_64(phys64);
+                               pageAndCount[1] = atop_32(segLen);
+                               err = IOHibernatePolledFileWrite(vars->fileVars,
+                                   (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
+                                   cryptvars);
+                               if (kIOReturnSuccess != err) {
+                                       break;
+                               }
+                               count += segLen;
+                               ppnum += sizeof(pageAndCount);
+                       }while (phys64);
+                       if (kIOReturnSuccess != err) {
+                               break;
+                       }
+
+                       src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
+
+                       ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
+
+                       count = vars->previewBuffer->getLength();
+
+                       header->previewPageListSize = ppnum;
+                       header->previewSize = count + ppnum;
+
+                       for (page = 0; page < count; page += page_size) {
+                               phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
+                               sum1 += hibernate_sum_page(src + page, atop_64(phys64));
+                       }
+                       err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
+                       if (kIOReturnSuccess != err) {
+                               break;
+                       }
+               }
+
+               // mark areas for no save
+               IOMemoryDescriptor * ioBuffer;
+               ioBuffer = IOPolledFileGetIOBuffer(vars->fileVars);
+               for (count = 0;
+                   (phys64 = ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+                   count += segLen) {
+                       hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+                           atop_64(phys64), atop_32(segLen),
+                           kIOHibernatePageStateFree);
+                       pageCount -= atop_32(segLen);
+               }
+
+               for (count = 0;
+                   (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+                   count += segLen) {
+                       hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+                           atop_64(phys64), atop_32(segLen),
+                           kIOHibernatePageStateFree);
+                       pageCount -= atop_32(segLen);
+               }
+
+               // copy out bitmap of pages available for trashing during restore
+
+               bitmap_size = vars->page_list_wired->list_size;
+               src = (uint8_t *) vars->page_list_wired;
+               err = IOHibernatePolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
+               if (kIOReturnSuccess != err) {
+                       break;
+               }
+
+               // mark more areas for no save, but these are not available
+               // for trashing during restore
+
+               hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
+
+
+               page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
+               count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
+               hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+                   page, count,
+                   kIOHibernatePageStateFree);
+               pageCount -= count;
+
+               if (vars->previewBuffer) {
+                       for (count = 0;
+                           (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+                           count += segLen) {
+                               hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+                                   atop_64(phys64), atop_32(segLen),
+                                   kIOHibernatePageStateFree);
+                               pageCount -= atop_32(segLen);
+                       }
+               }
+
+               for (count = 0;
+                   (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+                   count += segLen) {
+                       hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+                           atop_64(phys64), atop_32(segLen),
+                           kIOHibernatePageStateFree);
+                       pageCount -= atop_32(segLen);
+               }
+
+#if KASAN
+               vm_size_t shadow_pages_free = atop_64(shadow_ptop) - atop_64(shadow_pnext);
+
+               /* no need to save unused shadow pages */
+               hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+                   atop_64(shadow_pnext),
+                   shadow_pages_free,
+                   kIOHibernatePageStateFree);
 #endif
 
-    needEncryptStart = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
-
-    AbsoluteTime_to_scalar(&compTime) = 0;
-    AbsoluteTime_to_scalar(&decoTime) = 0;
-
-    clock_get_uptime(&allTime);
-
-    do 
-    {
-        compressedSize   = 0;
-        uncompressedSize = 0;
-        iterDone         = false;
-        pageType         = 0;          // wired pages first
-
-        IOPolledFileSeek(vars->fileVars, sizeof(IOHibernateImageHeader));
-    
-        HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n", 
-                ml_get_interrupts_enabled());
-        err = IOHibernatePollerOpen(vars->fileVars, kIOPolledBeforeSleepState, vars->ioBuffer);
-        HIBLOG("IOHibernatePollerOpen(%x)\n", err);
-        pollerOpen = (kIOReturnSuccess == err);
-        if (!pollerOpen)
-            break;
-    
-        // copy file block extent list if larger than header
-    
-        count = vars->fileExtents->getLength();
-        if (count > sizeof(header->fileExtentMap))
-        {
-            count -= sizeof(header->fileExtentMap);
-            err = IOPolledFileWrite(vars->fileVars,
-                                    ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
-            if (kIOReturnSuccess != err)
-                break;
-        }
-
-        // copy out restore1 code
-    
-        page = atop_32(sectHIBB);
-        count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
-        header->restore1CodePage = page;
-        header->restore1PageCount = count;
-        header->restore1CodeOffset = ((uint32_t) &hibernate_machine_entrypoint)      - sectHIBB;
-        header->restore1StackOffset = ((uint32_t) &gIOHibernateRestoreStackEnd[0]) - 64 - sectHIBB;
-
-        // sum __HIB sect, with zeros for the stack
-        src = (uint8_t *) trunc_page(sectHIBB);
-        for (page = 0; page < count; page++)
-        {
-            if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
-                restore1Sum += hibernate_sum(src, page_size);
-            else
-                restore1Sum += 0x10000001;
-            src += page_size;
-        }
-        sum1 = restore1Sum;
-    
-        // write the __HIB sect, with zeros for the stack
-
-        src = (uint8_t *) trunc_page(sectHIBB);
-        count = ((uint32_t) &gIOHibernateRestoreStack[0]) - trunc_page(sectHIBB);
-        if (count)
-        {
-            err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
-            if (kIOReturnSuccess != err)
-                break;
-        }
-        err = IOPolledFileWrite(vars->fileVars, 
-                                        (uint8_t *) 0,
-                                        &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
-                                        cryptvars);
-        if (kIOReturnSuccess != err)
-            break;
-        src = &gIOHibernateRestoreStackEnd[0];
-        count = round_page(sectHIBB + sectSizeHIB) - ((uint32_t) src);
-        if (count)
-        {
-            err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
-            if (kIOReturnSuccess != err)
-                break;
-        }
-
-        // write the preview buffer
-
-        addr64_t phys64;
-        IOByteCount segLen;
-
-        if (vars->previewBuffer)
-        {
-            ppnum = 0;
-            count = 0;
-            do
-            {
-                phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen);
-                pageAndCount[0] = atop_64(phys64);
-                pageAndCount[1] = atop_32(segLen);
-                err = IOPolledFileWrite(vars->fileVars, 
-                                        (const uint8_t *) &pageAndCount, sizeof(pageAndCount), 
-                                        cryptvars);
-                if (kIOReturnSuccess != err)
-                    break;
-                count += segLen;
-                ppnum += sizeof(pageAndCount);
-            }
-            while (phys64);
-            if (kIOReturnSuccess != err)
-                break;
-
-            src = (uint8_t *) vars->previewBuffer->getSourceSegment(0, NULL);
-            count = vars->previewBuffer->getLength();
-
-            header->previewPageListSize = ppnum;
-            header->previewSize = count + ppnum;
-
-            for (page = 0; page < count; page += page_size)
-                sum1 += hibernate_sum(src + page, page_size);
-
-            err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
-            if (kIOReturnSuccess != err)
-                break;
-        }
-
-        // mark areas for no save
-    
-        for (count = 0;
-            (phys64 = vars->ioBuffer->getPhysicalSegment64(count, &segLen));
-            count += segLen)
-        {
-            hibernate_set_page_state(vars->page_list, vars->page_list_wired, 
-                                        atop_64(phys64), atop_32(segLen),
-                                        kIOHibernatePageStateFree);
-            pageCount -= atop_32(segLen);
-        }
-    
-        for (count = 0;
-            (phys64 = vars->srcBuffer->getPhysicalSegment64(count, &segLen));
-            count += segLen)
-        {
-            hibernate_set_page_state(vars->page_list, vars->page_list_wired, 
-                                        atop_64(phys64), atop_32(segLen),
-                                        kIOHibernatePageStateFree);
-            pageCount -= atop_32(segLen);
-        }
-
-        // copy out bitmap of pages available for trashing during restore
-    
-        bitmap_size = vars->page_list_wired->list_size;
-        src = (uint8_t *) vars->page_list_wired;
-        err = IOPolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
-        if (kIOReturnSuccess != err)
-            break;
-
-        // mark more areas for no save, but these are not available 
-        // for trashing during restore
-
-       hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
-    
-        page = atop_32(sectHIBB);
-        count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
-        hibernate_set_page_state(vars->page_list, vars->page_list_wired,
-                                        page, count,
-                                        kIOHibernatePageStateFree);
-        pageCount -= count;
-
-        if (vars->previewBuffer) for (count = 0;
-                                        (phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen));
-                                        count += segLen)
-        {
-            hibernate_set_page_state(vars->page_list, vars->page_list_wired, 
-                                        atop_64(phys64), atop_32(segLen),
-                                        kIOHibernatePageStateFree);
-            pageCount -= atop_32(segLen);
-        }
-
-        src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
-    
-        ppnum     = 0;
-        pagesDone = 0;
-        lastBlob = 0;
-    
-        HIBLOG("writing %d pages\n", pageCount);
-
-        do
-        {
-            count = hibernate_page_list_iterate(pageType ? vars->page_list : vars->page_list_wired,
-                                                    &ppnum);
-//          kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
-    
-            iterDone = !count;
-
-            pageAndCount[0] = ppnum;
-            pageAndCount[1] = count;
-            err = IOPolledFileWrite(vars->fileVars, 
-                                    (const uint8_t *) &pageAndCount, sizeof(pageAndCount), 
-                                    cryptvars);
-            if (kIOReturnSuccess != err)
-                break;
-
-            for (page = 0; page < count; page++)
-            {
-                err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(ppnum), page_size);
-                if (err)
-                {
-                    HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%d] %x\n", __LINE__, ppnum, err);
-                    break;
-                }
-    
-                sum = hibernate_sum(src, page_size);
-   
-                clock_get_uptime(&startTime);
-
-                pageCompressedSize = WKdm_compress ((WK_word*) src, (WK_word*) (src + page_size), PAGE_SIZE_IN_WORDS);
-    
-                clock_get_uptime(&endTime);
-                ADD_ABSOLUTETIME(&compTime, &endTime);
-                SUB_ABSOLUTETIME(&compTime, &startTime);
-    
-                if (kIOHibernateModeEncrypt & gIOHibernateMode)
-                    pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
-
-                if (pageCompressedSize > page_size)
-                {
-//                  HIBLOG("------------lose: %d\n", pageCompressedSize);
-                    pageCompressedSize = page_size;
-                }
-
-                if (pageCompressedSize != page_size)
-                    data = (src + page_size);
-                else
-                    data = src;
-
-                tag = pageCompressedSize | kIOHibernateTagSignature;
-
-                if (pageType)
-                    sum2 += sum;
-                else
-                    sum1 += sum;
-
-                if (needEncryptStart && (ppnum >= atop_32(sectDATAB)))
-                {
-                    // start encrypting partway into the data about to be written
-                    vars->fileVars->encryptStart = (vars->fileVars->position + AES_BLOCK_SIZE - 1) 
-                                                    & ~(AES_BLOCK_SIZE - 1);
-                    needEncryptStart = false;
-                }
-
-                err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
-                if (kIOReturnSuccess != err)
-                    break;
-
-                err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
-                if (kIOReturnSuccess != err)
-                    break;
-
-                compressedSize += pageCompressedSize;
-                if (pageCompressedSize)
-                    uncompressedSize += page_size;
-                ppnum++;
-                pagesDone++;
-
-               if (vars->consoleMapping && (0 == (1023 & pagesDone)))
+               src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
+               compressed = src + page_size;
+               scratch    = compressed + page_size;
+
+               pagesDone  = 0;
+               lastBlob   = 0;
+
+               HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
+                   bitmap_size, header->previewSize,
+                   pageCount, vars->fileVars->position);
+
+               enum
+               // pageType
                {
-                   blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
-                   if (blob != lastBlob)
-                   {
-                       ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
-                       lastBlob = blob;
-                   }
-               }
-                if (0 == (8191 & pagesDone))
-                {
-                    clock_get_uptime(&endTime);
-                    SUB_ABSOLUTETIME(&endTime, &allTime);
-                    absolutetime_to_nanoseconds(endTime, &nsec);
-                    progressStamp = nsec / 750000000ULL;
-                    if (progressStamp != lastProgressStamp)
-                    {
-                        lastProgressStamp = progressStamp;
-                        HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
-                    }
-                }
-            }
-            if (kIOReturnSuccess != err)
-                break;
-            if (iterDone && !pageType)
-            {
-                err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
-                if (kIOReturnSuccess != err)
-                    break;
-
-                iterDone = false;
-                pageType = 1;
-                ppnum = 0;
-                image1Size = vars->fileVars->position;
-                if (cryptvars)
-                {
-                    bcopy(&cryptvars->aes_iv[0], 
-                            &gIOHibernateCryptWakeContext.aes_iv[0], 
-                            sizeof(cryptvars->aes_iv));
-                    cryptvars = &gIOHibernateCryptWakeContext;
-                }
-                HIBLOG("image1Size %qd\n", image1Size);
-            }
-        }
-        while (!iterDone);
-        if (kIOReturnSuccess != err)
-            break;
-        err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
-        if (kIOReturnSuccess != err)
-            break;
-
-        // Header:
-    
-        header->imageSize    = vars->fileVars->position;
-        header->image1Size   = image1Size;
-        header->bitmapSize   = bitmap_size;
-        header->pageCount    = pageCount;
-        header->encryptStart = vars->fileVars->encryptStart;
-    
-        header->restore1Sum  = restore1Sum;
-        header->image1Sum    = sum1;
-        header->image2Sum    = sum2;
-    
-        count = vars->fileExtents->getLength();
-        if (count > sizeof(header->fileExtentMap))
-        {
-            header->fileExtentMapSize = count;
-            count = sizeof(header->fileExtentMap);
-        }
-        else
-            header->fileExtentMapSize = sizeof(header->fileExtentMap);
-        bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
-    
-        IOPolledFileSeek(vars->fileVars, 0);
-        err = IOPolledFileWrite(vars->fileVars,
-                                    (uint8_t *) header, sizeof(IOHibernateImageHeader), 
-                                    cryptvars);
-        if (kIOReturnSuccess != err)
-            break;
-        err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
-        if (kIOReturnSuccess != err)
-            break;
-        err = IOHibernatePollerIODone(vars->fileVars, true);
-        if (kIOReturnSuccess != err)
-            break;
-    }
-    while (false);
-    
-    clock_get_uptime(&endTime);
-    SUB_ABSOLUTETIME(&endTime, &allTime);
-    absolutetime_to_nanoseconds(endTime, &nsec);
-    HIBLOG("all time: %qd ms, ", 
-               nsec / 1000000ULL);
-
-    absolutetime_to_nanoseconds(compTime, &nsec);
-    HIBLOG("comp time: %qd ms, ", 
-               nsec / 1000000ULL);
-
-    absolutetime_to_nanoseconds(decoTime, &nsec);
-    HIBLOG("deco time: %qd ms, ", 
-               nsec / 1000000ULL);
-
-    HIBLOG("\nimage %qd, uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n", 
-               header->imageSize,
-               uncompressedSize, atop_32(uncompressedSize), compressedSize,
-               uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
-               sum1, sum2);
-
-    if (vars->fileVars->io)
-        (void) IOHibernatePollerIODone(vars->fileVars, false);
-
-    if (pollerOpen)
-        IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
-
-    if (vars->consoleMapping)
-        ProgressUpdate(gIOHibernateGraphicsInfo, 
-                        vars->consoleMapping, 0, kIOHibernateProgressCount);
-
-    HIBLOG("hibernate_write_image done(%x)\n", err);
-
-    // should we come back via regular wake, set the state in memory.
-    gIOHibernateState = kIOHibernateStateInactive;
-
-    if (kIOReturnSuccess == err)
-    {
-       if (kIOHibernateModeSleep & gIOHibernateMode)
-       {
-           return (kIOHibernatePostWriteSleep);
-       }
-       else if(kIOHibernateModeRestart & gIOHibernateMode)
-       {
-           return (kIOHibernatePostWriteRestart);
-       }
-       else
-       {
-           /* by default, power down */
-           return (kIOHibernatePostWriteHalt);
-       }
-    }
-    else if (kIOReturnAborted == err)
-    {
-       return (kIOHibernatePostWriteWake);
-    }
-    else
-    {
-       /* on error, sleep */
-       return (kIOHibernatePostWriteSleep);
-    }
+                       kWired          = 0x02,
+                       kEncrypt        = 0x01,
+                       kWiredEncrypt   = kWired | kEncrypt,
+                       kWiredClear     = kWired,
+                       kUnwiredEncrypt = kEncrypt
+               };
+
+               bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features()));
+
+               for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--) {
+                       if (kUnwiredEncrypt == pageType) {
+                               // start unwired image
+                               if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) {
+                                       vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
+                                       vars->fileVars->encryptEnd   = UINT64_MAX;
+                                       HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
+                               }
+                               bcopy(&cryptvars->aes_iv[0],
+                                   &gIOHibernateCryptWakeContext.aes_iv[0],
+                                   sizeof(cryptvars->aes_iv));
+                               cryptvars = &gIOHibernateCryptWakeContext;
+                       }
+                       for (iterDone = false, ppnum = 0; !iterDone;) {
+                               if (cpuAES && (pageType == kWiredClear)) {
+                                       count = 0;
+                               } else {
+                                       count = hibernate_page_list_iterate((kWired & pageType) ? vars->page_list_wired : vars->page_list,
+                                           &ppnum);
+                               }
+//              kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
+                               iterDone = !count;
+
+                               if (!cpuAES) {
+                                       if (count && (kWired & pageType) && needEncrypt) {
+                                               uint32_t checkIndex;
+                                               for (checkIndex = 0;
+                                                   (checkIndex < count)
+                                                   && (((kEncrypt & pageType) == 0) == pmap_is_noencrypt(ppnum + checkIndex));
+                                                   checkIndex++) {
+                                               }
+                                               if (!checkIndex) {
+                                                       ppnum++;
+                                                       continue;
+                                               }
+                                               count = checkIndex;
+                                       }
+                               }
+
+                               switch (pageType) {
+                               case kWiredEncrypt:   wiredPagesEncrypted += count; break;
+                               case kWiredClear:     wiredPagesClear     += count; break;
+                               case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
+                               }
+
+                               if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */
+                               } else {
+                                       pageAndCount[0] = ppnum;
+                                       pageAndCount[1] = count;
+                                       err = IOHibernatePolledFileWrite(vars->fileVars,
+                                           (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
+                                           cryptvars);
+                                       if (kIOReturnSuccess != err) {
+                                               break;
+                                       }
+                               }
+
+                               for (page = ppnum; page < (ppnum + count); page++) {
+                                       err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
+                                       if (err) {
+                                               HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
+                                               break;
+                                       }
+
+                                       sum = hibernate_sum_page(src, page);
+                                       if (kWired & pageType) {
+                                               sum1 += sum;
+                                       } else {
+                                               sum2 += sum;
+                                       }
+
+                                       clock_get_uptime(&startTime);
+                                       wkresult = WKdm_compress_new((const WK_word*) src,
+                                           (WK_word*) compressed,
+                                           (WK_word*) scratch,
+                                           page_size - 4);
+
+                                       clock_get_uptime(&endTime);
+                                       ADD_ABSOLUTETIME(&compTime, &endTime);
+                                       SUB_ABSOLUTETIME(&compTime, &startTime);
+
+                                       compBytes += page_size;
+                                       pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
+
+                                       if (pageCompressedSize == 0) {
+                                               pageCompressedSize = 4;
+                                               data = src;
+
+                                               if (*(uint32_t *)src) {
+                                                       svPageCount++;
+                                               } else {
+                                                       zvPageCount++;
+                                               }
+                                       } else {
+                                               if (pageCompressedSize != page_size) {
+                                                       data = compressed;
+                                               } else {
+                                                       data = src;
+                                               }
+                                       }
+
+                                       tag = pageCompressedSize | kIOHibernateTagSignature;
+                                       err = IOHibernatePolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
+                                       if (kIOReturnSuccess != err) {
+                                               break;
+                                       }
+
+                                       err = IOHibernatePolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
+                                       if (kIOReturnSuccess != err) {
+                                               break;
+                                       }
+
+                                       compressedSize += pageCompressedSize;
+                                       uncompressedSize += page_size;
+                                       pagesDone++;
+
+                                       if (vars->consoleMapping && (0 == (1023 & pagesDone))) {
+                                               blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
+                                               if (blob != lastBlob) {
+                                                       ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
+                                                       lastBlob = blob;
+                                               }
+                                       }
+                                       if (0 == (8191 & pagesDone)) {
+                                               clock_get_uptime(&endTime);
+                                               SUB_ABSOLUTETIME(&endTime, &allTime);
+                                               absolutetime_to_nanoseconds(endTime, &nsec);
+                                               progressStamp = nsec / 750000000ULL;
+                                               if (progressStamp != lastProgressStamp) {
+                                                       lastProgressStamp = progressStamp;
+                                                       HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
+                                               }
+                                       }
+                               }
+                               if (kIOReturnSuccess != err) {
+                                       break;
+                               }
+                               ppnum = page;
+                       }
+
+                       if (kIOReturnSuccess != err) {
+                               break;
+                       }
+
+                       if ((kEncrypt & pageType) && vars->fileVars->encryptStart) {
+                               vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
+                               HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
+                       }
+
+                       if (kWiredEncrypt != pageType) {
+                               // end of image1/2 - fill to next block
+                               err = IOHibernatePolledFileWrite(vars->fileVars, NULL, 0, cryptvars);
+                               if (kIOReturnSuccess != err) {
+                                       break;
+                               }
+                       }
+                       if (kWiredClear == pageType) {
+                               // enlarge wired image for test
+//              err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
+
+                               // end wired image
+                               header->encryptStart = vars->fileVars->encryptStart;
+                               header->encryptEnd   = vars->fileVars->encryptEnd;
+                               image1Size = vars->fileVars->position;
+                               HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
+                                   image1Size, header->encryptStart, header->encryptEnd);
+                       }
+               }
+               if (kIOReturnSuccess != err) {
+                       if (kIOReturnOverrun == err) {
+                               // update actual compression ratio on not enough space (for retry)
+                               gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
+                       }
+
+                       // update partial amount written (for IOPolledFileClose cleanup/unmap)
+                       header->imageSize = vars->fileVars->position;
+                       break;
+               }
+
+               // Header:
+
+               header->imageSize    = vars->fileVars->position;
+               header->image1Size   = image1Size;
+               header->bitmapSize   = bitmap_size;
+               header->pageCount    = pageCount;
+
+               header->restore1Sum  = restore1Sum;
+               header->image1Sum    = sum1;
+               header->image2Sum    = sum2;
+               header->sleepTime    = gIOLastSleepTime.tv_sec;
+
+               header->compression     = (compressedSize << 8) / uncompressedSize;
+               gIOHibernateCompression = header->compression;
+
+               count = vars->fileVars->fileExtents->getLength();
+               if (count > sizeof(header->fileExtentMap)) {
+                       header->fileExtentMapSize = count;
+                       count = sizeof(header->fileExtentMap);
+               } else {
+                       header->fileExtentMapSize = sizeof(header->fileExtentMap);
+               }
+               bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
+
+               header->deviceBase      = vars->fileVars->block0;
+               header->deviceBlockSize = vars->fileVars->blockSize;
+
+               IOPolledFileSeek(vars->fileVars, 0);
+               err = IOHibernatePolledFileWrite(vars->fileVars,
+                   (uint8_t *) header, sizeof(IOHibernateImageHeader),
+                   cryptvars);
+               if (kIOReturnSuccess != err) {
+                       break;
+               }
+               err = IOHibernatePolledFileWrite(vars->fileVars, NULL, 0, cryptvars);
+       }while (false);
+
+       clock_get_uptime(&endTime);
+
+       IOService::getPMRootDomain()->pmStatsRecordEvent(
+               kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
+
+       SUB_ABSOLUTETIME(&endTime, &allTime);
+       absolutetime_to_nanoseconds(endTime, &nsec);
+       HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
+
+       absolutetime_to_nanoseconds(compTime, &nsec);
+       HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
+           compBytes,
+           nsec / 1000000ULL,
+           nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+       absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
+       HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
+           vars->fileVars->cryptBytes,
+           nsec / 1000000ULL,
+           nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+       HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
+           header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
+           uncompressedSize, atop_32(uncompressedSize), compressedSize,
+           uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
+           sum1, sum2);
+
+       HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
+           svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
+
+       if (pollerOpen) {
+               IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted );
+       }
+
+       if (vars->consoleMapping) {
+               ProgressUpdate(gIOHibernateGraphicsInfo,
+                   vars->consoleMapping, 0, kIOHibernateProgressCount);
+       }
+
+       HIBLOG("hibernate_write_image done(%x)\n", err);
+
+       // should we come back via regular wake, set the state in memory.
+       gIOHibernateState = kIOHibernateStateInactive;
+
+       KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, wiredPagesEncrypted,
+           wiredPagesClear, dirtyPagesEncrypted);
+
+       if (kIOReturnSuccess == err) {
+               if (kIOHibernateModeSleep & gIOHibernateMode) {
+                       return kIOHibernatePostWriteSleep;
+               } else if (kIOHibernateModeRestart & gIOHibernateMode) {
+                       return kIOHibernatePostWriteRestart;
+               } else {
+                       /* by default, power down */
+                       return kIOHibernatePostWriteHalt;
+               }
+       } else if (kIOReturnAborted == err) {
+               return kIOHibernatePostWriteWake;
+       } else {
+               /* on error, sleep */
+               return kIOHibernatePostWriteSleep;
+       }
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
-extern "C" void 
+extern "C" void
 hibernate_machine_init(void)
 {
-    IOReturn     err;
-    uint32_t     sum;
-    uint32_t     pagesDone;
-    AbsoluteTime allTime, endTime;
-    uint64_t     nsec;
-    uint32_t     lastProgressStamp = 0;
-    uint32_t     progressStamp;
-    uint64_t    progressZeroPosition = 0;
-    uint32_t    blob, lastBlob = (uint32_t) -1L;
-    hibernate_cryptvars_t * cryptvars = 0;
-
-    IOHibernateVars * vars  = &gIOHibernateVars;
-
-    if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
-       return;
-
-    sum = gIOHibernateCurrentHeader->actualImage1Sum;
-    pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
-
-    HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, image1Size %qx, conflictCount %d, nextFree %x\n",
-           gIOHibernateState, pagesDone, sum, gIOHibernateCurrentHeader->image1Size,
+       IOReturn     err;
+       uint32_t     sum;
+       uint32_t     pagesDone;
+       uint32_t     pagesRead = 0;
+       AbsoluteTime startTime, compTime;
+       AbsoluteTime allTime, endTime;
+       AbsoluteTime startIOTime, endIOTime;
+       uint64_t     nsec, nsecIO;
+       uint64_t     compBytes;
+       uint32_t     lastProgressStamp = 0;
+       uint32_t     progressStamp;
+       IOPolledFileCryptVars * cryptvars = NULL;
+
+       IOHibernateVars * vars  = &gIOHibernateVars;
+       bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
+
+       if (!vars->fileVars || !vars->fileVars->pollers) {
+               return;
+       }
+
+       sum = gIOHibernateCurrentHeader->actualImage1Sum;
+       pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
+
+       if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) {
+               HIBLOG("regular wake\n");
+               return;
+       }
+
+       HIBPRINT("diag %x %x %x %x\n",
+           gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
+           gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
+
+#define t40ms(x)        (tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)
+#define tStat(x, y)     gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
+       tStat(booterStart, booterStart);
+       gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart;
+       tStat(booterDuration0, booterTime0);
+       tStat(booterDuration1, booterTime1);
+       tStat(booterDuration2, booterTime2);
+       tStat(booterDuration, booterTime);
+       tStat(booterConnectDisplayDuration, connectDisplayTime);
+       tStat(booterSplashDuration, splashTime);
+       tStat(trampolineDuration, trampolineTime);
+
+       gIOHibernateStats->image1Size  = gIOHibernateCurrentHeader->image1Size;
+       gIOHibernateStats->imageSize   = gIOHibernateCurrentHeader->imageSize;
+       gIOHibernateStats->image1Pages = pagesDone;
+
+       /* HIBERNATE_stats */
+       KDBG(IOKDBG_CODE(DBG_HIBERNATE, 14), gIOHibernateStats->smcStart,
+           gIOHibernateStats->booterStart, gIOHibernateStats->booterDuration,
+           gIOHibernateStats->trampolineDuration);
+
+       HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
+           gIOHibernateStats->booterStart,
+           gIOHibernateStats->smcStart,
+           gIOHibernateStats->booterDuration0,
+           gIOHibernateStats->booterDuration1,
+           gIOHibernateStats->booterDuration2,
+           gIOHibernateStats->booterDuration,
+           gIOHibernateStats->booterConnectDisplayDuration,
+           gIOHibernateStats->booterSplashDuration,
+           gIOHibernateStats->trampolineDuration);
+
+       HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
+           gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
            gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
 
-    if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
-    {
-       HIBLOG("regular wake\n");
-       return;
-    }
-
-    HIBPRINT("diag %x %x %x %x\n",
-           gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1], 
-           gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]); 
-
-    HIBPRINT("video %x %d %d %d\n",
-           gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth, 
-           gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height); 
-
-    if ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)
-        hibernate_page_list_discard(vars->page_list);
-
-    boot_args *args = (boot_args *) PE_state.bootArgs;
-
-    if (vars->videoMapping 
-       && gIOHibernateGraphicsInfo->physicalAddress
-       && (args->Video.v_baseAddr == gIOHibernateGraphicsInfo->physicalAddress))
-    {
-        vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height 
-                                        * gIOHibernateGraphicsInfo->rowBytes);
-        IOMapPages(kernel_map, 
-                    vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
-                    vars->videoMapSize, kIOMapInhibitCache );
-    }
-
-    uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
-
-    if (gIOHibernateWakeMapSize)
-    {
-       err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(gIOHibernateWakeMap), 
-                                                   gIOHibernateWakeMapSize);
-       if (kIOReturnSuccess == err)
-           hibernate_newruntime_map(src, gIOHibernateWakeMapSize, 
-                                    gIOHibernateCurrentHeader->systemTableOffset);
-       gIOHibernateWakeMap = 0;
-       gIOHibernateWakeMapSize = 0;
-    }
-
-    uint32_t decoOffset;
-
-    clock_get_uptime(&allTime);
-
-    HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
-    err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0);
-    HIBLOG("IOHibernatePollerOpen(%x)\n", err);
-
-    if (gIOHibernateCurrentHeader->previewSize)
-        progressZeroPosition = gIOHibernateCurrentHeader->previewSize 
-                             + gIOHibernateCurrentHeader->fileExtentMapSize 
-                             - sizeof(gIOHibernateCurrentHeader->fileExtentMap) 
-                             + ptoa_64(gIOHibernateCurrentHeader->restore1PageCount);
-
-    IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
-
-    if (vars->videoMapSize)
-    {
-        lastBlob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
-                        / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
-        ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, 0, lastBlob);
-    }
-
-    cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
-    if (kIOHibernateModeEncrypt & gIOHibernateMode)
-    {
-        cryptvars = &gIOHibernateCryptWakeContext;
-        bcopy(&gIOHibernateCryptWakeVars->aes_iv[0], 
-                &cryptvars->aes_iv[0], 
-                sizeof(cryptvars->aes_iv));
-    }
-
-    // kick off the read ahead
-    vars->fileVars->io          = false;
-    vars->fileVars->bufferHalf   = 0;
-    vars->fileVars->bufferLimit  = 0;
-    vars->fileVars->lastRead     = 0;
-    vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
-
-    IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
-    vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
-    // --
-
-    HIBLOG("hibernate_machine_init reading\n");
-
-    uint32_t * header = (uint32_t *) src;
-    sum = 0;
-
-    do
-    {
-       unsigned int count;
-       unsigned int page;
-        uint32_t     tag;
-       vm_offset_t  ppnum, compressedSize;
-
-       err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
-       if (kIOReturnSuccess != err)
-           break;
-
-       ppnum = header[0];
-       count = header[1];
+       if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
+           && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))) {
+               hibernate_page_list_discard(vars->page_list);
+       }
 
-//     HIBPRINT("(%x, %x)\n", ppnum, count);
+       cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : NULL;
 
-       if (!count)
-           break;
+       if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount) {
+               panic("handoff overflow");
+       }
 
-       for (page = 0; page < count; page++)
-       {
-           err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
-           if (kIOReturnSuccess != err)
-               break;
+       IOHibernateHandoff * handoff;
+       bool                 done                   = false;
+       bool                 foundCryptData         = false;
+       bool                 foundVolumeEncryptData = false;
+
+       for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
+           !done;
+           handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount]) {
+//     HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
+               uint8_t * data = &handoff->data[0];
+               switch (handoff->type) {
+               case kIOHibernateHandoffTypeEnd:
+                       done = true;
+                       break;
+
+               case kIOHibernateHandoffTypeGraphicsInfo:
+                       if (handoff->bytecount == sizeof(*gIOHibernateGraphicsInfo)) {
+                               bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
+                       }
+                       break;
 
-           compressedSize = kIOHibernateTagLength & tag;
-           if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength))
-           {
-               err = kIOReturnIPCError;
-               break;
-           }
+               case kIOHibernateHandoffTypeCryptVars:
+                       if (cryptvars) {
+                               hibernate_cryptwakevars_t *
+                                   wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
+                               bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
+                       }
+                       foundCryptData = true;
+                       bzero(data, handoff->bytecount);
+                       break;
+
+               case kIOHibernateHandoffTypeVolumeCryptKey:
+                       if (handoff->bytecount == vars->volumeCryptKeySize) {
+                               bcopy(data, &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
+                               foundVolumeEncryptData = true;
+                       } else {
+                               panic("kIOHibernateHandoffTypeVolumeCryptKey(%d)", handoff->bytecount);
+                       }
+                       break;
 
-           if (!compressedSize)
-           {
-               ppnum++;
-               pagesDone++;
-               continue;
-           }
+               case kIOHibernateHandoffTypeMemoryMap:
 
-           err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
-           if (kIOReturnSuccess != err)
-               break;
+                       clock_get_uptime(&allTime);
 
-           if (compressedSize < page_size)
-           {
-               decoOffset = page_size;
-               WKdm_decompress((WK_word*) src, (WK_word*) (src + decoOffset), PAGE_SIZE_IN_WORDS);
-           }
-           else
-               decoOffset = 0;
+                       hibernate_newruntime_map(data, handoff->bytecount,
+                           gIOHibernateCurrentHeader->systemTableOffset);
 
-           sum += hibernate_sum((src + decoOffset), page_size);
+                       clock_get_uptime(&endTime);
 
-           err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
-           if (err)
-           {
-               HIBLOG("IOMemoryDescriptorReadToPhysical [%d] %x\n", ppnum, err);
-               break;
-           }
-
-           ppnum++;
-           pagesDone++;
-
-            if (vars->videoMapSize && (0 == (1023 & pagesDone)))
-            {
-                blob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
-                        / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
-                if (blob != lastBlob)
-                {
-                    ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, lastBlob, blob);
-                    lastBlob = blob;
-                }
-            }
-
-           if (0 == (8191 & pagesDone))
-           {
-               clock_get_uptime(&endTime);
-               SUB_ABSOLUTETIME(&endTime, &allTime);
-               absolutetime_to_nanoseconds(endTime, &nsec);
-               progressStamp = nsec / 750000000ULL;
-               if (progressStamp != lastProgressStamp)
+                       SUB_ABSOLUTETIME(&endTime, &allTime);
+                       absolutetime_to_nanoseconds(endTime, &nsec);
+
+                       HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
+
+                       break;
+
+               case kIOHibernateHandoffTypeDeviceTree:
                {
-                   lastProgressStamp = progressStamp;
-                   HIBPRINT("pages %d (%d%%)\n", pagesDone, 
-                           (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
+//                 DTEntry chosen = NULL;
+//                 HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen));
+               }
+               break;
+
+               default:
+                       done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
+                       break;
+               }
+       }
+
+       if (vars->hwEncrypt && !foundVolumeEncryptData) {
+               panic("no volumeCryptKey");
+       } else if (cryptvars && !foundCryptData) {
+               panic("hibernate handoff");
+       }
+
+       HIBPRINT("video 0x%llx %d %d %d status %x\n",
+           gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
+           gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
+
+       if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress) {
+               vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
+                   * gIOHibernateGraphicsInfo->rowBytes);
+               if (vars->videoMapSize > vars->videoAllocSize) {
+                       vars->videoMapSize = 0;
+               } else {
+                       IOMapPages(kernel_map,
+                           vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
+                           vars->videoMapSize, kIOMapInhibitCache );
                }
-           }
        }
-    }
-    while (true);
 
-    if (kIOReturnSuccess != err)
-       panic("Hibernate restore error %x", err);
+       if (vars->videoMapSize) {
+               ProgressUpdate(gIOHibernateGraphicsInfo,
+                   (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
+       }
+
+       uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
+       uint8_t * compressed = src + page_size;
+       uint8_t * scratch    = compressed + page_size;
+       uint32_t  decoOffset;
+
+       clock_get_uptime(&allTime);
+       AbsoluteTime_to_scalar(&compTime) = 0;
+       compBytes = 0;
+
+       HIBLOG("IOPolledFilePollersOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
+       err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledAfterSleepState, false);
+       clock_get_uptime(&startIOTime);
+       endTime = startIOTime;
+       SUB_ABSOLUTETIME(&endTime, &allTime);
+       absolutetime_to_nanoseconds(endTime, &nsec);
+       HIBLOG("IOPolledFilePollersOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
+
+       if (vars->hwEncrypt) {
+               err = IOPolledFilePollersSetEncryptionKey(vars->fileVars,
+                   &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
+               HIBLOG("IOPolledFilePollersSetEncryptionKey(%x) %ld\n", err, vars->volumeCryptKeySize);
+               if (kIOReturnSuccess != err) {
+                       panic("IOPolledFilePollersSetEncryptionKey(0x%x)", err);
+               }
+               cryptvars = NULL;
+       }
 
-    gIOHibernateCurrentHeader->actualImage2Sum = sum;
+       IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
 
-    if (vars->fileVars->io)
-        (void) IOHibernatePollerIODone(vars->fileVars, false);
+       // kick off the read ahead
+       vars->fileVars->bufferHalf   = 0;
+       vars->fileVars->bufferLimit  = 0;
+       vars->fileVars->lastRead     = 0;
+       vars->fileVars->readEnd      = gIOHibernateCurrentHeader->imageSize;
+       vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
+       vars->fileVars->cryptBytes   = 0;
+       AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
 
-    err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
+       err = IOPolledFileRead(vars->fileVars, NULL, 0, cryptvars);
+       vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
+       // --
 
-    if (vars->videoMapSize)
-        ProgressUpdate(gIOHibernateGraphicsInfo, 
-                        (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
+       HIBLOG("hibernate_machine_init reading\n");
 
-    clock_get_uptime(&endTime);
-    SUB_ABSOLUTETIME(&endTime, &allTime);
-    absolutetime_to_nanoseconds(endTime, &nsec);
+       uint32_t * header = (uint32_t *) src;
+       sum = 0;
 
-    HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %qd ms\n", 
-               pagesDone, sum, nsec / 1000000ULL);
+       while (kIOReturnSuccess == err) {
+               unsigned int count;
+               unsigned int page;
+               uint32_t     tag;
+               vm_offset_t  ppnum, compressedSize;
+
+               err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
+               if (kIOReturnSuccess != err) {
+                       break;
+               }
+
+               ppnum = header[0];
+               count = header[1];
+
+//     HIBPRINT("(%x, %x)\n", ppnum, count);
+
+               if (!count) {
+                       break;
+               }
+
+               for (page = 0; page < count; page++) {
+                       err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
+                       if (kIOReturnSuccess != err) {
+                               break;
+                       }
+
+                       compressedSize = kIOHibernateTagLength & tag;
+                       if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength)) {
+                               err = kIOReturnIPCError;
+                               break;
+                       }
+
+                       err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
+                       if (kIOReturnSuccess != err) {
+                               break;
+                       }
+
+                       if (compressedSize < page_size) {
+                               decoOffset = page_size;
+                               clock_get_uptime(&startTime);
+
+                               if (compressedSize == 4) {
+                                       int i;
+                                       uint32_t *s, *d;
+
+                                       s = (uint32_t *)src;
+                                       d = (uint32_t *)(uintptr_t)compressed;
+
+                                       for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++) {
+                                               *d++ = *s;
+                                       }
+                               } else {
+                                       WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, compressedSize);
+                               }
+                               clock_get_uptime(&endTime);
+                               ADD_ABSOLUTETIME(&compTime, &endTime);
+                               SUB_ABSOLUTETIME(&compTime, &startTime);
+                               compBytes += page_size;
+                       } else {
+                               decoOffset = 0;
+                       }
+
+                       sum += hibernate_sum_page((src + decoOffset), ppnum);
+                       err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
+                       if (err) {
+                               HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
+                               break;
+                       }
+
+                       ppnum++;
+                       pagesDone++;
+                       pagesRead++;
+
+                       if (0 == (8191 & pagesDone)) {
+                               clock_get_uptime(&endTime);
+                               SUB_ABSOLUTETIME(&endTime, &allTime);
+                               absolutetime_to_nanoseconds(endTime, &nsec);
+                               progressStamp = nsec / 750000000ULL;
+                               if (progressStamp != lastProgressStamp) {
+                                       lastProgressStamp = progressStamp;
+                                       HIBPRINT("pages %d (%d%%)\n", pagesDone,
+                                           (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
+                               }
+                       }
+               }
+       }
+       if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages)) {
+               err = kIOReturnLockedRead;
+       }
+
+       if (kIOReturnSuccess != err) {
+               panic("Hibernate restore error %x", err);
+       }
+
+       gIOHibernateCurrentHeader->actualImage2Sum = sum;
+       gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
+
+       clock_get_uptime(&endIOTime);
+
+       err = IOPolledFilePollersClose(vars->fileVars, kIOPolledAfterSleepState);
+
+       clock_get_uptime(&endTime);
+
+       IOService::getPMRootDomain()->pmStatsRecordEvent(
+               kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
+       IOService::getPMRootDomain()->pmStatsRecordEvent(
+               kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
+
+       SUB_ABSOLUTETIME(&endTime, &allTime);
+       absolutetime_to_nanoseconds(endTime, &nsec);
+
+       SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
+       absolutetime_to_nanoseconds(endIOTime, &nsecIO);
+
+       gIOHibernateStats->kernelImageReadDuration = nsec / 1000000ULL;
+       gIOHibernateStats->imagePages              = pagesDone;
+
+       HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
+           pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
+           nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
+
+       absolutetime_to_nanoseconds(compTime, &nsec);
+       HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
+           compBytes,
+           nsec / 1000000ULL,
+           nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+       absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
+       HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
+           vars->fileVars->cryptBytes,
+           nsec / 1000000ULL,
+           nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+       KDBG(IOKDBG_CODE(DBG_HIBERNATE, 2), pagesRead, pagesDone);
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+void
+IOHibernateSetWakeCapabilities(uint32_t capability)
+{
+       if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
+               gIOHibernateStats->wakeCapability = capability;
+
+               if (kIOPMSystemCapabilityGraphics & capability) {
+                       vm_compressor_do_warmup();
+               }
+       }
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+void
+IOHibernateSystemRestart(void)
+{
+       static uint8_t    noteStore[32] __attribute__((aligned(32)));
+       IORegistryEntry * regEntry;
+       const OSSymbol *  sym;
+       OSData *          noteProp;
+       OSData *          data;
+       uintptr_t *       smcVars;
+       uint8_t *         smcBytes;
+       size_t            len;
+       addr64_t          element;
+
+       data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
+       if (!data) {
+               return;
+       }
+
+       smcVars = (typeof(smcVars))data->getBytesNoCopy();
+       smcBytes = (typeof(smcBytes))smcVars[1];
+       len = smcVars[0];
+       if (len > sizeof(noteStore)) {
+               len = sizeof(noteStore);
+       }
+       noteProp = OSData::withCapacity(3 * sizeof(element));
+       if (!noteProp) {
+               return;
+       }
+       element = len;
+       noteProp->appendBytes(&element, sizeof(element));
+       element = crc32(0, smcBytes, len);
+       noteProp->appendBytes(&element, sizeof(element));
+
+       bcopy(smcBytes, noteStore, len);
+       element = (addr64_t) &noteStore[0];
+       element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
+       noteProp->appendBytes(&element, sizeof(element));
+
+       if (!gIOOptionsEntry) {
+               regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
+               gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
+               if (regEntry && !gIOOptionsEntry) {
+                       regEntry->release();
+               }
+       }
+
+       sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
+       if (gIOOptionsEntry && sym) {
+               gIOOptionsEntry->setProperty(sym, noteProp);
+       }
+       if (noteProp) {
+               noteProp->release();
+       }
+       if (sym) {
+               sym->release();
+       }
+}