]> git.saurik.com Git - apple/xnu.git/blobdiff - iokit/Kernel/IOHibernateIO.cpp
xnu-3789.70.16.tar.gz
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
index ae66cb9b8a91f1deb1f1ca66c46fdd48ad61896d..953c6d44530a4a6fed9776f99cae9509ae474b50 100644 (file)
@@ -1,8 +1,8 @@
 /*
- * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2016 Apple Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
- * 
+ *
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
- * 
+ *
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
- * 
+ *
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
- * 
+ *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
-
 /*
 
 Sleep:
@@ -58,12 +57,12 @@ Sleep:
   by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
   The image header and block list are written. The header includes the second file extent so
   only the header block is needed to read the file, regardless of filesystem.
-  The kernel section "__HIB" is written uncompressed to the image. This section of code and data 
+  The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data 
   (only) is used to decompress the image during wake/boot.
   Some additional pages are removed from the bitmaps - the buffers used for hibernation.
   The bitmaps are written to the image.
   More areas are removed from the bitmaps (after they have been written to the image) - the 
-  section "__HIB" pages and interrupt stack.
+  segment "__HIB" pages and interrupt stack.
   Each wired page is compressed and written and then each non-wired page. Compression and 
   disk writes are in parallel.
   The image header is written to the start of the file and the polling driver closed.
@@ -151,81 +150,96 @@ to restrict I/O ops.
 #include <IOKit/pwr_mgt/IOPowerConnection.h>
 #include "IOPMPowerStateQueue.h"
 #include <IOKit/IOBufferMemoryDescriptor.h>
-#include <crypto/aes.h>
+#include <IOKit/AppleKeyStoreInterface.h>
+#include <libkern/crypto/aes.h>
 
 #include <sys/uio.h>
 #include <sys/conf.h>
 #include <sys/stat.h>
 #include <sys/fcntl.h>                       // (FWRITE, ...)
 #include <sys/sysctl.h>
+#include <sys/kdebug.h>
+#include <stdint.h>
 
 #include <IOKit/IOHibernatePrivate.h>
 #include <IOKit/IOPolledInterface.h>
 #include <IOKit/IONVRAM.h>
 #include "IOHibernateInternal.h"
-#include "WKdm.h"
+#include <vm/WKdm_new.h>
+#include <vm/vm_protos.h>
 #include "IOKitKernelInternal.h"
+#include <pexpert/device_tree.h>
 
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+#include <machine/pal_routines.h>
+#include <machine/pal_hibernate.h>
+#include <i386/tsc.h>
+#include <i386/cpuid.h>
 
-OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
-
-OSMetaClassDefineReservedUnused(IOPolledInterface, 0);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
+extern "C" addr64_t            kvtophys(vm_offset_t va);
+extern "C" ppnum_t             pmap_find_phys(pmap_t pmap, addr64_t va);
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
+#define        DISABLE_TRIM            0
+#define TRIM_DELAY             25000
+
+extern unsigned int            save_kdebug_enable;
 extern uint32_t                gIOHibernateState;
 uint32_t                       gIOHibernateMode;
 static char                    gIOHibernateBootSignature[256+1];
 static char                    gIOHibernateFilename[MAXPATHLEN+1];
-static uint32_t                        gIOHibernateFreeRatio = 0;              // free page target (percent)
-uint32_t                       gIOHibernateFreeTime  = 0*1000; // max time to spend freeing pages (ms)
+static uint32_t                        gIOHibernateFreeRatio = 0;       // free page target (percent)
+uint32_t                       gIOHibernateFreeTime  = 0*1000;  // max time to spend freeing pages (ms)
+static uint64_t                        gIOHibernateCompression = 0x80;  // default compression 50%
+boolean_t                       gIOHibernateStandbyDisabled;
 
 static IODTNVRAM *             gIOOptionsEntry;
 static IORegistryEntry *       gIOChosenEntry;
-#ifdef __i386__
-static const OSSymbol *         gIOCreateEFIDevicePathSymbol;
-#endif
 
-static IOPolledFileIOVars                gFileVars;
-static IOHibernateVars                   gIOHibernateVars;
-static struct kern_direct_file_io_ref_t * gIOHibernateFileRef;
-static hibernate_cryptvars_t             gIOHibernateCryptWakeContext;
+static const OSSymbol *        gIOHibernateBootImageKey;
 
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+#if defined(__i386__) || defined(__x86_64__)
 
-enum { kXPRamAudioVolume = 8 };
-enum { kDefaultIOSize = 128 * 1024 };
-enum { kVideoMapSize  = 32 * 1024 * 1024 };
+static const OSSymbol *        gIOHibernateRTCVariablesKey;
+static const OSSymbol *         gIOHibernateBoot0082Key;
+static const OSSymbol *         gIOHibernateBootNextKey;
+static OSData *                        gIOHibernateBoot0082Data;
+static OSData *                        gIOHibernateBootNextData;
+static OSObject *              gIOHibernateBootNextSave;
 
-#ifndef kIOMediaPreferredBlockSizeKey
-#define kIOMediaPreferredBlockSizeKey  "Preferred Block Size"
-#endif
+static IOPolledFileIOVars *     gDebugImageFileVars;
+static IOLock             *     gDebugImageLock;
 
-#ifndef kIOBootPathKey 
-#define kIOBootPathKey                 "bootpath"
-#endif
-#ifndef kIOSelectedBootDeviceKey       
-#define kIOSelectedBootDeviceKey       "boot-device"
-#endif
+#endif /* defined(__i386__) || defined(__x86_64__) */
+
+static IOLock *                           gFSLock;
+static uint32_t                           gFSState;
+static thread_call_t                      gIOHibernateTrimCalloutEntry;
+static IOPolledFileIOVars                gFileVars;
+static IOHibernateVars                   gIOHibernateVars;
+static IOPolledFileCryptVars             gIOHibernateCryptWakeContext;
+static hibernate_graphics_t              _hibernateGraphics;
+static hibernate_graphics_t *            gIOHibernateGraphicsInfo = &_hibernateGraphics;
+static hibernate_statistics_t            _hibernateStats;
+static hibernate_statistics_t *                  gIOHibernateStats = &_hibernateStats;
+
+enum 
+{
+    kFSIdle      = 0,
+    kFSOpening   = 2,
+    kFSOpened    = 3,
+    kFSTimedOut  = 4,
+    kFSTrimDelay = 5
+};
+
+static IOReturn IOHibernateDone(IOHibernateVars * vars);
+static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature);
+static void     IOSetBootImageNVRAM(OSData * data);
 
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
-enum { kIOHibernateMinPollersNeeded = 2 };
+enum { kDefaultIOSize = 128 * 1024 };
+enum { kVideoMapSize  = 80 * 1024 * 1024 };
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
@@ -243,7 +257,7 @@ IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
         addr64_t    dstAddr64;
         IOByteCount dstLen;
 
-        dstAddr64 = md->getPhysicalSegment64(offset, &dstLen);
+        dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
         if (!dstAddr64)
             break;
 
@@ -281,7 +295,7 @@ IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
         addr64_t    srcAddr64;
         IOByteCount dstLen;
 
-        srcAddr64 = md->getPhysicalSegment64(offset, &dstLen);
+        srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
         if (!srcAddr64)
             break;
 
@@ -371,957 +385,509 @@ hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
-static IOReturn
-IOHibernatePollerProbe(IOPolledFileIOVars * vars, IOService * target)
+IOReturn
+IOHibernateSystemSleep(void)
 {
-    IOReturn            err = kIOReturnError;
-    int32_t            idx;
-    IOPolledInterface * poller;
-
-    for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
-    {
-        poller = (IOPolledInterface *) vars->pollers->getObject(idx);
-        err = poller->probe(target);
-        if (err)
-        {
-            HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
-            break;
-        }
-    }
+    IOReturn   err;
+    OSData *   nvramData;
+    OSObject * obj;
+    OSString * str;
+    OSNumber * num;
+    bool       dsSSD, vmflush, swapPinned;
+    IOHibernateVars * vars;
+    uint64_t   setFileSize = 0;
 
-    return (err);
-}
+    gIOHibernateState = kIOHibernateStateInactive;
 
-static IOReturn
-IOHibernatePollerOpen(IOPolledFileIOVars * vars, uint32_t state, IOMemoryDescriptor * md)
-{
-    IOReturn            err = kIOReturnError;
-    int32_t            idx;
-    IOPolledInterface * poller;
+    gIOHibernateDebugFlags = 0;
+    if (kIOLogHibernate & gIOKitDebug)
+       gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
 
-    for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
+    if (IOService::getPMRootDomain()->getHibernateSettings(
+        &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime))
     {
-        poller = (IOPolledInterface *) vars->pollers->getObject(idx);
-        err = poller->open(state, md);
-        if (err)
-        {
-            HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
-            break;
-        }
+        if (kIOHibernateModeSleep & gIOHibernateMode)
+            // default to discard clean for safe sleep
+            gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive 
+                                | kIOHibernateModeDiscardCleanActive);
     }
 
-    return (err);
-}
-
-static IOReturn
-IOHibernatePollerClose(IOPolledFileIOVars * vars, uint32_t state)
-{
-    IOReturn            err = kIOReturnError;
-    int32_t            idx;
-    IOPolledInterface * poller;
-
-    for (idx = 0;
-         (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
-         idx++)
+    if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
     {
-        err = poller->close(state);
-        if (err)
-            HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
+       if ((str = OSDynamicCast(OSString, obj)))
+           strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
+                           sizeof(gIOHibernateFilename));
+       obj->release();
     }
 
-    return (err);
-}
-
-static void
-IOHibernatePollerIOComplete(void *   target,
-                            void *   parameter,
-                            IOReturn status,
-                            UInt64   actualByteCount)
-{
-    IOPolledFileIOVars * vars = (IOPolledFileIOVars *) parameter;
-
-    vars->ioStatus = status;
-}
-
-static IOReturn
-IOHibernatePollerIO(IOPolledFileIOVars * vars, 
-                    uint32_t operation, uint32_t bufferOffset, 
-                   uint64_t deviceOffset, uint64_t length)
-{
-
-    IOReturn            err = kIOReturnError;
-    IOPolledInterface * poller;
-    IOPolledCompletion  completion;
+    if (!gIOHibernateMode || !gIOHibernateFilename[0])
+       return (kIOReturnUnsupported);
 
-    completion.target    = 0;
-    completion.action    = &IOHibernatePollerIOComplete;
-    completion.parameter = vars;
+    HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
 
-    vars->ioStatus = -1;
+    vars = IONew(IOHibernateVars, 1);
+    if (!vars) return (kIOReturnNoMemory);
+    bzero(vars, sizeof(*vars));
 
-    poller = (IOPolledInterface *) vars->pollers->getObject(0);
-    err = poller->startIO(operation, bufferOffset, deviceOffset + vars->block0, length, completion);
-    if (err)
-        HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
+    IOLockLock(gFSLock);
+    if (!gIOHibernateTrimCalloutEntry)
+    {
+        gIOHibernateTrimCalloutEntry = thread_call_allocate(&IOHibernateSystemPostWakeTrim, &gFSLock);
+    }
+    IOHibernateSystemPostWakeTrim(NULL, NULL);
+    if (kFSIdle != gFSState)
+    {
+       HIBLOG("hibernate file busy\n");
+       IOLockUnlock(gFSLock);
+       IODelete(vars, IOHibernateVars, 1);
+        return (kIOReturnBusy);
+    }
+    gFSState = kFSOpening;
+    IOLockUnlock(gFSLock);
 
-    return (err);
-}
+    swapPinned = false;
+    do
+    {
+        vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
+                                   2 * page_size + WKdm_SCRATCH_BUF_SIZE_INTERNAL, page_size);
 
-static IOReturn
-IOHibernatePollerIODone(IOPolledFileIOVars * vars, bool abortable)
-{
-    IOReturn            err = kIOReturnSuccess;
-    int32_t            idx = 0;
-    IOPolledInterface * poller;
+       vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, 
+                                   ptoa_64(gIOHibernateHandoffPageCount), page_size);
 
-    while (-1 == vars->ioStatus)
-    {
-        for (idx = 0; 
-           (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
-             idx++)
+        if (!vars->srcBuffer || !vars->handoffBuffer)
         {
-           IOReturn newErr;
-            newErr = poller->checkForWork();
-           if ((newErr == kIOReturnAborted) && !abortable)
-               newErr = kIOReturnSuccess;
-           if (kIOReturnSuccess == err)
-               err = newErr;
+            err = kIOReturnNoMemory;
+            break;
         }
-    }
 
-    if (err)
-    {
-       HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
-    }
-    else 
-    {
-       err = vars->ioStatus;
-       if (kIOReturnSuccess != err)
-           HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
-    }
+       if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey)))
+       {
+           if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMinSize = num->unsigned64BitValue();
+           obj->release();
+       }
+       if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey)))
+       {
+           if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMaxSize = num->unsigned64BitValue();
+           obj->release();
+       }
 
-    return (err);
-}
+        boolean_t encryptedswap = true;
+        uint32_t pageCount;
+        AbsoluteTime startTime, endTime;
+        uint64_t nsec;
 
-IOReturn
-IOPolledInterface::checkAllForWork(void)
-{
-    IOReturn            err = kIOReturnNotReady;
-    int32_t            idx;
-    IOPolledInterface * poller;
+       bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
+       gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
+       gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
 
-    IOHibernateVars * vars  = &gIOHibernateVars;
+       vmflush = ((kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
+        err = hibernate_alloc_page_lists(&vars->page_list, 
+                                        &vars->page_list_wired,
+                                        &vars->page_list_pal);
+        if (KERN_SUCCESS != err) break;
 
-    if (!vars->fileVars || !vars->fileVars->pollers)
-       return (err);
+       err = hibernate_pin_swap(TRUE);
+       if (KERN_SUCCESS != err) break;
+       swapPinned = true;
 
-    for (idx = 0;
-            (poller = (IOPolledInterface *) vars->fileVars->pollers->getObject(idx));
-            idx++)
-    {
-        err = poller->checkForWork();
-        if (err)
-            HIBLOG("IOPolledInterface::checkAllForWork[%d] 0x%x\n", idx, err);
+       if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode))
+       {
+           hibernate_page_list_setall(vars->page_list,
+                                      vars->page_list_wired,
+                                      vars->page_list_pal,
+                                      true /* preflight */,
+                                      vmflush /* discard */,
+                                      &pageCount);
+           PE_Video consoleInfo;
+           bzero(&consoleInfo, sizeof(consoleInfo));
+           IOService::getPlatform()->getConsoleInfo(&consoleInfo);
+
+           // estimate: 6% increase in pages compressed
+           // screen preview 2 images compressed 0%
+           setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
+                               + vars->page_list->list_size
+                               + (consoleInfo.v_width * consoleInfo.v_height * 8);
+           enum { setFileRound = 1024*1024ULL };
+           setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
+       
+           HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n", 
+                   pageCount, (100ULL * gIOHibernateCompression) >> 8,
+                   setFileSize, vars->fileMinSize);
+
+           if (!(kIOHibernateModeFileResize & gIOHibernateMode)
+            && (setFileSize < vars->fileMinSize))
+           { 
+               setFileSize = vars->fileMinSize;
+           }
+       }
+    
+       // Invalidate the image file
+    if (gDebugImageLock) {
+        IOLockLock(gDebugImageLock);
+        if (gDebugImageFileVars != 0) {
+            IOSetBootImageNVRAM(0);
+            IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
+        }
+        IOLockUnlock(gDebugImageLock);
     }
 
-    return (err);
-}
+        err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, 0,
+                               gIOHibernateCurrentHeader, sizeof(gIOHibernateCurrentHeader),
+                                &vars->fileVars, &nvramData, 
+                                &vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
 
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+        if (KERN_SUCCESS != err)
+        {
+           IOLockLock(gFSLock);
+           if (kFSOpening != gFSState) err = kIOReturnTimeout;
+           IOLockUnlock(gFSLock);
+       }
 
-struct _OpenFileContext
-{
-    OSData * extents;
-    uint64_t size;
-};
+        if (KERN_SUCCESS != err)
+        {
+           HIBLOG("IOPolledFileOpen(%x)\n", err);
+            break;
+        }
 
-static void
-file_extent_callback(void * ref, uint64_t start, uint64_t length)
-{
-    _OpenFileContext * ctx = (_OpenFileContext *) ref;
-    IOPolledFileExtent extent;
+       // write extents for debug data usage in EFI
+        IOWriteExtentsToFile(vars->fileVars, kIOHibernateHeaderOpenSignature);
 
-    extent.start  = start;
-    extent.length = length;
+        err = IOPolledFilePollersSetup(vars->fileVars, kIOPolledPreflightState);
+        if (KERN_SUCCESS != err) break;
 
-    ctx->extents->appendBytes(&extent, sizeof(extent));
-    ctx->size += length;
-}
+        clock_get_uptime(&startTime);
+        err = hibernate_setup(gIOHibernateCurrentHeader, 
+                                vmflush,
+                                vars->page_list, vars->page_list_wired, vars->page_list_pal);
+        clock_get_uptime(&endTime);
+        SUB_ABSOLUTETIME(&endTime, &startTime);
+        absolutetime_to_nanoseconds(endTime, &nsec);
 
-IOReturn
-IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
-                           IOPolledFileIOVars ** fileVars, OSData ** fileExtents,
-                           OSData ** imagePath)
-{
-    IOReturn                   err = kIOReturnError;
-    IOPolledFileIOVars *       vars;
-    _OpenFileContext           ctx;
-    OSData *                   extentsData;
-    OSNumber *                 num;
-    IORegistryEntry *          part = 0;
-    OSDictionary *             matching;
-    OSIterator *               iter;
-    dev_t                      hibernate_image_dev;
-    uint64_t                   maxiobytes;
-
-    vars = &gFileVars;
-    do
-    {
-       HIBLOG("sizeof(IOHibernateImageHeader) == %ld\n", sizeof(IOHibernateImageHeader));
-       if (sizeof(IOHibernateImageHeader) != 512)
-           continue;
-    
-       vars->io           = false;
-       vars->buffer       = (uint8_t *) ioBuffer->getBytesNoCopy();
-       vars->bufferHalf   = 0;
-       vars->bufferOffset = 0;
-       vars->bufferSize   = ioBuffer->getLength() >> 1;
-    
-       extentsData = OSData::withCapacity(32);
-    
-       ctx.extents = extentsData;
-       ctx.size    = 0;
-       vars->fileRef = kern_open_file_for_direct_io(filename, 
-                                                   &file_extent_callback, &ctx, 
-                                                   &hibernate_image_dev,
-                                                    &vars->block0,
-                                                    &maxiobytes);
-       if (!vars->fileRef)
-       {
-           err = kIOReturnNoSpace;
-           break;
-       }
-       HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx\n", filename, ctx.size, 
-                    vars->block0, maxiobytes);
-       if (ctx.size < 1*1024*1024)             // check against image size estimate!
-       {
-           err = kIOReturnNoSpace;
-           break;
-       }
+        boolean_t haveSwapPin, hibFileSSD;
+        haveSwapPin = vm_swap_files_pinned();
 
-        if (maxiobytes < vars->bufferSize)
-            vars->bufferSize = maxiobytes;
-    
-       vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
-    
-       matching = IOService::serviceMatching("IOMedia");
-       num = OSNumber::withNumber(major(hibernate_image_dev), 32);
-       matching->setObject(kIOBSDMajorKey, num);
-       num->release();
-       num = OSNumber::withNumber(minor(hibernate_image_dev), 32);
-       matching->setObject(kIOBSDMinorKey, num);
-       num->release();
-       iter = IOService::getMatchingServices(matching);
-       matching->release();
-       if (iter)
-       {
-           part = (IORegistryEntry *) iter->getNextObject();
-           part->retain();
-           iter->release();
-       }
-       if (!part)
-           break;
+        hibFileSSD = (kIOPolledFileSSD & vars->fileVars->flags);
 
-       int minor, major;
-       IORegistryEntry * next;
-       IORegistryEntry * child;
-       OSData * data;
+        HIBLOG("hibernate_setup(%d) took %qd ms, swapPin(%d) ssd(%d)\n",
+                err, nsec / 1000000ULL,
+                haveSwapPin, hibFileSSD);
+        if (KERN_SUCCESS != err) break;
 
-       num = (OSNumber *) part->getProperty(kIOBSDMajorKey);
-       if (!num)
-           break;
-       major = num->unsigned32BitValue();
-       num = (OSNumber *) part->getProperty(kIOBSDMinorKey);
-       if (!num)
-           break;
-       minor = num->unsigned32BitValue();
+        gIOHibernateStandbyDisabled = ((!haveSwapPin || !hibFileSSD));
 
-       hibernate_image_dev = makedev(major, minor);
+        dsSSD = ((0 != (kIOPolledFileSSD & vars->fileVars->flags))
+                && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
 
-        vars->pollers = OSArray::withCapacity(4);
-       if (!vars->pollers)
-           break;
+        if (dsSSD) gIOHibernateCurrentHeader->options |= kIOHibernateOptionSSD | kIOHibernateOptionColor;
+        else       gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
 
-       vars->blockSize = 512;
-       next = part;
-       do
+
+#if defined(__i386__) || defined(__x86_64__)
+       if (!uuid_is_null(vars->volumeCryptKey) &&
+             (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey)))
        {
-            IOPolledInterface * poller;
-           OSObject *          obj;
+           uintptr_t smcVars[2];
+           smcVars[0] = sizeof(vars->volumeCryptKey);
+           smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
 
-           obj = next->getProperty(kIOPolledInterfaceSupportKey);
-           if (kOSBooleanFalse == obj)
-           {
-               vars->pollers->flushCollection();
-               break;
-           }
-            else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
-                vars->pollers->setObject(poller);
-           if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey))))
-               vars->blockSize = num->unsigned32BitValue();
-            child = next;
+           IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
+           bzero(smcVars, sizeof(smcVars));
        }
-       while ((next = child->getParentEntry(gIOServicePlane)) 
-                && child->isParent(next, gIOServicePlane, true));
-
-       HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
-                   major, minor, vars->blockSize, vars->pollers->getCount());
-       if (vars->pollers->getCount() < kIOHibernateMinPollersNeeded)
-           continue;
+#endif
 
-       err = IOHibernatePollerProbe(vars, (IOService *) part);
-       if (kIOReturnSuccess != err)
-           break;
 
-       err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer);
-       if (kIOReturnSuccess != err)
-           break;
+        if (encryptedswap || !uuid_is_null(vars->volumeCryptKey))
+            gIOHibernateMode ^= kIOHibernateModeEncrypt; 
 
-       *fileVars    = vars;
-       *fileExtents = extentsData;
-    
-       // make imagePath
+        if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
+        {
+            vars->videoAllocSize = kVideoMapSize;
+            if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize, VM_KERN_MEMORY_IOKIT))
+                vars->videoMapping = 0;
+        }
 
-       if ((extentsData->getLength() >= sizeof(IOPolledFileExtent)))
-       {
-           char str2[24];
+       // generate crypt keys
+        for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
+            vars->wiredCryptKey[i] = random();
+        for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
+            vars->cryptKey[i] = random();
 
-#if __i386__
-           if (!gIOCreateEFIDevicePathSymbol)
-               gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
+       // set nvram
 
-           snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
+       IOSetBootImageNVRAM(nvramData);
+        nvramData->release();
 
-           err = IOService::getPlatform()->callPlatformFunction(
-                                               gIOCreateEFIDevicePathSymbol, false,
-                                               (void *) part, (void *) str2, (void *) true,
-                                               (void *) &data);
-#else
-           char str1[256];
-           int len = sizeof(str1);
+#if defined(__i386__) || defined(__x86_64__)
+       {
+           struct AppleRTCHibernateVars
+           {
+               uint8_t     signature[4];
+               uint32_t    revision;
+               uint8_t     booterSignature[20];
+               uint8_t     wiredCryptKey[16];
+           };
+           AppleRTCHibernateVars rtcVars;
+           OSData * data;
 
-           if (!part->getPath(str1, &len, gIODTPlane))
-               err = kIOReturnNotFound;
-           else
+           rtcVars.signature[0] = 'A';
+           rtcVars.signature[1] = 'A';
+           rtcVars.signature[2] = 'P';
+           rtcVars.signature[3] = 'L';
+           rtcVars.revision     = 1;
+           bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
+           if (gIOHibernateBootSignature[0])
            {
-               snprintf(str2, sizeof(str2), ",%qx", vars->extentMap[0].start);
-               // (strip the plane name)
-               char * tail = strchr(str1, ':');
-               if (!tail)
-                   tail = str1 - 1;
-               data = OSData::withBytes(tail + 1, strlen(tail + 1));
-               data->appendBytes(str2, strlen(str2));
+               char c;
+               uint8_t value = 0;
+               for (uint32_t i = 0;
+                   (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
+                   i++)
+               {
+                   if (c >= 'a')      c -= 'a' - 10;
+                   else if (c >= 'A') c -= 'A' - 10;
+                   else if (c >= '0') c -= '0';
+                   else               continue;
+                   value = (value << 4) | c;
+                   if (i & 1) rtcVars.booterSignature[i >> 1] = value;
+               }
            }
-#endif
-       if (kIOReturnSuccess == err)
-           *imagePath = data;
-       else
-           HIBLOG("error 0x%x getting path\n", err);
+           data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
+           if (data)
+           { 
+               if (gIOHibernateRTCVariablesKey)
+                   IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
+               data->release();
+           }
+            if (gIOChosenEntry)
+            {
+                data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
+                if (data) gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
+               // set BootNext
+               if (!gIOHibernateBoot0082Data)
+               {
+                   data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
+                   if (data)
+                   {
+                       // AppleNVRAM_EFI_LOAD_OPTION
+                       struct {
+                           uint32_t Attributes;
+                           uint16_t FilePathLength;
+                           uint16_t Desc;
+                       } loadOptionHeader;
+                       loadOptionHeader.Attributes     = 1;
+                       loadOptionHeader.FilePathLength = data->getLength();
+                       loadOptionHeader.Desc           = 0;
+                       gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
+                       if (gIOHibernateBoot0082Data)
+                       {
+                           gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
+                           gIOHibernateBoot0082Data->appendBytes(data);
+                       }
+                   }
+               }
+               if (!gIOHibernateBootNextData)
+               {
+                   uint16_t bits = 0x0082;
+                   gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
+               }
+               if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData)
+               {
+                   gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
+                   gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
+                   gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
+               }
+               // BootNext
+            }
        }
+#endif /* !i386 && !x86_64 */
     }
     while (false);
 
-    if (kIOReturnSuccess != err)
+    if (swapPinned) hibernate_pin_swap(FALSE);
+
+    IOLockLock(gFSLock);
+    if ((kIOReturnSuccess == err) && (kFSOpening != gFSState))
+    {
+       HIBLOG("hibernate file close due timeout\n");
+       err = kIOReturnTimeout;
+    }
+    if (kIOReturnSuccess == err)
+    {
+       gFSState = kFSOpened;
+       gIOHibernateVars = *vars;
+       gFileVars = *vars->fileVars;
+       gFileVars.allocated = false;
+       gIOHibernateVars.fileVars = &gFileVars;
+       gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
+       gIOHibernateState = kIOHibernateStateHibernating;
+    }
+    else
     {
-        HIBLOG("error 0x%x opening hibernation file\n", err);
-       if (vars->fileRef)
-           kern_close_file_for_direct_io(vars->fileRef);
+       IOPolledFileIOVars * fileVars = vars->fileVars;
+       IOHibernateDone(vars);
+    IOPolledFileClose(&fileVars,
+#if DISABLE_TRIM
+                      0, NULL, 0, 0, 0);
+#else
+                      0, NULL, 0, sizeof(IOHibernateImageHeader), setFileSize);
+#endif
+       gFSState = kFSIdle;
     }
+    IOLockUnlock(gFSLock);
 
-    if (part)
-       part->release();
+    if (vars->fileVars) IODelete(vars->fileVars, IOPolledFileIOVars, 1);
+    IODelete(vars, IOHibernateVars, 1);
 
     return (err);
 }
 
-IOReturn
-IOPolledFileClose( IOPolledFileIOVars * vars )
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+static void
+IOSetBootImageNVRAM(OSData * data)
 {
-    if (vars->pollers)
+    IORegistryEntry * regEntry;
+
+    if (!gIOOptionsEntry)
     {
-       IOHibernatePollerClose(vars, kIOPolledPostflightState);
-        vars->pollers->release();
+        regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
+        gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
+        if (regEntry && !gIOOptionsEntry)
+            regEntry->release();
+    }
+    if (gIOOptionsEntry && gIOHibernateBootImageKey)
+    {
+       if (data) gIOOptionsEntry->setProperty(gIOHibernateBootImageKey, data);
+       else
+       {
+           gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
+           gIOOptionsEntry->sync();
+       }
     }
-
-    gIOHibernateFileRef = vars->fileRef;
-
-    bzero(vars, sizeof(IOPolledFileIOVars));
-
-    return (kIOReturnSuccess);
 }
 
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+/* 
+ * Writes header to disk with signature, block size and file extents data.
+ * If there are more than 2 extents, then they are written on second block.
+ */
 static IOReturn
-IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
+IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature)
 {
-    IOPolledFileExtent * extentMap;
-
-    extentMap = vars->extentMap;
+    IOHibernateImageHeader hdr;
+    IOItemCount            count;
+    IOReturn               err = kIOReturnSuccess;
+    int                    rc;
+    IOPolledFileExtent *   fileExtents;
 
-    vars->position = position;
+    fileExtents = (typeof(fileExtents)) vars->fileExtents->getBytesNoCopy();
 
-    while (position >= extentMap->length)
+    memset(&hdr, 0, sizeof(IOHibernateImageHeader));
+    count = vars->fileExtents->getLength();
+    if (count > sizeof(hdr.fileExtentMap))
     {
-       position -= extentMap->length;
-       extentMap++;
+        hdr.fileExtentMapSize = count;
+        count = sizeof(hdr.fileExtentMap);
     }
+    else
+        hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap);
 
-    vars->currentExtent   = extentMap;
-    vars->extentRemaining = extentMap->length - position;
-    vars->extentPosition  = vars->position - position;
+    bcopy(fileExtents, &hdr.fileExtentMap[0], count);
 
-    if (vars->bufferSize <= vars->extentRemaining)
-       vars->bufferLimit = vars->bufferSize;
-    else
-       vars->bufferLimit = vars->extentRemaining;
+    // copy file block extent list if larger than header
+    if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap))
+    {
+            count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap);
+            rc = kern_write_file(vars->fileRef, vars->blockSize, 
+                                 (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)), 
+                                 count, IO_SKIP_ENCRYPTION);
+            if (rc != 0) {
+                HIBLOG("kern_write_file returned %d\n", rc);
+                err = kIOReturnIOError;
+                goto exit;
+            }    
+    }
+    hdr.signature = signature;
+    hdr.deviceBlockSize = vars->blockSize;
+
+    rc = kern_write_file(vars->fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION);
+    if (rc != 0) {
+        HIBLOG("kern_write_file returned %d\n", rc);
+        err = kIOReturnIOError;
+        goto exit;
+    }
 
-    return (kIOReturnSuccess);
+exit:
+    return err;
 }
 
-static IOReturn
-IOPolledFileWrite(IOPolledFileIOVars * vars,
-                    const uint8_t * bytes, IOByteCount size,
-                    hibernate_cryptvars_t * cryptvars)
-{
-    IOReturn    err = kIOReturnSuccess;
-    IOByteCount copy;
-    bool       flush = false;
+extern "C" boolean_t root_is_CF_drive;
 
-    do
-    {
-       if (!bytes && !size)
-       {
-           // seek to end of block & flush
-           size = vars->position & (vars->blockSize - 1);
-           if (size)
-               size = vars->blockSize - size;
-           flush = true;
-            // use some garbage for the fill
-            bytes = vars->buffer + vars->bufferOffset;
-       }
-
-       copy = vars->bufferLimit - vars->bufferOffset;
-       if (copy > size)
-           copy = size;
-       else
-           flush = true;
-
-       if (bytes)
-       {
-           bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
-           bytes += copy;
-       }
-        else
-           bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
-        
-       size -= copy;
-       vars->bufferOffset += copy;
-       vars->position += copy;
-
-       if (flush && vars->bufferOffset)
-       {
-           uint64_t offset = (vars->position - vars->bufferOffset 
-                               - vars->extentPosition + vars->currentExtent->start);
-           uint32_t length = (vars->bufferOffset);
-
-#if CRYPTO
-            if (cryptvars && vars->encryptStart && (vars->position > vars->encryptStart))
-            {
-                uint32_t encryptLen, encryptStart;
-                encryptLen = vars->position - vars->encryptStart;
-                if (encryptLen > length)
-                    encryptLen = length;
-                encryptStart = length - encryptLen;
-                
-                // encrypt the buffer
-                aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
-                                &cryptvars->aes_iv[0],
-                                encryptLen / AES_BLOCK_SIZE,
-                                vars->buffer + vars->bufferHalf + encryptStart,
-                                &cryptvars->ctx.encrypt);
-                // save initial vector for following encrypts
-                bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
-                        &cryptvars->aes_iv[0],
-                        AES_BLOCK_SIZE);
-            }
-#endif /* CRYPTO */
-
-           if (vars->io)
-            {
-               err = IOHibernatePollerIODone(vars, true);
-                if (kIOReturnSuccess != err)
-                    break;
-            }
-
-if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
-//if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
-
-           err = IOHibernatePollerIO(vars, kIOPolledWrite, vars->bufferHalf, offset, length);
-            if (kIOReturnSuccess != err)
-                break;
-           vars->io = true;
-
-           vars->extentRemaining -= vars->bufferOffset;
-           if (!vars->extentRemaining)
-           {
-               vars->currentExtent++;
-               vars->extentRemaining = vars->currentExtent->length;
-               vars->extentPosition  = vars->position;
-                if (!vars->extentRemaining)
-                {
-                    err = kIOReturnOverrun;
-                    break;
-                }
-           }
-
-           vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
-           vars->bufferOffset = 0;
-           if (vars->bufferSize <= vars->extentRemaining)
-               vars->bufferLimit = vars->bufferSize;
-           else
-               vars->bufferLimit = vars->extentRemaining;
-
-           flush = false;
-       }
-    }
-    while (size);
-
-    return (err);
-}
-
-static IOReturn
-IOPolledFileRead(IOPolledFileIOVars * vars,
-                    uint8_t * bytes, IOByteCount size,
-                    hibernate_cryptvars_t * cryptvars)
-{
-    IOReturn    err = kIOReturnSuccess;
-    IOByteCount copy;
-
-//    bytesWritten += size;
-
-    do
-    {
-       copy = vars->bufferLimit - vars->bufferOffset;
-       if (copy > size)
-           copy = size;
-
-       if (bytes)
-       {
-           bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
-           bytes += copy;
-       }
-       size -= copy;
-       vars->bufferOffset += copy;
-//     vars->position += copy;
-
-       if (vars->bufferOffset == vars->bufferLimit)
-       {
-           if (vars->io)
-            {
-               err = IOHibernatePollerIODone(vars, false);
-                if (kIOReturnSuccess != err)
-                    break;
-            }
-            else
-                cryptvars = 0;
-
-if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
-
-           vars->position += vars->lastRead;
-           vars->extentRemaining -= vars->lastRead;
-           vars->bufferLimit = vars->lastRead;
-
-           if (!vars->extentRemaining)
-           {
-               vars->currentExtent++;
-               vars->extentRemaining = vars->currentExtent->length;
-               vars->extentPosition  = vars->position;
-                if (!vars->extentRemaining)
-                {
-                    err = kIOReturnOverrun;
-                    break;
-                }
-           }
-
-           uint64_t length;
-           uint64_t lastReadLength = vars->lastRead;
-           uint64_t offset = (vars->position 
-                               - vars->extentPosition + vars->currentExtent->start);
-           if (vars->extentRemaining <= vars->bufferSize)
-               length = vars->extentRemaining;
-           else
-               length = vars->bufferSize;
-           vars->lastRead = length;
-
-//if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
-
-           err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
-            if (kIOReturnSuccess != err)
-                break;
-           vars->io = true;
-
-           vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
-           vars->bufferOffset = 0;
-
-#if CRYPTO
-            if (cryptvars)
-            {
-                uint8_t thisVector[AES_BLOCK_SIZE];
-                // save initial vector for following decrypts
-                bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
-                bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE, 
-                        &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
-                // decrypt the buffer
-                aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
-                                &thisVector[0],
-                                lastReadLength / AES_BLOCK_SIZE,
-                                vars->buffer + vars->bufferHalf,
-                                &cryptvars->ctx.decrypt);
-            }
-#endif CRYPTO
-       }
-    }
-    while (size);
-
-    return (err);
-}
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-               
-IOReturn
-IOHibernateSystemSleep(void)
+void
+IOOpenDebugDataFile(const char *fname, uint64_t size)
 {
     IOReturn   err;
-    OSData *   data;
-    OSObject * obj;
-    OSString * str;
-    OSNumber * num;
-    OSDictionary *sleepOverrideOptions;
+    OSData *   imagePath = NULL;
+    uint64_t   padding;
 
-    IOHibernateVars * vars  = &gIOHibernateVars;
-
-    if (vars->fileVars && vars->fileVars->fileRef)
-       // already on the way down
-       return (kIOReturnSuccess);
-
-    gIOHibernateState = kIOHibernateStateInactive;
-
-    /* The invocation of IOPMSleepSystemWithOptions() may override
-     * existing hibernation settings.
-     */
-    sleepOverrideOptions = (OSDictionary *)OSDynamicCast( OSDictionary,
-        IOService::getPMRootDomain()->copyProperty(kRootDomainSleepOptionsKey));
-
-
-    /* Hibernate mode overriden by sleep otions ? */
-    obj = NULL;
-    
-    if (sleepOverrideOptions) {
-        obj = sleepOverrideOptions->getObject(kIOHibernateModeKey);
-        if (obj) obj->retain();
+    if (!gDebugImageLock) {
+        gDebugImageLock = IOLockAlloc();
     }
 
-    if(!obj) {
-        obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateModeKey);
-    }
+    if (root_is_CF_drive) return;
 
-    if (obj && (num = OSDynamicCast(OSNumber, obj)) )
-    {
-           gIOHibernateMode = num->unsigned32BitValue();
-        if (kIOHibernateModeSleep & gIOHibernateMode)
-            // default to discard clean for safe sleep
-            gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive 
-                                | kIOHibernateModeDiscardCleanActive);
+    // Try to get a lock, but don't block for getting lock
+    if (!IOLockTryLock(gDebugImageLock)) {
+        HIBLOG("IOOpenDebugDataFile: Failed to get lock\n");
+        return;
     }
 
-    if (obj) obj->release();
-    
-    /* Hibernate free rotio overriden by sleep options ? */
-    obj = NULL;
-    
-    if (sleepOverrideOptions) {
-        obj = sleepOverrideOptions->getObject(kIOHibernateFreeRatioKey);
-        if (obj) obj->retain();
+    if (gDebugImageFileVars ||  !fname || !size) {
+        HIBLOG("IOOpenDebugDataFile: conditions failed\n");
+        goto exit;
     }
-    
-    if(!obj) {
-        obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeRatioKey);
-    }
-    if (obj && (num = OSDynamicCast(OSNumber, obj)))
-    {
-           gIOHibernateFreeRatio = num->unsigned32BitValue();
-    }
-    if (obj) obj->release();
-    
-    if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeTimeKey)))
-    {
-       if ((num = OSDynamicCast(OSNumber, obj)))
-           gIOHibernateFreeTime = num->unsigned32BitValue();
-       obj->release();
-    }
-    if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
-    {
-       if ((str = OSDynamicCast(OSString, obj)))
-           strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
-                           sizeof(gIOHibernateFilename));
-       obj->release();
-    }
-
-    if (sleepOverrideOptions)
-        sleepOverrideOptions->release();
-
-    if (!gIOHibernateMode || !gIOHibernateFilename[0])
-       return (kIOReturnUnsupported);
 
-    HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
+    padding = (PAGE_SIZE*2);  // allocate couple more pages for header and fileextents
+    err = IOPolledFileOpen(fname, size+padding, 32ULL*1024*1024*1024,
+                           NULL, 0,
+                           &gDebugImageFileVars, &imagePath, NULL, 0);
 
-
-    do
+    if ((kIOReturnSuccess == err) && imagePath)
     {
-        vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
-                                   4 * page_size, page_size);
-        vars->ioBuffer  = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, 
-                                   2 * kDefaultIOSize, page_size);
-
-        if (!vars->srcBuffer || !vars->ioBuffer)
-        {
-            err = kIOReturnNoMemory;
-            break;
-        }
-
-        err = IOPolledFileOpen(gIOHibernateFilename, vars->ioBuffer,
-                                &vars->fileVars, &vars->fileExtents, &data);
-        if (KERN_SUCCESS != err)
-        {
-           HIBLOG("IOPolledFileOpen(%x)\n", err);
-            break;
+        if ((gDebugImageFileVars->fileSize < (size+padding)) ||
+            (gDebugImageFileVars->fileExtents->getLength() > PAGE_SIZE)) {
+            // Can't use the file
+            IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
+            HIBLOG("IOOpenDebugDataFile: too many file extents\n");
+            goto exit;
         }
-       if (vars->fileVars->fileRef)
-       {
-           // invalidate the image file
-           gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
-           int err = kern_write_file(vars->fileVars->fileRef, 0,
-                                       (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
-            if (KERN_SUCCESS != err)
-                HIBLOG("kern_write_file(%d)\n", err);
-       }
-
-       bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
 
-        boolean_t encryptedswap;
-        err = hibernate_setup(gIOHibernateCurrentHeader, 
-                                gIOHibernateFreeRatio, gIOHibernateFreeTime,
-                                &vars->page_list, &vars->page_list_wired, &encryptedswap);
-        if (KERN_SUCCESS != err)
-        {
-           HIBLOG("hibernate_setup(%d)\n", err);
-            break;
-        }
-
-        if (encryptedswap)
-            gIOHibernateMode ^= kIOHibernateModeEncrypt; 
+        // write extents for debug data usage in EFI
+        IOWriteExtentsToFile(gDebugImageFileVars, kIOHibernateHeaderOpenSignature);
+        IOSetBootImageNVRAM(imagePath);
+    }
 
-        vars->videoAllocSize = kVideoMapSize;
-        if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
-            vars->videoMapping = 0;
+exit:
+    IOLockUnlock(gDebugImageLock);
 
-       // generate crypt keys
-        for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
-            vars->wiredCryptKey[i] = random();
-        for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
-            vars->cryptKey[i] = random();
+    if (imagePath) imagePath->release();
+    return;
+}
 
-       // set nvram
+void
+IOCloseDebugDataFile()
+{
+    IOSetBootImageNVRAM(0);
 
-        IORegistryEntry * regEntry;
-        if (!gIOOptionsEntry)
-        {
-            regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
-            gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
-            if (regEntry && !gIOOptionsEntry)
-                regEntry->release();
+    if (gDebugImageLock) {
+        IOLockLock(gDebugImageLock);
+        if (gDebugImageFileVars != 0) {
+            IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
         }
-        if (!gIOChosenEntry)
-            gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
-
-       if (gIOOptionsEntry)
-       {
-            const OSSymbol *  sym;
-
-            sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
-            if (sym)
-            {
-                gIOOptionsEntry->setProperty(sym, data);
-                sym->release();
-            }
-            data->release();
-
-#ifdef __ppc__
-            size_t           len;
-            char              valueString[16];
-
-           vars->saveBootDevice = gIOOptionsEntry->copyProperty(kIOSelectedBootDeviceKey);
-            if (gIOChosenEntry)
-            {
-               OSData * bootDevice = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOBootPathKey));
-               if (bootDevice)
-               {
-                   sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
-                   OSString * str2 = OSString::withCStringNoCopy((const char *) bootDevice->getBytesNoCopy());
-                   if (sym && str2)
-                       gIOOptionsEntry->setProperty(sym, str2);
-                   if (sym)
-                       sym->release();
-                   if (str2)
-                       str2->release();
-               }
-                data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMemorySignatureKey));
-                if (data)
-                {
-                    vars->haveFastBoot = true;
-
-                    len = sprintf(valueString, "0x%lx", *((UInt32 *)data->getBytesNoCopy()));
-                    data = OSData::withBytes(valueString, len + 1);
-                    sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
-                    if (sym && data)
-                        gIOOptionsEntry->setProperty(sym, data);
-                    if (sym)
-                        sym->release();
-                    if (data)
-                        data->release();
-                }
-                data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
-                if (data)
-                    gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
-            }
-#endif /* __ppc__ */
-#ifdef __i386__
-           struct AppleRTCHibernateVars
-           {
-               uint8_t     signature[4];
-               uint32_t    revision;
-               uint8_t     booterSignature[20];
-               uint8_t     wiredCryptKey[16];
-           };
-           AppleRTCHibernateVars rtcVars;
-
-           rtcVars.signature[0] = 'A';
-           rtcVars.signature[1] = 'A';
-           rtcVars.signature[2] = 'P';
-           rtcVars.signature[3] = 'L';
-           rtcVars.revision     = 1;
-           bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
-           if (gIOHibernateBootSignature[0])
-           {
-               char c;
-               uint8_t value = 0;
-               for (uint32_t i = 0;
-                   (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
-                   i++)
-               {
-                   if (c >= 'a')
-                       c -= 'a' - 10;
-                   else if (c >= 'A')
-                       c -= 'A' - 10;
-                   else if (c >= '0')
-                       c -= '0';
-                   else
-                       continue;
-                   value = (value << 4) | c;
-                   if (i & 1)
-                       rtcVars.booterSignature[i >> 1] = value;
-               }
-           }
-           data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
-           if (data)
-           { 
-                       IOService::getPMRootDomain()->setProperty(kIOHibernateRTCVariablesKey, data);
-               
-                       if( gIOOptionsEntry )
-                       {
-                               if( gIOHibernateMode & kIOHibernateModeSwitch )
-                               {
-                                       const OSSymbol *sym;
-                                       sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSwitchVarsKey);
-                                       if( sym )
-                                       {
-                                               gIOOptionsEntry->setProperty(sym, data); /* intentional insecure backup of rtc boot vars */
-                                               sym->release();
-                                       }
-                               }       
-                       }
-
-                       data->release();
-           }
-            if (gIOChosenEntry)
-            {
-                data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
-                if (data)
-                    gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
-            }
-#else /* !__i386__ */
-            if (kIOHibernateModeEncrypt & gIOHibernateMode)
-            {
-                data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
-                sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
-                if (sym && data)
-                    gIOOptionsEntry->setProperty(sym, data);
-                if (sym)
-                    sym->release();
-                if (data)
-                    data->release();
-                if (false && gIOHibernateBootSignature[0])
-                {
-                    data = OSData::withCapacity(16);
-                    sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
-                    if (sym && data)
-                    {
-                        char c;
-                        uint8_t value = 0;
-                        for (uint32_t i = 0; (c = gIOHibernateBootSignature[i]); i++)
-                        {
-                            if (c >= 'a')
-                                c -= 'a' - 10;
-                            else if (c >= 'A')
-                                c -= 'A' - 10;
-                            else if (c >= '0')
-                                c -= '0';
-                            else
-                                continue;
-                            value = (value << 4) | c;
-                            if (i & 1)
-                                data->appendBytes(&value, sizeof(value));
-                        }
-                        gIOOptionsEntry->setProperty(sym, data);
-                    }
-                    if (sym)
-                        sym->release();
-                    if (data)
-                        data->release();
-                }
-            }
-            if (!vars->haveFastBoot)
-            {
-                // set boot volume to zero
-                IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
-                if (platform && (kIOReturnSuccess == platform->readXPRAM(kXPRamAudioVolume, 
-                                            &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume))))
-                {
-                    uint8_t newVolume;
-                    newVolume = vars->saveBootAudioVolume & 0xf8;
-                    platform->writeXPRAM(kXPRamAudioVolume, 
-                                            &newVolume, sizeof(newVolume));
-                }
-            }
-#endif /* !__i386__ */
-       }
-       // --
-
-       gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
-       gIOHibernateState = kIOHibernateStateHibernating;
+        IOLockUnlock(gDebugImageLock);
     }
-    while (false);
 
-    return (err);
+
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@@ -1444,14 +1010,40 @@ ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBl
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
+IOReturn
+IOHibernateIOKitSleep(void)
+{
+    IOReturn ret = kIOReturnSuccess;
+    IOLockLock(gFSLock);
+    if (kFSOpening == gFSState)
+    {
+       gFSState = kFSTimedOut;
+       HIBLOG("hibernate file open timed out\n");
+       ret = kIOReturnTimeout;
+    }
+    IOLockUnlock(gFSLock);
+    return (ret);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
 IOReturn
 IOHibernateSystemHasSlept(void)
 {
+    IOReturn          ret = kIOReturnSuccess;
     IOHibernateVars * vars  = &gIOHibernateVars;
-    OSObject        * obj;
+    OSObject        * obj = 0;
     OSData          * data;
 
-    obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
+    IOLockLock(gFSLock);
+    if ((kFSOpened != gFSState) && gIOHibernateMode)
+    {
+       ret = kIOReturnTimeout;
+    }
+    IOLockUnlock(gFSLock);
+    if (kIOReturnSuccess != ret) return (ret);
+
+    if (gIOHibernateMode) obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
     vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
     if (obj && !vars->previewBuffer)
        obj->release();
@@ -1463,11 +1055,13 @@ IOHibernateSystemHasSlept(void)
        vars->previewBuffer = 0;
     }
 
-    if (vars->previewBuffer && (data = OSDynamicCast(OSData, 
+    if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
+        && vars->previewBuffer 
+        && (data = OSDynamicCast(OSData, 
        IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
     {
        UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
-       HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", flags);
+       HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
 
        IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
 
@@ -1485,8 +1079,8 @@ IOHibernateSystemHasSlept(void)
            vars->consoleMapping   = (uint8_t *) consoleInfo.v_baseAddr;
 
            HIBPRINT("video %p %d %d %d\n",
-                       vars->consoleMapping, gIOHibernateGraphicsInfo->depth, 
-                       gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height);
+                       vars->consoleMapping, graphicsInfo->depth, 
+                       graphicsInfo->width, graphicsInfo->height);
            if (vars->consoleMapping)
                        ProgressInit(graphicsInfo, vars->consoleMapping,
                                        &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
@@ -1496,17 +1090,63 @@ IOHibernateSystemHasSlept(void)
     if (gIOOptionsEntry)
         gIOOptionsEntry->sync();
 
-    return (kIOReturnSuccess);
+    return (ret);
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
+static DeviceTreeNode *
+MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry)
+{
+    DeviceTreeNodeProperty * prop;
+    DeviceTreeNode *         child;
+    IORegistryEntry *        childRegEntry;
+    const char *             nameProp;
+    unsigned int             propLen, idx;
+
+    prop = (DeviceTreeNodeProperty *) (entry + 1);
+    for (idx = 0; idx < entry->nProperties; idx++)
+    {
+       if (regEntry && (0 != strcmp("name", prop->name)))
+       {
+           regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
+//         HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
+       }
+       prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
+    }
+
+    child = (DeviceTreeNode *) prop;
+    for (idx = 0; idx < entry->nChildren; idx++)
+    {
+       if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen))
+           panic("no name");
+       childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
+//     HIBPRINT("%s == %p\n", nameProp, childRegEntry);
+       child = MergeDeviceTree(child, childRegEntry);
+    }
+    return (child);
+}
+
 IOReturn
 IOHibernateSystemWake(void)
 {
-    IOHibernateVars * vars  = &gIOHibernateVars;
+    if (kFSOpened == gFSState)
+    {
+       IOPolledFilePollersClose(gIOHibernateVars.fileVars, kIOPolledPostflightState);
+       IOHibernateDone(&gIOHibernateVars);
+    }
+    else
+    {
+        IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
+        IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
+    }
+    return (kIOReturnSuccess);
+}
 
-    hibernate_teardown(vars->page_list, vars->page_list_wired);
+static IOReturn
+IOHibernateDone(IOHibernateVars * vars)
+{
+    hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
 
     if (vars->videoMapping)
     {
@@ -1515,7 +1155,7 @@ IOHibernateSystemWake(void)
             IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
         if (vars->videoAllocSize)
             // dealloc range
-            kmem_free(kernel_map, trunc_page_32(vars->videoMapping), vars->videoAllocSize);
+            kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
     }
 
     if (vars->previewBuffer)
@@ -1524,95 +1164,106 @@ IOHibernateSystemWake(void)
         vars->previewBuffer = 0;
     }
 
-    if (vars->fileVars)
+    if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
     {
-       IOPolledFileClose(vars->fileVars);
+        IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey, 
+                                            gIOHibernateCurrentHeader->options, 32);
     }
-
-    // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
-
-#ifdef __ppc__
-    OSData * data = OSData::withCapacity(4);
-    if (gIOOptionsEntry && data)
+    else
     {
-        const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
-        if (sym)
-        {
-            gIOOptionsEntry->setProperty(sym, data);
-            sym->release();
-        }
-        sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
-        if (sym)
-        {
-           if (vars->saveBootDevice)
-           {
-               gIOOptionsEntry->setProperty(sym, vars->saveBootDevice);
-               vars->saveBootDevice->release();
-           }
-            sym->release();
-        }
-        sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
-        if (sym)
-        {
-            gIOOptionsEntry->setProperty(sym, data);
-            sym->release();
-        }
-        sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
-        if (sym)
-        {
-            gIOOptionsEntry->removeProperty(sym);
-            sym->release();
-        }
+        IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
     }
-    if (data)
-        data->release();
 
-    if (gIOOptionsEntry)
+    if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
+      && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus))
     {
-       if (!vars->haveFastBoot)
-       {
-           // reset boot audio volume
-           IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
-           if (platform)
-               platform->writeXPRAM(kXPRamAudioVolume, 
-                                       &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume));
-       }
-
-       // sync now to hardware if the booter has not
-       if (kIOHibernateStateInactive == gIOHibernateState)
-           gIOOptionsEntry->sync();
-       else
-           // just sync the variables in case a later panic syncs nvram (it won't sync variables)
-           gIOOptionsEntry->syncOFVariables();
+        IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey, 
+                                        &gIOHibernateGraphicsInfo->gfxStatus,
+                                        sizeof(gIOHibernateGraphicsInfo->gfxStatus));
     }
-#endif
+    else
+    {
+        IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
+    }
+
+    // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
 
-#ifdef __i386__
-       IOService::getPMRootDomain()->removeProperty(kIOHibernateRTCVariablesKey);
+#if defined(__i386__) || defined(__x86_64__)
+       IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
+       IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
 
        /*
         * Hibernate variable is written to NVRAM on platforms in which RtcRam
         * is not backed by coin cell.  Remove Hibernate data from NVRAM.
         */
        if (gIOOptionsEntry) {
-               const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
 
-               if (sym) {
-                       if (gIOOptionsEntry->getProperty(sym)) {
-                               gIOOptionsEntry->removeProperty(sym);
-                               gIOOptionsEntry->sync();
-                       }
-                       sym->release();
+           if (gIOHibernateRTCVariablesKey) {
+               if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
+                   gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
+               }
+           }
+
+           if (gIOHibernateBootNextKey)
+           {
+               if (gIOHibernateBootNextSave)
+               {
+                   gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
+                   gIOHibernateBootNextSave->release();
+                   gIOHibernateBootNextSave = NULL;
                }
+               else
+                   gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
+           }
+           if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) gIOOptionsEntry->sync();
        }
 #endif
 
-    if (vars->srcBuffer)
-       vars->srcBuffer->release();
-    if (vars->ioBuffer)
-       vars->ioBuffer->release();
-    if (vars->fileExtents)
-       vars->fileExtents->release();
+    if (vars->srcBuffer) vars->srcBuffer->release();
+    bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
+    if (vars->handoffBuffer)
+    {
+       if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
+       {
+           IOHibernateHandoff * handoff;
+           bool done = false;
+           for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
+                !done;
+                handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
+           {
+               HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
+               uint8_t * data = &handoff->data[0];
+               switch (handoff->type)
+               {
+                   case kIOHibernateHandoffTypeEnd:
+                       done = true;
+                       break;
+
+                   case kIOHibernateHandoffTypeDeviceTree:
+                       MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
+                       break;
+       
+                   case kIOHibernateHandoffTypeKeyStore:
+#if defined(__i386__) || defined(__x86_64__)
+                       {
+                           IOBufferMemoryDescriptor *
+                           md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
+                           if (md)
+                           {
+                               IOSetKeyStoreData(md);
+                           }
+                       }
+#endif
+                       break;
+       
+                   default:
+                       done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
+                       break;
+               }    
+           }
+       }
+       vars->handoffBuffer->release();
+    }
 
     bzero(vars, sizeof(*vars));
 
@@ -1621,39 +1272,107 @@ IOHibernateSystemWake(void)
     return (kIOReturnSuccess);
 }
 
+void
+IOHibernateSystemPostWakeTrim(void * p1, void * p2)
+{
+    // invalidate & close the image file
+    if (p1) IOLockLock(gFSLock);
+    if (kFSTrimDelay == gFSState)
+    {
+       IOPolledFileIOVars * vars = &gFileVars;
+       IOPolledFileClose(&vars,
+#if DISABLE_TRIM
+                         0, NULL, 0, 0, 0);
+#else
+                         0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader),
+                         sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize);
+#endif
+        gFSState = kFSIdle;
+    }
+    if (p1) IOLockUnlock(gFSLock);
+}
+
 IOReturn
 IOHibernateSystemPostWake(void)
 {
-    if (gIOHibernateFileRef)
+    gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
+    IOLockLock(gFSLock);
+    if (kFSTrimDelay == gFSState) IOHibernateSystemPostWakeTrim(NULL, NULL);
+    else if (kFSOpened != gFSState) gFSState = kFSIdle;
+    else
     {
-       // invalidate the image file
-       gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
-       int err = kern_write_file(gIOHibernateFileRef, 0,
-                                   (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
-       if (KERN_SUCCESS != err)
-           HIBLOG("kern_write_file(%d)\n", err);
+       AbsoluteTime deadline;
 
-       kern_close_file_for_direct_io(gIOHibernateFileRef);
-        gIOHibernateFileRef = 0;
+        gFSState = kFSTrimDelay;
+       clock_interval_to_deadline(TRIM_DELAY, kMillisecondScale, &deadline );
+       thread_call_enter1_delayed(gIOHibernateTrimCalloutEntry, NULL, deadline);
     }
+    IOLockUnlock(gFSLock);
+
+    // IOCloseDebugDataFile() calls IOSetBootImageNVRAM() unconditionally
+    IOCloseDebugDataFile( );
     return (kIOReturnSuccess);
 }
 
+uint32_t IOHibernateWasScreenLocked(void)
+{
+    uint32_t ret = 0;
+    if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry)
+    {
+       OSData *
+       data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
+       if (data)
+       {
+           ret = ((uint32_t *)data->getBytesNoCopy())[0];
+           gIOChosenEntry->setProperty(kIOBooterScreenLockStateKey, data);
+        }
+    }
+    else gIOChosenEntry->removeProperty(kIOBooterScreenLockStateKey);
+
+    return (ret);
+}
+
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 SYSCTL_STRING(_kern, OID_AUTO, hibernatefile, 
-               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN
+               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
                gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
 SYSCTL_STRING(_kern, OID_AUTO, bootsignature, 
-               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN
+               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
                gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
 SYSCTL_UINT(_kern, OID_AUTO, hibernatemode, 
-               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN
+               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
                &gIOHibernateMode, 0, "");
+SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
+               CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
+               &_hibernateStats, hibernate_statistics_t, "");
+
+SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
+               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+               &_hibernateStats.graphicsReadyTime, 0, "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
+               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+               &_hibernateStats.wakeNotificationTime, 0, "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
+               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+               &_hibernateStats.lockScreenReadyTime, 0, "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
+               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+               &_hibernateStats.hidReadyTime, 0, "");
+
 
 void
 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
 {
+    gIOHibernateBootImageKey = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
+
+#if defined(__i386__) || defined(__x86_64__)
+    gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
+    gIOHibernateBoot0082Key     = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
+    gIOHibernateBootNextKey     = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
+    gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
+#endif /* defined(__i386__) || defined(__x86_64__) */
+
     OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
     if (data)
     {
@@ -1669,24 +1388,34 @@ IOHibernateSystemInit(IOPMrootDomain * rootDomain)
     sysctl_register_oid(&sysctl__kern_hibernatefile);
     sysctl_register_oid(&sysctl__kern_bootsignature);
     sysctl_register_oid(&sysctl__kern_hibernatemode);
-}
+    sysctl_register_oid(&sysctl__kern_hibernatestatistics);
+    sysctl_register_oid(&sysctl__kern_hibernategraphicsready);
+    sysctl_register_oid(&sysctl__kern_hibernatewakenotification);
+    sysctl_register_oid(&sysctl__kern_hibernatelockscreenready);
+    sysctl_register_oid(&sysctl__kern_hibernatehidready);
+
+    gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
 
+    gFSLock = IOLockAlloc();
+}
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
-static void
-hibernate_setup_for_wake(void)
+static IOReturn 
+IOHibernatePolledFileWrite(IOPolledFileIOVars * vars,
+                          const uint8_t * bytes, IOByteCount size,
+                          IOPolledFileCryptVars * cryptvars)
 {
-#if __ppc__
-    // go slow (state needed for wake)
-    ml_set_processor_speed(1);
-#endif
+    IOReturn err;
+
+    err = IOPolledFileWrite(vars, bytes, size, cryptvars);
+    if ((kIOReturnSuccess == err) && hibernate_should_abort()) err = kIOReturnAborted;
+
+    return (err);
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
-#define C_ASSERT(e) typedef char    __C_ASSERT__[(e) ? 1 : -1]
-
 extern "C" uint32_t
 hibernate_write_image(void)
 {
@@ -1694,36 +1423,63 @@ hibernate_write_image(void)
     IOHibernateVars *        vars  = &gIOHibernateVars;
     IOPolledFileExtent *     fileExtents;
 
-    C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
+    _static_assert_1_arg(sizeof(IOHibernateImageHeader) == 512);
 
     uint32_t    pageCount, pagesDone;
     IOReturn     err;
-    vm_offset_t  ppnum;
-    IOItemCount  page, count;
+    vm_offset_t  ppnum, page;
+    IOItemCount  count;
     uint8_t *   src;
     uint8_t *   data;
+    uint8_t *   compressed;
+    uint8_t *   scratch;
     IOByteCount  pageCompressedSize;
     uint64_t    compressedSize, uncompressedSize;
     uint64_t    image1Size = 0;
     uint32_t    bitmap_size;
-    bool        iterDone, pollerOpen, needEncryptStart;
+    bool        iterDone, pollerOpen, needEncrypt;
     uint32_t    restore1Sum, sum, sum1, sum2;
+    int          wkresult;
     uint32_t    tag;
     uint32_t    pageType;
     uint32_t    pageAndCount[2];
+    addr64_t     phys64;
+    IOByteCount  segLen;
+    uintptr_t    hibernateBase;
+    uintptr_t    hibernateEnd;
 
     AbsoluteTime startTime, endTime;
-    AbsoluteTime allTime, compTime, decoTime;
+    AbsoluteTime allTime, compTime;
+    uint64_t     compBytes;
     uint64_t     nsec;
     uint32_t     lastProgressStamp = 0;
     uint32_t     progressStamp;
     uint32_t    blob, lastBlob = (uint32_t) -1L;
 
-    hibernate_cryptvars_t _cryptvars;
-    hibernate_cryptvars_t * cryptvars = 0;
+    uint32_t    wiredPagesEncrypted;
+    uint32_t    dirtyPagesEncrypted;
+    uint32_t    wiredPagesClear;
+    uint32_t    svPageCount;
+    uint32_t    zvPageCount;
+
+    IOPolledFileCryptVars _cryptvars;
+    IOPolledFileCryptVars * cryptvars = 0;
+
+    wiredPagesEncrypted = 0;
+    dirtyPagesEncrypted = 0;
+    wiredPagesClear     = 0;
+    svPageCount         = 0;
+    zvPageCount         = 0;
+
+    if (!vars->fileVars
+    || !vars->fileVars->pollers
+    || !(kIOHibernateModeOn & gIOHibernateMode))      return (kIOHibernatePostWriteSleep);
 
-    if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
-        return (false /* sleep */ );
+    if (kIOHibernateModeSleep & gIOHibernateMode)
+       kdebug_enable = save_kdebug_enable;
+
+    KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START, 0, 0, 0, 0, 0);
+    IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
 
     restore1Sum = sum1 = sum2 = 0;
 
@@ -1736,7 +1492,7 @@ hibernate_write_image(void)
              0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
     
         cryptvars = &gIOHibernateCryptWakeContext;
-        bzero(cryptvars, sizeof(hibernate_cryptvars_t));
+        bzero(cryptvars, sizeof(IOPolledFileCryptVars));
         aes_encrypt_key(vars->cryptKey,
                         kIOHibernateAESKeySize,
                         &cryptvars->ctx.encrypt);
@@ -1745,7 +1501,10 @@ hibernate_write_image(void)
                         &cryptvars->ctx.decrypt);
 
         cryptvars = &_cryptvars;
-        bzero(cryptvars, sizeof(hibernate_cryptvars_t));
+        bzero(cryptvars, sizeof(IOPolledFileCryptVars));
+        for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++)
+            vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
+        bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
         aes_encrypt_key(vars->wiredCryptKey,
                         kIOHibernateAESKeySize,
                         &cryptvars->ctx.encrypt);
@@ -1753,19 +1512,21 @@ hibernate_write_image(void)
         bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
         bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
         bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
-        bzero(gIOHibernateCryptWakeVars, sizeof(hibernate_cryptwakevars_t));
     }
 #endif /* CRYPTO */
 
-    hibernate_setup_for_wake();
-
     hibernate_page_list_setall(vars->page_list,
                                vars->page_list_wired,
+                               vars->page_list_pal,
+                              false /* !preflight */,
+                              /* discard_all */
+                              ((0 == (kIOHibernateModeSleep & gIOHibernateMode)) 
+                              && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
                                &pageCount);
 
     HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
 
-    fileExtents = (IOPolledFileExtent *) vars->fileExtents->getBytesNoCopy();
+    fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy();
 
 #if 0
     count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
@@ -1777,25 +1538,27 @@ hibernate_write_image(void)
     }
 #endif
 
-    needEncryptStart = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
-
+    needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
     AbsoluteTime_to_scalar(&compTime) = 0;
-    AbsoluteTime_to_scalar(&decoTime) = 0;
+    compBytes = 0;
 
     clock_get_uptime(&allTime);
-
+    IOService::getPMRootDomain()->pmStatsRecordEvent( 
+                        kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
     do 
     {
         compressedSize   = 0;
         uncompressedSize = 0;
-        iterDone         = false;
-        pageType         = 0;          // wired pages first
+        svPageCount      = 0;
+        zvPageCount      = 0;
 
-        IOPolledFileSeek(vars->fileVars, sizeof(IOHibernateImageHeader));
+        IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
     
         HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n", 
                 ml_get_interrupts_enabled());
-        err = IOHibernatePollerOpen(vars->fileVars, kIOPolledBeforeSleepState, vars->ioBuffer);
+        err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState,
+                // abortable if not low battery
+                !IOService::getPMRootDomain()->mustHibernate());
         HIBLOG("IOHibernatePollerOpen(%x)\n", err);
         pollerOpen = (kIOReturnSuccess == err);
         if (!pollerOpen)
@@ -1803,66 +1566,84 @@ hibernate_write_image(void)
     
         // copy file block extent list if larger than header
     
-        count = vars->fileExtents->getLength();
+        count = vars->fileVars->fileExtents->getLength();
         if (count > sizeof(header->fileExtentMap))
         {
             count -= sizeof(header->fileExtentMap);
-            err = IOPolledFileWrite(vars->fileVars,
+            err = IOHibernatePolledFileWrite(vars->fileVars,
                                     ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
             if (kIOReturnSuccess != err)
                 break;
         }
 
+        hibernateBase = HIB_BASE; /* Defined in PAL headers */
+        hibernateEnd = (segHIBB + segSizeHIB);
+
         // copy out restore1 code
-    
-        page = atop_32(sectHIBB);
-        count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
-        header->restore1CodePage = page;
+
+        for (count = 0;
+            (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+            count += segLen)
+        {
+           for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++)
+           {
+               gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
+           }
+        }
+
+        page = atop_32(kvtophys(hibernateBase));
+        count = atop_32(round_page(hibernateEnd) - hibernateBase);
+        header->restore1CodePhysPage = page;
+        header->restore1CodeVirt = hibernateBase;
         header->restore1PageCount = count;
-        header->restore1CodeOffset = ((uint32_t) &hibernate_machine_entrypoint)      - sectHIBB;
-        header->restore1StackOffset = ((uint32_t) &gIOHibernateRestoreStackEnd[0]) - 64 - sectHIBB;
+        header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint)      - hibernateBase;
+        header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
 
-        // sum __HIB sect, with zeros for the stack
-        src = (uint8_t *) trunc_page(sectHIBB);
+        // sum __HIB seg, with zeros for the stack
+        src = (uint8_t *) trunc_page(hibernateBase);
         for (page = 0; page < count; page++)
         {
             if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
-                restore1Sum += hibernate_sum(src, page_size);
+                restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
             else
-                restore1Sum += 0x10000001;
+                restore1Sum += 0x00000000;
             src += page_size;
         }
         sum1 = restore1Sum;
     
-        // write the __HIB sect, with zeros for the stack
+        // write the __HIB seg, with zeros for the stack
 
-        src = (uint8_t *) trunc_page(sectHIBB);
-        count = ((uint32_t) &gIOHibernateRestoreStack[0]) - trunc_page(sectHIBB);
+        src = (uint8_t *) trunc_page(hibernateBase);
+        count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
         if (count)
         {
-            err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
+            err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
             if (kIOReturnSuccess != err)
                 break;
         }
-        err = IOPolledFileWrite(vars->fileVars, 
+        err = IOHibernatePolledFileWrite(vars->fileVars, 
                                         (uint8_t *) 0,
                                         &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
                                         cryptvars);
         if (kIOReturnSuccess != err)
             break;
         src = &gIOHibernateRestoreStackEnd[0];
-        count = round_page(sectHIBB + sectSizeHIB) - ((uint32_t) src);
+        count = round_page(hibernateEnd) - ((uintptr_t) src);
         if (count)
         {
-            err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
+            err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
             if (kIOReturnSuccess != err)
                 break;
         }
 
-        // write the preview buffer
+       if (kIOHibernateModeEncrypt & gIOHibernateMode)
+       {
+           vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
+           vars->fileVars->encryptEnd   = UINT64_MAX;
+           HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
+       }
 
-        addr64_t phys64;
-        IOByteCount segLen;
+        // write the preview buffer
 
         if (vars->previewBuffer)
         {
@@ -1870,10 +1651,10 @@ hibernate_write_image(void)
             count = 0;
             do
             {
-                phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen);
+                phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
                 pageAndCount[0] = atop_64(phys64);
                 pageAndCount[1] = atop_32(segLen);
-                err = IOPolledFileWrite(vars->fileVars, 
+                err = IOHibernatePolledFileWrite(vars->fileVars, 
                                         (const uint8_t *) &pageAndCount, sizeof(pageAndCount), 
                                         cryptvars);
                 if (kIOReturnSuccess != err)
@@ -1885,24 +1666,30 @@ hibernate_write_image(void)
             if (kIOReturnSuccess != err)
                 break;
 
-            src = (uint8_t *) vars->previewBuffer->getSourceSegment(0, NULL);
+            src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
+
+                       ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
+
             count = vars->previewBuffer->getLength();
 
             header->previewPageListSize = ppnum;
             header->previewSize = count + ppnum;
 
             for (page = 0; page < count; page += page_size)
-                sum1 += hibernate_sum(src + page, page_size);
-
-            err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
+            {
+                phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
+                sum1 += hibernate_sum_page(src + page, atop_64(phys64));
+            }
+            err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
             if (kIOReturnSuccess != err)
                 break;
         }
 
         // mark areas for no save
-    
+        IOMemoryDescriptor * ioBuffer;
+        ioBuffer = IOPolledFileGetIOBuffer(vars->fileVars);
         for (count = 0;
-            (phys64 = vars->ioBuffer->getPhysicalSegment64(count, &segLen));
+            (phys64 = ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
             count += segLen)
         {
             hibernate_set_page_state(vars->page_list, vars->page_list_wired, 
@@ -1912,7 +1699,7 @@ hibernate_write_image(void)
         }
     
         for (count = 0;
-            (phys64 = vars->srcBuffer->getPhysicalSegment64(count, &segLen));
+            (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
             count += segLen)
         {
             hibernate_set_page_state(vars->page_list, vars->page_list_wired, 
@@ -1925,7 +1712,7 @@ hibernate_write_image(void)
     
         bitmap_size = vars->page_list_wired->list_size;
         src = (uint8_t *) vars->page_list_wired;
-        err = IOPolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
+        err = IOHibernatePolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
         if (kIOReturnSuccess != err)
             break;
 
@@ -1934,15 +1721,16 @@ hibernate_write_image(void)
 
        hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
     
-        page = atop_32(sectHIBB);
-        count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
+
+        page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
+        count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
         hibernate_set_page_state(vars->page_list, vars->page_list_wired,
                                         page, count,
                                         kIOHibernatePageStateFree);
         pageCount -= count;
 
         if (vars->previewBuffer) for (count = 0;
-                                        (phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen));
+                                        (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
                                         count += segLen)
         {
             hibernate_set_page_state(vars->page_list, vars->page_list_wired, 
@@ -1951,142 +1739,226 @@ hibernate_write_image(void)
             pageCount -= atop_32(segLen);
         }
 
-        src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
-    
-        ppnum     = 0;
-        pagesDone = 0;
-        lastBlob = 0;
-    
-        HIBLOG("writing %d pages\n", pageCount);
-
-        do
+        for (count = 0;
+            (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+            count += segLen)
         {
-            count = hibernate_page_list_iterate(pageType ? vars->page_list : vars->page_list_wired,
-                                                    &ppnum);
-//          kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
-    
-            iterDone = !count;
+            hibernate_set_page_state(vars->page_list, vars->page_list_wired, 
+                                        atop_64(phys64), atop_32(segLen),
+                                        kIOHibernatePageStateFree);
+            pageCount -= atop_32(segLen);
+        }
 
-            pageAndCount[0] = ppnum;
-            pageAndCount[1] = count;
-            err = IOPolledFileWrite(vars->fileVars, 
-                                    (const uint8_t *) &pageAndCount, sizeof(pageAndCount), 
-                                    cryptvars);
-            if (kIOReturnSuccess != err)
-                break;
+        src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
+       compressed = src + page_size;
+        scratch    = compressed + page_size;
 
-            for (page = 0; page < count; page++)
+        pagesDone  = 0;
+        lastBlob   = 0;
+    
+        HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n", 
+               bitmap_size, header->previewSize,
+               pageCount, vars->fileVars->position);
+
+        enum
+        // pageType
+        { 
+            kWired          = 0x02,
+            kEncrypt        = 0x01,
+            kWiredEncrypt   = kWired | kEncrypt,
+            kWiredClear     = kWired,
+            kUnwiredEncrypt = kEncrypt
+        };
+
+        bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features()));
+#define _pmap_is_noencrypt(x) (cpuAES ? false : pmap_is_noencrypt((x)))
+
+        for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)
+        {
+           if (kUnwiredEncrypt == pageType)
+          {
+               // start unwired image
+               if (kIOHibernateModeEncrypt & gIOHibernateMode)
+               {
+                   vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
+                   vars->fileVars->encryptEnd   = UINT64_MAX;
+                   HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
+               }
+               bcopy(&cryptvars->aes_iv[0], 
+                       &gIOHibernateCryptWakeContext.aes_iv[0], 
+                       sizeof(cryptvars->aes_iv));
+               cryptvars = &gIOHibernateCryptWakeContext;
+            }
+            for (iterDone = false, ppnum = 0; !iterDone; )
             {
-                err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(ppnum), page_size);
-                if (err)
+                count = hibernate_page_list_iterate((kWired & pageType) 
+                                                            ? vars->page_list_wired : vars->page_list,
+                                                        &ppnum);
+//              kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
+                iterDone = !count;
+
+                if (count && (kWired & pageType) && needEncrypt)
                 {
-                    HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%d] %x\n", __LINE__, ppnum, err);
-                    break;
+                    uint32_t checkIndex;
+                    for (checkIndex = 0;
+                            (checkIndex < count) 
+                                && (((kEncrypt & pageType) == 0) == _pmap_is_noencrypt(ppnum + checkIndex));
+                            checkIndex++)
+                    {}
+                    if (!checkIndex)
+                    {
+                        ppnum++;
+                        continue;
+                    }
+                    count = checkIndex;
                 }
-    
-                sum = hibernate_sum(src, page_size);
-   
-                clock_get_uptime(&startTime);
-
-                pageCompressedSize = WKdm_compress ((WK_word*) src, (WK_word*) (src + page_size), PAGE_SIZE_IN_WORDS);
-    
-                clock_get_uptime(&endTime);
-                ADD_ABSOLUTETIME(&compTime, &endTime);
-                SUB_ABSOLUTETIME(&compTime, &startTime);
-    
-                if (kIOHibernateModeEncrypt & gIOHibernateMode)
-                    pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
 
-                if (pageCompressedSize > page_size)
+                switch (pageType)
                 {
-//                  HIBLOG("------------lose: %d\n", pageCompressedSize);
-                    pageCompressedSize = page_size;
+                    case kWiredEncrypt:   wiredPagesEncrypted += count; break;
+                    case kWiredClear:     wiredPagesClear     += count; break;
+                    case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
                 }
-
-                if (pageCompressedSize != page_size)
-                    data = (src + page_size);
-                else
-                    data = src;
-
-                tag = pageCompressedSize | kIOHibernateTagSignature;
-
-                if (pageType)
-                    sum2 += sum;
+    
+                if (iterDone && (kWiredEncrypt == pageType))   {/* not yet end of wired list */}
                 else
-                    sum1 += sum;
-
-                if (needEncryptStart && (ppnum >= atop_32(sectDATAB)))
                 {
-                    // start encrypting partway into the data about to be written
-                    vars->fileVars->encryptStart = (vars->fileVars->position + AES_BLOCK_SIZE - 1) 
-                                                    & ~(AES_BLOCK_SIZE - 1);
-                    needEncryptStart = false;
+                    pageAndCount[0] = ppnum;
+                    pageAndCount[1] = count;
+                    err = IOHibernatePolledFileWrite(vars->fileVars, 
+                                            (const uint8_t *) &pageAndCount, sizeof(pageAndCount), 
+                                            cryptvars);
+                    if (kIOReturnSuccess != err)
+                        break;
                 }
+    
+                for (page = ppnum; page < (ppnum + count); page++)
+                {
+                    err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
+                    if (err)
+                    {
+                        HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
+                        break;
+                    }
+        
+                    sum = hibernate_sum_page(src, page);
+                    if (kWired & pageType)
+                        sum1 += sum;
+                    else
+                        sum2 += sum;
+       
+                    clock_get_uptime(&startTime);
+                    wkresult = WKdm_compress_new((const WK_word*) src,
+                                                (WK_word*) compressed, 
+                                                (WK_word*) scratch,
+                                                page_size - 4);
 
-                err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
-                if (kIOReturnSuccess != err)
-                    break;
+                    clock_get_uptime(&endTime);
+                    ADD_ABSOLUTETIME(&compTime, &endTime);
+                    SUB_ABSOLUTETIME(&compTime, &startTime);
 
-                err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
-                if (kIOReturnSuccess != err)
-                    break;
+                    compBytes += page_size;
+                    pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
 
-                compressedSize += pageCompressedSize;
-                if (pageCompressedSize)
-                    uncompressedSize += page_size;
-                ppnum++;
-                pagesDone++;
+                   if (pageCompressedSize == 0) 
+                   {
+                       pageCompressedSize = 4;
+                        data = src;
 
-               if (vars->consoleMapping && (0 == (1023 & pagesDone)))
-               {
-                   blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
-                   if (blob != lastBlob)
+                       if (*(uint32_t *)src)
+                               svPageCount++;
+                       else
+                               zvPageCount++;
+                   }
+                   else 
                    {
-                       ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
-                       lastBlob = blob;
+                       if (pageCompressedSize != page_size)
+                           data = compressed;
+                       else
+                           data = src;
                    }
-               }
-                if (0 == (8191 & pagesDone))
-                {
-                    clock_get_uptime(&endTime);
-                    SUB_ABSOLUTETIME(&endTime, &allTime);
-                    absolutetime_to_nanoseconds(endTime, &nsec);
-                    progressStamp = nsec / 750000000ULL;
-                    if (progressStamp != lastProgressStamp)
+    
+                    tag = pageCompressedSize | kIOHibernateTagSignature;
+                    err = IOHibernatePolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
+                    if (kIOReturnSuccess != err)
+                        break;
+    
+                    err = IOHibernatePolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
+                    if (kIOReturnSuccess != err)
+                        break;
+    
+                    compressedSize += pageCompressedSize;
+                    uncompressedSize += page_size;
+                    pagesDone++;
+    
+                    if (vars->consoleMapping && (0 == (1023 & pagesDone)))
+                    {
+                        blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
+                        if (blob != lastBlob)
+                        {
+                            ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
+                            lastBlob = blob;
+                        }
+                    }
+                    if (0 == (8191 & pagesDone))
                     {
-                        lastProgressStamp = progressStamp;
-                        HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
+                        clock_get_uptime(&endTime);
+                        SUB_ABSOLUTETIME(&endTime, &allTime);
+                        absolutetime_to_nanoseconds(endTime, &nsec);
+                        progressStamp = nsec / 750000000ULL;
+                        if (progressStamp != lastProgressStamp)
+                        {
+                            lastProgressStamp = progressStamp;
+                            HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
+                        }
                     }
                 }
+                if (kIOReturnSuccess != err)
+                    break;
+                ppnum = page;
             }
+
             if (kIOReturnSuccess != err)
                 break;
-            if (iterDone && !pageType)
+
+            if ((kEncrypt & pageType) && vars->fileVars->encryptStart)
+            {
+                vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
+                HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
+            }
+
+            if (kWiredEncrypt != pageType)
             {
-                err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
+                // end of image1/2 - fill to next block
+                err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
                 if (kIOReturnSuccess != err)
                     break;
+            }
+            if (kWiredClear == pageType)
+            {
+               // enlarge wired image for test
+//              err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
 
-                iterDone = false;
-                pageType = 1;
-                ppnum = 0;
+                // end wired image
+                header->encryptStart = vars->fileVars->encryptStart;
+                header->encryptEnd   = vars->fileVars->encryptEnd;
                 image1Size = vars->fileVars->position;
-                if (cryptvars)
-                {
-                    bcopy(&cryptvars->aes_iv[0], 
-                            &gIOHibernateCryptWakeContext.aes_iv[0], 
-                            sizeof(cryptvars->aes_iv));
-                    cryptvars = &gIOHibernateCryptWakeContext;
-                }
-                HIBLOG("image1Size %qd\n", image1Size);
+                HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
+                        image1Size, header->encryptStart, header->encryptEnd);
             }
         }
-        while (!iterDone);
-        if (kIOReturnSuccess != err)
-            break;
-        err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
         if (kIOReturnSuccess != err)
+        {
+            if (kIOReturnOverrun == err)
+            {
+                // update actual compression ratio on not enough space (for retry)
+                gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
+            }
+
+            // update partial amount written (for IOPolledFileClose cleanup/unmap)
+            header->imageSize = vars->fileVars->position;
             break;
+        }
 
         // Header:
     
@@ -2094,13 +1966,16 @@ hibernate_write_image(void)
         header->image1Size   = image1Size;
         header->bitmapSize   = bitmap_size;
         header->pageCount    = pageCount;
-        header->encryptStart = vars->fileVars->encryptStart;
     
         header->restore1Sum  = restore1Sum;
         header->image1Sum    = sum1;
         header->image2Sum    = sum2;
+        header->sleepTime    = gIOLastSleepTime.tv_sec;
+
+       header->compression     = (compressedSize << 8) / uncompressedSize;
+       gIOHibernateCompression = header->compression;
     
-        count = vars->fileExtents->getLength();
+        count = vars->fileVars->fileExtents->getLength();
         if (count > sizeof(header->fileExtentMap))
         {
             header->fileExtentMapSize = count;
@@ -2109,44 +1984,52 @@ hibernate_write_image(void)
         else
             header->fileExtentMapSize = sizeof(header->fileExtentMap);
         bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
+
+        header->deviceBase      = vars->fileVars->block0;
+        header->deviceBlockSize = vars->fileVars->blockSize;
     
         IOPolledFileSeek(vars->fileVars, 0);
-        err = IOPolledFileWrite(vars->fileVars,
+        err = IOHibernatePolledFileWrite(vars->fileVars,
                                     (uint8_t *) header, sizeof(IOHibernateImageHeader), 
                                     cryptvars);
         if (kIOReturnSuccess != err)
             break;
-        err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
-        if (kIOReturnSuccess != err)
-            break;
-        err = IOHibernatePollerIODone(vars->fileVars, true);
-        if (kIOReturnSuccess != err)
-            break;
+        err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
     }
     while (false);
     
     clock_get_uptime(&endTime);
+
+    IOService::getPMRootDomain()->pmStatsRecordEvent( 
+                        kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
+
     SUB_ABSOLUTETIME(&endTime, &allTime);
     absolutetime_to_nanoseconds(endTime, &nsec);
-    HIBLOG("all time: %qd ms, ", 
-               nsec / 1000000ULL);
+    HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
 
     absolutetime_to_nanoseconds(compTime, &nsec);
-    HIBLOG("comp time: %qd ms, ", 
-               nsec / 1000000ULL);
-
-    absolutetime_to_nanoseconds(decoTime, &nsec);
-    HIBLOG("deco time: %qd ms, ", 
-               nsec / 1000000ULL);
-
-    HIBLOG("\nimage %qd, uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n", 
-               header->imageSize,
+    HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ", 
+               compBytes, 
+               nsec / 1000000ULL,
+               nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+    absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
+    HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ", 
+               vars->fileVars->cryptBytes, 
+               nsec / 1000000ULL, 
+               nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+    HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n", 
+               header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
                uncompressedSize, atop_32(uncompressedSize), compressedSize,
                uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
                sum1, sum2);
 
+    HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n", 
+          svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
+
     if (pollerOpen)
-        IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
+        IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted );
 
     if (vars->consoleMapping)
         ProgressUpdate(gIOHibernateGraphicsInfo, 
@@ -2157,6 +2040,9 @@ hibernate_write_image(void)
     // should we come back via regular wake, set the state in memory.
     gIOHibernateState = kIOHibernateStateInactive;
 
+    KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END,
+                         wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted, 0, 0);
+
     if (kIOReturnSuccess == err)
     {
        if (kIOHibernateModeSleep & gIOHibernateMode)
@@ -2192,26 +2078,25 @@ hibernate_machine_init(void)
     IOReturn     err;
     uint32_t     sum;
     uint32_t     pagesDone;
+    uint32_t     pagesRead = 0;
+    AbsoluteTime startTime, compTime;
     AbsoluteTime allTime, endTime;
-    uint64_t     nsec;
+    AbsoluteTime startIOTime, endIOTime;
+    uint64_t     nsec, nsecIO;
+    uint64_t     compBytes;
     uint32_t     lastProgressStamp = 0;
     uint32_t     progressStamp;
-    uint64_t    progressZeroPosition = 0;
-    uint32_t    blob, lastBlob = (uint32_t) -1L;
-    hibernate_cryptvars_t * cryptvars = 0;
+    IOPolledFileCryptVars * cryptvars = 0;
 
     IOHibernateVars * vars  = &gIOHibernateVars;
+    bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
 
-    if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
+    if (!vars->fileVars || !vars->fileVars->pollers)
        return;
 
     sum = gIOHibernateCurrentHeader->actualImage1Sum;
     pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
 
-    HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, image1Size %qx, conflictCount %d, nextFree %x\n",
-           gIOHibernateState, pagesDone, sum, gIOHibernateCurrentHeader->image1Size,
-           gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
-
     if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
     {
        HIBLOG("regular wake\n");
@@ -2220,81 +2105,165 @@ hibernate_machine_init(void)
 
     HIBPRINT("diag %x %x %x %x\n",
            gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1], 
-           gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]); 
-
-    HIBPRINT("video %x %d %d %d\n",
-           gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth, 
-           gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height); 
+           gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
+
+#define t40ms(x)       (tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)
+#define tStat(x, y)    gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
+    tStat(booterStart, booterStart);
+    gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart;
+    tStat(booterDuration0, booterTime0);
+    tStat(booterDuration1, booterTime1);
+    tStat(booterDuration2, booterTime2);
+    tStat(booterDuration, booterTime);
+    tStat(booterConnectDisplayDuration, connectDisplayTime);
+    tStat(booterSplashDuration, splashTime);
+    tStat(trampolineDuration, trampolineTime);
+
+    gIOHibernateStats->image1Size  = gIOHibernateCurrentHeader->image1Size;
+    gIOHibernateStats->imageSize   = gIOHibernateCurrentHeader->imageSize;
+    gIOHibernateStats->image1Pages = pagesDone;
+
+    HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n", 
+          gIOHibernateStats->booterStart,
+          gIOHibernateStats->smcStart,
+          gIOHibernateStats->booterDuration0,
+          gIOHibernateStats->booterDuration1,
+          gIOHibernateStats->booterDuration2,
+          gIOHibernateStats->booterDuration,
+          gIOHibernateStats->booterConnectDisplayDuration,
+          gIOHibernateStats->booterSplashDuration,
+          gIOHibernateStats->trampolineDuration);
+
+    HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
+           gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
+           gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
 
-    if ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)
+    if ((0 != (kIOHibernateModeSleep & gIOHibernateMode)) 
+     && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)))
+    {
         hibernate_page_list_discard(vars->page_list);
+    }
 
-    boot_args *args = (boot_args *) PE_state.bootArgs;
+    cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
 
-    if (vars->videoMapping 
-       && gIOHibernateGraphicsInfo->physicalAddress
-       && (args->Video.v_baseAddr == gIOHibernateGraphicsInfo->physicalAddress))
-    {
-        vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height 
-                                        * gIOHibernateGraphicsInfo->rowBytes);
-        IOMapPages(kernel_map, 
-                    vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
-                    vars->videoMapSize, kIOMapInhibitCache );
-    }
+    if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount)
+       panic("handoff overflow");
 
-    uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
+    IOHibernateHandoff * handoff;
+    bool                 done           = false;
+    bool                 foundCryptData = false;
 
-    if (gIOHibernateWakeMapSize)
+    for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
+        !done;
+        handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
     {
-       err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(gIOHibernateWakeMap), 
-                                                   gIOHibernateWakeMapSize);
-       if (kIOReturnSuccess == err)
-           hibernate_newruntime_map(src, gIOHibernateWakeMapSize, 
-                                    gIOHibernateCurrentHeader->systemTableOffset);
-       gIOHibernateWakeMap = 0;
-       gIOHibernateWakeMapSize = 0;
-    }
+//     HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
+       uint8_t * data = &handoff->data[0];
+       switch (handoff->type)
+       {
+           case kIOHibernateHandoffTypeEnd:
+               done = true;
+               break;
 
-    uint32_t decoOffset;
+           case kIOHibernateHandoffTypeGraphicsInfo:
+               if (handoff->bytecount == sizeof(*gIOHibernateGraphicsInfo))
+               {
+                   bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
+               }
+               break;
 
-    clock_get_uptime(&allTime);
+           case kIOHibernateHandoffTypeCryptVars:
+               if (cryptvars)
+               {
+                   hibernate_cryptwakevars_t *
+                   wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
+                   bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
+               }
+               foundCryptData = true;
+               bzero(data, handoff->bytecount);
+               break;
 
-    HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
-    err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0);
-    HIBLOG("IOHibernatePollerOpen(%x)\n", err);
+           case kIOHibernateHandoffTypeMemoryMap:
 
-    if (gIOHibernateCurrentHeader->previewSize)
-        progressZeroPosition = gIOHibernateCurrentHeader->previewSize 
-                             + gIOHibernateCurrentHeader->fileExtentMapSize 
-                             - sizeof(gIOHibernateCurrentHeader->fileExtentMap) 
-                             + ptoa_64(gIOHibernateCurrentHeader->restore1PageCount);
+               clock_get_uptime(&allTime);
 
-    IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
+               hibernate_newruntime_map(data, handoff->bytecount, 
+                                        gIOHibernateCurrentHeader->systemTableOffset);
 
-    if (vars->videoMapSize)
-    {
-        lastBlob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
-                        / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
-        ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, 0, lastBlob);
+               clock_get_uptime(&endTime);
+           
+               SUB_ABSOLUTETIME(&endTime, &allTime);
+               absolutetime_to_nanoseconds(endTime, &nsec);
+           
+               HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
+
+               break;
+
+           case kIOHibernateHandoffTypeDeviceTree:
+               {
+//                 DTEntry chosen = NULL;
+//                 HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen));
+               }
+               break;
+
+           default:
+               done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
+               break;
+       }    
     }
+    if (cryptvars && !foundCryptData)
+       panic("hibernate handoff");
 
-    cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
-    if (kIOHibernateModeEncrypt & gIOHibernateMode)
+    HIBPRINT("video 0x%llx %d %d %d status %x\n",
+           gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth, 
+           gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus); 
+
+    if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)
     {
-        cryptvars = &gIOHibernateCryptWakeContext;
-        bcopy(&gIOHibernateCryptWakeVars->aes_iv[0], 
-                &cryptvars->aes_iv[0], 
-                sizeof(cryptvars->aes_iv));
+        vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height 
+                                        * gIOHibernateGraphicsInfo->rowBytes);
+       if (vars->videoMapSize > vars->videoAllocSize) vars->videoMapSize = 0;
+       else
+       {
+           IOMapPages(kernel_map, 
+                       vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
+                       vars->videoMapSize, kIOMapInhibitCache );
+       }
     }
 
+    if (vars->videoMapSize)
+        ProgressUpdate(gIOHibernateGraphicsInfo, 
+                        (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
+
+    uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
+    uint8_t * compressed = src + page_size;
+    uint8_t * scratch    = compressed + page_size;
+    uint32_t  decoOffset;
+
+    clock_get_uptime(&allTime);
+    AbsoluteTime_to_scalar(&compTime) = 0;
+    compBytes = 0;
+
+    HIBLOG("IOPolledFilePollersOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
+    err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledAfterSleepState, false);
+    clock_get_uptime(&startIOTime);
+    endTime = startIOTime;
+    SUB_ABSOLUTETIME(&endTime, &allTime);
+    absolutetime_to_nanoseconds(endTime, &nsec);
+    HIBLOG("IOPolledFilePollersOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
+
+    IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
+
     // kick off the read ahead
-    vars->fileVars->io          = false;
     vars->fileVars->bufferHalf   = 0;
     vars->fileVars->bufferLimit  = 0;
     vars->fileVars->lastRead     = 0;
+    vars->fileVars->readEnd      = gIOHibernateCurrentHeader->imageSize;
     vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
+    vars->fileVars->cryptBytes   = 0;
+    AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
 
-    IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
+    err = IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
     vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
     // --
 
@@ -2303,7 +2272,7 @@ hibernate_machine_init(void)
     uint32_t * header = (uint32_t *) src;
     sum = 0;
 
-    do
+    while (kIOReturnSuccess == err)
     {
        unsigned int count;
        unsigned int page;
@@ -2335,47 +2304,44 @@ hibernate_machine_init(void)
                break;
            }
 
-           if (!compressedSize)
-           {
-               ppnum++;
-               pagesDone++;
-               continue;
-           }
-
            err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
-           if (kIOReturnSuccess != err)
-               break;
+           if (kIOReturnSuccess != err) break;
 
            if (compressedSize < page_size)
            {
                decoOffset = page_size;
-               WKdm_decompress((WK_word*) src, (WK_word*) (src + decoOffset), PAGE_SIZE_IN_WORDS);
-           }
-           else
-               decoOffset = 0;
+               clock_get_uptime(&startTime);
+
+               if (compressedSize == 4) {
+                   int i;
+                   uint32_t *s, *d;
+                       
+                   s = (uint32_t *)src;
+                   d = (uint32_t *)(uintptr_t)compressed;
 
-           sum += hibernate_sum((src + decoOffset), page_size);
+                   for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++)
+                       *d++ = *s;
+               }
+               else 
+                   WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, compressedSize);
+               clock_get_uptime(&endTime);
+               ADD_ABSOLUTETIME(&compTime, &endTime);
+               SUB_ABSOLUTETIME(&compTime, &startTime);
+               compBytes += page_size;
+           }
+           else decoOffset = 0;
 
+           sum += hibernate_sum_page((src + decoOffset), ppnum);
            err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
            if (err)
            {
-               HIBLOG("IOMemoryDescriptorReadToPhysical [%d] %x\n", ppnum, err);
-               break;
+                   HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
+                   break;
            }
 
            ppnum++;
            pagesDone++;
-
-            if (vars->videoMapSize && (0 == (1023 & pagesDone)))
-            {
-                blob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
-                        / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
-                if (blob != lastBlob)
-                {
-                    ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, lastBlob, blob);
-                    lastBlob = blob;
-                }
-            }
+           pagesRead++;
 
            if (0 == (8191 & pagesDone))
            {
@@ -2392,28 +2358,115 @@ hibernate_machine_init(void)
            }
        }
     }
-    while (true);
+    if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages))
+       err = kIOReturnLockedRead;
 
     if (kIOReturnSuccess != err)
        panic("Hibernate restore error %x", err);
 
     gIOHibernateCurrentHeader->actualImage2Sum = sum;
+    gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
 
-    if (vars->fileVars->io)
-        (void) IOHibernatePollerIODone(vars->fileVars, false);
-
-    err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
+    clock_get_uptime(&endIOTime);
 
-    if (vars->videoMapSize)
-        ProgressUpdate(gIOHibernateGraphicsInfo, 
-                        (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
+    err = IOPolledFilePollersClose(vars->fileVars, kIOPolledAfterSleepState);
 
     clock_get_uptime(&endTime);
+
+    IOService::getPMRootDomain()->pmStatsRecordEvent( 
+                        kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
+    IOService::getPMRootDomain()->pmStatsRecordEvent( 
+                        kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
+
     SUB_ABSOLUTETIME(&endTime, &allTime);
     absolutetime_to_nanoseconds(endTime, &nsec);
 
-    HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %qd ms\n", 
-               pagesDone, sum, nsec / 1000000ULL);
+    SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
+    absolutetime_to_nanoseconds(endIOTime, &nsecIO);
+
+    gIOHibernateStats->kernelImageReadDuration = nsec / 1000000ULL;
+    gIOHibernateStats->imagePages              = pagesDone;
+
+    HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ", 
+               pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
+               nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
+
+    absolutetime_to_nanoseconds(compTime, &nsec);
+    HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ", 
+               compBytes, 
+               nsec / 1000000ULL,
+               nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+    absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
+    HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n", 
+               vars->fileVars->cryptBytes, 
+               nsec / 1000000ULL, 
+               nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+    KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_NONE, pagesRead, pagesDone, 0, 0, 0);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+void IOHibernateSetWakeCapabilities(uint32_t capability)
+{
+    if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
+    {
+       gIOHibernateStats->wakeCapability = capability;
+
+       if (kIOPMSystemCapabilityGraphics & capability)
+       {
+               vm_compressor_do_warmup();
+       }
+    }
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+void IOHibernateSystemRestart(void)
+{
+    static uint8_t    noteStore[32] __attribute__((aligned(32)));
+    IORegistryEntry * regEntry;
+    const OSSymbol *  sym;
+    OSData *          noteProp;
+    OSData *          data;
+    uintptr_t *       smcVars;
+    uint8_t *         smcBytes;
+    size_t            len;
+    addr64_t          element;
+
+    data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
+    if (!data) return;
+
+    smcVars = (typeof(smcVars)) data->getBytesNoCopy();
+    smcBytes = (typeof(smcBytes)) smcVars[1];
+    len = smcVars[0];
+    if (len > sizeof(noteStore)) len = sizeof(noteStore);
+    noteProp = OSData::withCapacity(3 * sizeof(element));
+    if (!noteProp) return;
+    element = len;
+    noteProp->appendBytes(&element, sizeof(element));
+    element = crc32(0, smcBytes, len);
+    noteProp->appendBytes(&element, sizeof(element));
+
+    bcopy(smcBytes, noteStore, len);
+    element = (addr64_t) &noteStore[0];
+    element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
+    noteProp->appendBytes(&element, sizeof(element));
+
+    if (!gIOOptionsEntry)
+    {
+       regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
+       gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
+       if (regEntry && !gIOOptionsEntry)
+           regEntry->release();
+    }
+
+    sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
+    if (gIOOptionsEntry && sym) gIOOptionsEntry->setProperty(sym, noteProp);
+    if (noteProp)               noteProp->release();
+    if (sym)                    sym->release();
+}
+
+
+