]> git.saurik.com Git - apple/xnu.git/blobdiff - iokit/Kernel/IOHibernateIO.cpp
xnu-3789.70.16.tar.gz
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
index 851adac6035d329bc50785a11bf56e6cfbac0d96..953c6d44530a4a6fed9776f99cae9509ae474b50 100644 (file)
@@ -1,8 +1,8 @@
 /*
- * Copyright (c) 2004-2008 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2016 Apple Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
- * 
+ *
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
- * 
+ *
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
- * 
+ *
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
- * 
+ *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
-
 /*
 
 Sleep:
@@ -160,18 +159,21 @@ to restrict I/O ops.
 #include <sys/fcntl.h>                       // (FWRITE, ...)
 #include <sys/sysctl.h>
 #include <sys/kdebug.h>
+#include <stdint.h>
 
 #include <IOKit/IOHibernatePrivate.h>
 #include <IOKit/IOPolledInterface.h>
 #include <IOKit/IONVRAM.h>
 #include "IOHibernateInternal.h"
 #include <vm/WKdm_new.h>
+#include <vm/vm_protos.h>
 #include "IOKitKernelInternal.h"
 #include <pexpert/device_tree.h>
 
 #include <machine/pal_routines.h>
 #include <machine/pal_hibernate.h>
 #include <i386/tsc.h>
+#include <i386/cpuid.h>
 
 extern "C" addr64_t            kvtophys(vm_offset_t va);
 extern "C" ppnum_t             pmap_find_phys(pmap_t pmap, addr64_t va);
@@ -179,7 +181,7 @@ extern "C" ppnum_t          pmap_find_phys(pmap_t pmap, addr64_t va);
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 #define        DISABLE_TRIM            0
-#define TRIM_DELAY             5000
+#define TRIM_DELAY             25000
 
 extern unsigned int            save_kdebug_enable;
 extern uint32_t                gIOHibernateState;
@@ -189,25 +191,33 @@ static char                       gIOHibernateFilename[MAXPATHLEN+1];
 static uint32_t                        gIOHibernateFreeRatio = 0;       // free page target (percent)
 uint32_t                       gIOHibernateFreeTime  = 0*1000;  // max time to spend freeing pages (ms)
 static uint64_t                        gIOHibernateCompression = 0x80;  // default compression 50%
+boolean_t                       gIOHibernateStandbyDisabled;
 
 static IODTNVRAM *             gIOOptionsEntry;
 static IORegistryEntry *       gIOChosenEntry;
+
+static const OSSymbol *        gIOHibernateBootImageKey;
+
 #if defined(__i386__) || defined(__x86_64__)
-static const OSSymbol *         gIOCreateEFIDevicePathSymbol;
+
 static const OSSymbol *        gIOHibernateRTCVariablesKey;
 static const OSSymbol *         gIOHibernateBoot0082Key;
 static const OSSymbol *         gIOHibernateBootNextKey;
 static OSData *                        gIOHibernateBoot0082Data;
 static OSData *                        gIOHibernateBootNextData;
 static OSObject *              gIOHibernateBootNextSave;
-#endif
+
+static IOPolledFileIOVars *     gDebugImageFileVars;
+static IOLock             *     gDebugImageLock;
+
+#endif /* defined(__i386__) || defined(__x86_64__) */
 
 static IOLock *                           gFSLock;
 static uint32_t                           gFSState;
+static thread_call_t                      gIOHibernateTrimCalloutEntry;
 static IOPolledFileIOVars                gFileVars;
 static IOHibernateVars                   gIOHibernateVars;
-static struct kern_direct_file_io_ref_t * gIOHibernateFileRef;
-static hibernate_cryptvars_t             gIOHibernateCryptWakeContext;
+static IOPolledFileCryptVars             gIOHibernateCryptWakeContext;
 static hibernate_graphics_t              _hibernateGraphics;
 static hibernate_graphics_t *            gIOHibernateGraphicsInfo = &_hibernateGraphics;
 static hibernate_statistics_t            _hibernateStats;
@@ -215,31 +225,22 @@ static hibernate_statistics_t *             gIOHibernateStats = &_hibernateStats;
 
 enum 
 {
-    kFSIdle     = 0,
-    kFSOpening  = 2,
-    kFSOpened   = 3,
-    kFSTimedOut = 4,
+    kFSIdle      = 0,
+    kFSOpening   = 2,
+    kFSOpened    = 3,
+    kFSTimedOut  = 4,
+    kFSTrimDelay = 5
 };
 
 static IOReturn IOHibernateDone(IOHibernateVars * vars);
+static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature);
+static void     IOSetBootImageNVRAM(OSData * data);
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
-enum { kXPRamAudioVolume = 8 };
 enum { kDefaultIOSize = 128 * 1024 };
 enum { kVideoMapSize  = 80 * 1024 * 1024 };
 
-#ifndef kIOMediaPreferredBlockSizeKey
-#define kIOMediaPreferredBlockSizeKey  "Preferred Block Size"
-#endif
-
-#ifndef kIOBootPathKey 
-#define kIOBootPathKey                 "bootpath"
-#endif
-#ifndef kIOSelectedBootDeviceKey       
-#define kIOSelectedBootDeviceKey       "boot-device"
-#endif
-
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 // copy from phys addr to MD
@@ -368,783 +369,18 @@ hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
        count = hibernate_page_bitmap_count(bitmap, TRUE, page);
        if (!count)
            break;
-       page += count;
-       if (page <= bitmap->last_page)
-           break;
-    }
-
-    *pPage = page;
-    if (bitmap)
-       count = hibernate_page_bitmap_count(bitmap, FALSE, page);
-    else
-       count = 0;
-
-    return (count);
-}
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-static IOReturn
-IOHibernatePollerProbe(IOPolledFileIOVars * vars, IOService * target)
-{
-    IOReturn            err = kIOReturnError;
-    int32_t            idx;
-    IOPolledInterface * poller;
-
-    for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
-    {
-        poller = (IOPolledInterface *) vars->pollers->getObject(idx);
-        err = poller->probe(target);
-        if (err)
-        {
-            HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
-            break;
-        }
-    }
-
-    return (err);
-}
-
-static IOReturn
-IOHibernatePollerOpen(IOPolledFileIOVars * vars, uint32_t state, IOMemoryDescriptor * md)
-{
-    IOReturn            err = kIOReturnError;
-    int32_t            idx;
-    IOPolledInterface * poller;
-
-    for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
-    {
-        poller = (IOPolledInterface *) vars->pollers->getObject(idx);
-        err = poller->open(state, md);
-        if (err)
-        {
-            HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
-            break;
-        }
-    }
-
-    return (err);
-}
-
-static IOReturn
-IOHibernatePollerClose(IOPolledFileIOVars * vars, uint32_t state)
-{
-    IOReturn            err = kIOReturnError;
-    int32_t            idx;
-    IOPolledInterface * poller;
-
-    for (idx = 0;
-         (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
-         idx++)
-    {
-        err = poller->close(state);
-        if (err)
-            HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
-    }
-
-    return (err);
-}
-
-static void
-IOHibernatePollerIOComplete(void *   target,
-                            void *   parameter,
-                            IOReturn status,
-                            UInt64   actualByteCount)
-{
-    IOPolledFileIOVars * vars = (IOPolledFileIOVars *) parameter;
-
-    vars->ioStatus = status;
-}
-
-static IOReturn
-IOHibernatePollerIO(IOPolledFileIOVars * vars, 
-                    uint32_t operation, uint32_t bufferOffset, 
-                   uint64_t deviceOffset, uint64_t length)
-{
-
-    IOReturn            err = kIOReturnError;
-    IOPolledInterface * poller;
-    IOPolledCompletion  completion;
-
-    completion.target    = 0;
-    completion.action    = &IOHibernatePollerIOComplete;
-    completion.parameter = vars;
-
-    vars->ioStatus = -1;
-
-    poller = (IOPolledInterface *) vars->pollers->getObject(0);
-    err = poller->startIO(operation, bufferOffset, deviceOffset + vars->block0, length, completion);
-    if (err)
-        HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
-
-    return (err);
-}
-
-static IOReturn
-IOHibernatePollerIODone(IOPolledFileIOVars * vars, bool abortable)
-{
-    IOReturn            err = kIOReturnSuccess;
-    int32_t            idx = 0;
-    IOPolledInterface * poller;
-
-    while (-1 == vars->ioStatus)
-    {
-        for (idx = 0; 
-           (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
-             idx++)
-        {
-           IOReturn newErr;
-            newErr = poller->checkForWork();
-           if ((newErr == kIOReturnAborted) && !abortable)
-               newErr = kIOReturnSuccess;
-           if (kIOReturnSuccess == err)
-               err = newErr;
-        }
-    }
-
-    if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort())
-    {
-        err = kIOReturnAborted;
-       HIBLOG("IOPolledInterface::checkForWork sw abort\n");
-    }
-
-    if (err)
-    {
-       HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
-    }
-    else 
-    {
-       err = vars->ioStatus;
-       if (kIOReturnSuccess != err)
-           HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
-    }
-
-    return (err);
-}
-
-IOReturn
-IOPolledInterface::checkAllForWork(void)
-{
-    IOReturn            err = kIOReturnNotReady;
-    int32_t            idx;
-    IOPolledInterface * poller;
-
-    IOHibernateVars * vars  = &gIOHibernateVars;
-
-    if (!vars->fileVars || !vars->fileVars->pollers)
-       return (err);
-
-    for (idx = 0;
-            (poller = (IOPolledInterface *) vars->fileVars->pollers->getObject(idx));
-            idx++)
-    {
-        err = poller->checkForWork();
-        if (err)
-            HIBLOG("IOPolledInterface::checkAllForWork[%d] 0x%x\n", idx, err);
-    }
-
-    return (err);
-}
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-struct _OpenFileContext
-{
-    OSData * extents;
-    uint64_t size;
-};
-
-static void
-file_extent_callback(void * ref, uint64_t start, uint64_t length)
-{
-    _OpenFileContext * ctx = (_OpenFileContext *) ref;
-    IOPolledFileExtent extent;
-
-    extent.start  = start;
-    extent.length = length;
-
-    HIBLOG("[0x%qx, 0x%qx]\n", start, length);
-
-    ctx->extents->appendBytes(&extent, sizeof(extent));
-    ctx->size += length;
-}
-
-static IOService * 
-IOCopyMediaForDev(dev_t device)
-{
-    OSDictionary * matching;
-    OSNumber *     num;
-    OSIterator *   iter;
-    IOService *    result = 0;
-
-    matching = IOService::serviceMatching("IOMedia");
-    if (!matching)
-        return (0);
-    do
-    {
-        num = OSNumber::withNumber(major(device), 32);
-        if (!num)
-            break;
-        matching->setObject(kIOBSDMajorKey, num);
-        num->release();
-        num = OSNumber::withNumber(minor(device), 32);
-        if (!num)
-            break;
-        matching->setObject(kIOBSDMinorKey, num);
-        num->release();
-        if (!num)
-            break;
-        iter = IOService::getMatchingServices(matching);
-        if (iter)
-        {
-            result = (IOService *) iter->getNextObject();
-            result->retain();
-            iter->release();
-        }
-    }
-    while (false);
-    matching->release();
-
-    return (result);
-}
-
-IOReturn
-IOPolledFileOpen( const char * filename, uint64_t setFileSize,
-                 IOBufferMemoryDescriptor * ioBuffer, 
-                 IOPolledFileIOVars ** fileVars, OSData ** fileExtents,
-                 OSData ** imagePath, uint8_t * volumeCryptKey)
-{
-    IOReturn                   err = kIOReturnSuccess;
-    IOPolledFileIOVars *       vars;
-    _OpenFileContext           ctx;
-    OSData *                   extentsData;
-    OSNumber *                 num;
-    IOService *                 part = 0;
-    OSString *                  keyUUID = 0;
-    OSString *                  keyStoreUUID = 0;
-    dev_t                      block_dev;
-    dev_t                      hibernate_image_dev;
-    uint64_t                   maxiobytes;
-    AbsoluteTime                startTime, endTime;
-    uint64_t                    nsec;
-
-    vars = IONew(IOPolledFileIOVars, 1);
-    if (!vars) return (kIOReturnNoMemory);
-    bzero(vars, sizeof(*vars));
-
-    do
-    {
-       vars->io           = false;
-       vars->buffer       = (uint8_t *) ioBuffer->getBytesNoCopy();
-       vars->bufferHalf   = 0;
-       vars->bufferOffset = 0;
-       vars->bufferSize   = ioBuffer->getLength() >> 1;
-    
-       extentsData = OSData::withCapacity(32);
-       ctx.extents = extentsData;
-       ctx.size    = 0;
-       clock_get_uptime(&startTime);
-       vars->fileRef = kern_open_file_for_direct_io(filename, 
-                                                   &file_extent_callback, &ctx, 
-                                                   setFileSize,
-                                                   // write file:
-                                                    0, (caddr_t) gIOHibernateCurrentHeader, 
-                                                    sizeof(IOHibernateImageHeader),
-                                                    // results
-                                                   &block_dev,
-                                                   &hibernate_image_dev,
-                                                    &vars->block0,
-                                                    &maxiobytes,
-                                                    &vars->flags);
-#if 0
-       uint32_t msDelay = (131071 & random());
-       HIBLOG("sleep %d\n", msDelay);
-       IOSleep(msDelay);
-#endif
-        clock_get_uptime(&endTime);
-        SUB_ABSOLUTETIME(&endTime, &startTime);
-        absolutetime_to_nanoseconds(endTime, &nsec);
-
-       if (!vars->fileRef) err = kIOReturnNoSpace;
-
-       IOLockLock(gFSLock);
-       if (kFSOpening != gFSState) err = kIOReturnTimeout;
-       IOLockUnlock(gFSLock);
-
-        HIBLOG("kern_open_file_for_direct_io(%d) took %qd ms\n", err, nsec / 1000000ULL);
-       if (kIOReturnSuccess != err) break;
-
-        if (kIOHibernateModeSSDInvert & gIOHibernateMode)
-            vars->flags ^= kIOHibernateOptionSSD;
-
-       HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx ssd %d\n", filename, ctx.size, 
-                    vars->block0, maxiobytes, kIOHibernateOptionSSD & vars->flags);
-       if (ctx.size < 1*1024*1024)             // check against image size estimate!
-       {
-           err = kIOReturnNoSpace;
-           break;
-       }
-
-       vars->fileSize = ctx.size;
-        if (maxiobytes < vars->bufferSize) vars->bufferSize = maxiobytes;
-    
-       vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
-
-        part = IOCopyMediaForDev(block_dev);
-        if (!part)
-        {
-            err = kIOReturnNotFound;
-            break;
-       }
-        err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false, 
-                                         (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
-        if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID)
-        {
-//            IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
-            uuid_t                  volumeKeyUUID;
-            aks_volume_key_t        vek;
-            static IOService *      sKeyStore;
-            static const OSSymbol * sAKSGetKey;
-
-            if (!sAKSGetKey)
-                sAKSGetKey = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY);
-            if (!sKeyStore)
-                sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
-            if (sKeyStore)
-                err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
-            else
-                err = kIOReturnNoResources;
-            if (kIOReturnSuccess == err)    
-                err = sKeyStore->callPlatformFunction(sAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
-            if (kIOReturnSuccess != err)    
-                IOLog("volume key err 0x%x\n", err);
-            else
-            {
-                size_t bytes = (kIOHibernateAESKeySize / 8);
-                if (vek.key.keybytecount < bytes)
-                     bytes = vek.key.keybytecount;
-                bcopy(&vek.key.keybytes[0], volumeCryptKey, bytes);
-            }
-            bzero(&vek, sizeof(vek));
-        }
-        part->release();
-
-        part = IOCopyMediaForDev(hibernate_image_dev);
-        if (!part)
-        {
-            err = kIOReturnNotFound;
-            break;
-       }
-
-       IORegistryEntry * next;
-       IORegistryEntry * child;
-       IOService       * service;
-       OSData          * data;
-
-        vars->pollers = OSArray::withCapacity(4);
-       if (!vars->pollers)
-        {
-            err = kIOReturnNoMemory;
-            break;
-       }
-
-       vars->blockSize = 512;
-       next = part;
-       do
-       {
-            IOPolledInterface * poller;
-           OSObject *          obj;
-
-           obj = next->getProperty(kIOPolledInterfaceSupportKey);
-           if (kOSBooleanFalse == obj)
-           {
-               vars->pollers->flushCollection();
-               break;
-           }
-            else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
-                vars->pollers->setObject(poller);
-
-           if ((service = OSDynamicCast(IOService, next)) 
-               && service->getDeviceMemory()
-               && !vars->pollers->getCount())  break;
-
-           if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey))))
-               vars->blockSize = num->unsigned32BitValue();
-            child = next;
-       }
-       while ((next = child->getParentEntry(gIOServicePlane)) 
-                && child->isParent(next, gIOServicePlane, true));
-
-       if (vars->blockSize < 4096) vars->blockSize = 4096;
-
-       HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
-                   major(hibernate_image_dev), minor(hibernate_image_dev), (long)vars->blockSize, 
-                   vars->pollers->getCount());
-
-       if (!vars->pollers->getCount())
-       {
-            err = kIOReturnUnsupported;
-           continue;
-       }
-       if (vars->blockSize < sizeof(IOHibernateImageHeader))
-       {
-           err = kIOReturnError;
-           continue;
-       }
-
-       err = IOHibernatePollerProbe(vars, (IOService *) part);
-       if (kIOReturnSuccess != err) break;
-
-       err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer);
-       if (kIOReturnSuccess != err) break;
-
-       vars->media = part;
-        next = part;
-       while (next)
-       {
-           next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue);
-           next = next->getParentEntry(gIOServicePlane);
-       }
-
-       *fileVars    = vars;
-       *fileExtents = extentsData;
-    
-       // make imagePath
-
-       if ((extentsData->getLength() >= sizeof(IOPolledFileExtent)))
-       {
-           char str2[24 + sizeof(uuid_string_t) + 2];
-
-#if defined(__i386__) || defined(__x86_64__)
-           if (!gIOCreateEFIDevicePathSymbol)
-               gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
-
-            if (keyUUID)
-                snprintf(str2, sizeof(str2), "%qx:%s", 
-                                vars->extentMap[0].start, keyUUID->getCStringNoCopy());
-            else
-                snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
-
-           err = IOService::getPlatform()->callPlatformFunction(
-                                               gIOCreateEFIDevicePathSymbol, false,
-                                               (void *) part, (void *) str2,
-                                               (void *) (uintptr_t) true, (void *) &data);
-#else
-           char str1[256];
-           int len = sizeof(str1);
-
-           if (!part->getPath(str1, &len, gIODTPlane))
-               err = kIOReturnNotFound;
-           else
-           {
-               snprintf(str2, sizeof(str2), ",%qx", vars->extentMap[0].start);
-               // (strip the plane name)
-               char * tail = strchr(str1, ':');
-               if (!tail)
-                   tail = str1 - 1;
-               data = OSData::withBytes(tail + 1, strlen(tail + 1));
-               data->appendBytes(str2, strlen(str2));
-           }
-#endif
-       if (kIOReturnSuccess == err)
-           *imagePath = data;
-       else
-           HIBLOG("error 0x%x getting path\n", err);
-       }
-    }
-    while (false);
-
-    if (kIOReturnSuccess != err)
-    {
-        HIBLOG("error 0x%x opening hibernation file\n", err);
-       if (vars->fileRef)
-       {
-           kern_close_file_for_direct_io(vars->fileRef, 0, 0, 0, 0, 0);
-           vars->fileRef = NULL;
-       }
-    }
-
-    if (part)
-       part->release();
-
-    return (err);
-}
-
-IOReturn
-IOPolledFileClose( IOPolledFileIOVars * vars )
-{
-    if (vars->pollers)
-    {
-       IOHibernatePollerClose(vars, kIOPolledPostflightState);
-        vars->pollers->release();
-    }
-
-    bzero(vars, sizeof(IOPolledFileIOVars));
-
-    return (kIOReturnSuccess);
-}
-
-static IOReturn
-IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
-{
-    IOPolledFileExtent * extentMap;
-
-    extentMap = vars->extentMap;
-
-    vars->position = position;
-
-    while (position >= extentMap->length)
-    {
-       position -= extentMap->length;
-       extentMap++;
-    }
-
-    vars->currentExtent   = extentMap;
-    vars->extentRemaining = extentMap->length - position;
-    vars->extentPosition  = vars->position - position;
-
-    if (vars->bufferSize <= vars->extentRemaining)
-       vars->bufferLimit = vars->bufferSize;
-    else
-       vars->bufferLimit = vars->extentRemaining;
-
-    return (kIOReturnSuccess);
-}
-
-static IOReturn
-IOPolledFileWrite(IOPolledFileIOVars * vars,
-                    const uint8_t * bytes, IOByteCount size,
-                    hibernate_cryptvars_t * cryptvars)
-{
-    IOReturn    err = kIOReturnSuccess;
-    IOByteCount copy;
-    bool       flush = false;
-
-    do
-    {
-       if (!bytes && !size)
-       {
-           // seek to end of block & flush
-           size = vars->position & (vars->blockSize - 1);
-           if (size)
-               size = vars->blockSize - size;
-           flush = true;
-            // use some garbage for the fill
-            bytes = vars->buffer + vars->bufferOffset;
-       }
-
-       copy = vars->bufferLimit - vars->bufferOffset;
-       if (copy > size)
-           copy = size;
-       else
-           flush = true;
-
-       if (bytes)
-       {
-           bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
-           bytes += copy;
-       }
-        else
-           bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
-        
-       size -= copy;
-       vars->bufferOffset += copy;
-       vars->position += copy;
-
-       if (flush && vars->bufferOffset)
-       {
-           uint64_t offset = (vars->position - vars->bufferOffset 
-                               - vars->extentPosition + vars->currentExtent->start);
-           uint32_t length = (vars->bufferOffset);
-
-#if CRYPTO
-            if (cryptvars && vars->encryptStart
-                && (vars->position > vars->encryptStart)
-                && ((vars->position - length) < vars->encryptEnd))
-            {
-                AbsoluteTime startTime, endTime;
-
-                uint64_t encryptLen, encryptStart;
-                encryptLen = vars->position - vars->encryptStart;
-                if (encryptLen > length)
-                    encryptLen = length;
-                encryptStart = length - encryptLen;
-                if (vars->position > vars->encryptEnd)
-                    encryptLen -= (vars->position - vars->encryptEnd);
-
-                clock_get_uptime(&startTime);
-
-                // encrypt the buffer
-                aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
-                                &cryptvars->aes_iv[0],
-                                encryptLen / AES_BLOCK_SIZE,
-                                vars->buffer + vars->bufferHalf + encryptStart,
-                                &cryptvars->ctx.encrypt);
-    
-                clock_get_uptime(&endTime);
-                ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
-                SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
-                vars->cryptBytes += encryptLen;
-
-                // save initial vector for following encrypts
-                bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
-                        &cryptvars->aes_iv[0],
-                        AES_BLOCK_SIZE);
-            }
-#endif /* CRYPTO */
-
-           if (vars->io)
-            {
-               err = IOHibernatePollerIODone(vars, true);
-                if (kIOReturnSuccess != err)
-                    break;
-            }
-
-if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
-//if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
-
-           err = IOHibernatePollerIO(vars, kIOPolledWrite, vars->bufferHalf, offset, length);
-            if (kIOReturnSuccess != err)
-                break;
-           vars->io = true;
-
-           vars->extentRemaining -= vars->bufferOffset;
-           if (!vars->extentRemaining)
-           {
-               vars->currentExtent++;
-               vars->extentRemaining = vars->currentExtent->length;
-               vars->extentPosition  = vars->position;
-                if (!vars->extentRemaining)
-                {
-                    err = kIOReturnOverrun;
-                    break;
-                }
-           }
-
-           vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
-           vars->bufferOffset = 0;
-           if (vars->bufferSize <= vars->extentRemaining)
-               vars->bufferLimit = vars->bufferSize;
-           else
-               vars->bufferLimit = vars->extentRemaining;
-
-           flush = false;
-       }
-    }
-    while (size);
-
-    return (err);
-}
-
-static IOReturn
-IOPolledFileRead(IOPolledFileIOVars * vars,
-                    uint8_t * bytes, IOByteCount size,
-                    hibernate_cryptvars_t * cryptvars)
-{
-    IOReturn    err = kIOReturnSuccess;
-    IOByteCount copy;
-
-//    bytesWritten += size;
-
-    do
-    {
-       copy = vars->bufferLimit - vars->bufferOffset;
-       if (copy > size)
-           copy = size;
-
-       if (bytes)
-       {
-           bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
-           bytes += copy;
-       }
-       size -= copy;
-       vars->bufferOffset += copy;
-//     vars->position += copy;
-
-       if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd))
-       {
-           if (vars->io)
-            {
-               err = IOHibernatePollerIODone(vars, false);
-                if (kIOReturnSuccess != err)
-                    break;
-            }
-            else
-                cryptvars = 0;
-
-if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
-
-           vars->position        += vars->lastRead;
-           vars->extentRemaining -= vars->lastRead;
-           vars->bufferLimit      = vars->lastRead;
-
-           if (!vars->extentRemaining)
-           {
-               vars->currentExtent++;
-               vars->extentRemaining = vars->currentExtent->length;
-               vars->extentPosition  = vars->position;
-                if (!vars->extentRemaining)
-                {
-                    err = kIOReturnOverrun;
-                    break;
-                }
-           }
-
-           uint64_t length;
-           uint64_t lastReadLength = vars->lastRead;
-           uint64_t offset = (vars->position 
-                               - vars->extentPosition + vars->currentExtent->start);
-           if (vars->extentRemaining <= vars->bufferSize)
-               length = vars->extentRemaining;
-           else
-               length = vars->bufferSize;
-           if ((length + vars->position) > vars->readEnd)
-               length = vars->readEnd - vars->position;
-
-           vars->lastRead = length;
-           if (length)
-           {
-//if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
-               err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
-               if (kIOReturnSuccess != err)
-                   break;
-               vars->io = true;
-           }
-
-           vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
-           vars->bufferOffset = 0;
-
-#if CRYPTO
-            if (cryptvars)
-            {
-                uint8_t thisVector[AES_BLOCK_SIZE];
-                AbsoluteTime startTime, endTime;
-
-                // save initial vector for following decrypts
-                bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
-                bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE, 
-                        &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
-
-                // decrypt the buffer
-                clock_get_uptime(&startTime);
-
-                aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
-                                &thisVector[0],
-                                lastReadLength / AES_BLOCK_SIZE,
-                                vars->buffer + vars->bufferHalf,
-                                &cryptvars->ctx.decrypt);
-
-                clock_get_uptime(&endTime);
-                ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
-                SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
-                vars->cryptBytes += lastReadLength;
-            }
-#endif /* CRYPTO */
-       }
+       page += count;
+       if (page <= bitmap->last_page)
+           break;
     }
-    while (size);
 
-    return (err);
+    *pPage = page;
+    if (bitmap)
+       count = hibernate_page_bitmap_count(bitmap, FALSE, page);
+    else
+       count = 0;
+
+    return (count);
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@@ -1153,18 +389,16 @@ IOReturn
 IOHibernateSystemSleep(void)
 {
     IOReturn   err;
-    OSData *   data;
+    OSData *   nvramData;
     OSObject * obj;
     OSString * str;
     OSNumber * num;
-    bool       dsSSD, vmflush;
+    bool       dsSSD, vmflush, swapPinned;
     IOHibernateVars * vars;
+    uint64_t   setFileSize = 0;
 
     gIOHibernateState = kIOHibernateStateInactive;
 
-    if (!gIOChosenEntry)
-       gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
-
     gIOHibernateDebugFlags = 0;
     if (kIOLogHibernate & gIOKitDebug)
        gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
@@ -1196,6 +430,11 @@ IOHibernateSystemSleep(void)
     bzero(vars, sizeof(*vars));
 
     IOLockLock(gFSLock);
+    if (!gIOHibernateTrimCalloutEntry)
+    {
+        gIOHibernateTrimCalloutEntry = thread_call_allocate(&IOHibernateSystemPostWakeTrim, &gFSLock);
+    }
+    IOHibernateSystemPostWakeTrim(NULL, NULL);
     if (kFSIdle != gFSState)
     {
        HIBLOG("hibernate file busy\n");
@@ -1206,17 +445,16 @@ IOHibernateSystemSleep(void)
     gFSState = kFSOpening;
     IOLockUnlock(gFSLock);
 
+    swapPinned = false;
     do
     {
         vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
-                                   2 * page_size + WKdm_SCRATCH_BUF_SIZE, page_size);
-        vars->ioBuffer  = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, 
-                                   2 * kDefaultIOSize, page_size);
+                                   2 * page_size + WKdm_SCRATCH_BUF_SIZE_INTERNAL, page_size);
 
        vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, 
                                    ptoa_64(gIOHibernateHandoffPageCount), page_size);
 
-        if (!vars->srcBuffer || !vars->ioBuffer || !vars->handoffBuffer)
+        if (!vars->srcBuffer || !vars->handoffBuffer)
         {
             err = kIOReturnNoMemory;
             break;
@@ -1242,13 +480,15 @@ IOHibernateSystemSleep(void)
        gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
        gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
 
-       vmflush = (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey));
-       uint64_t setFileSize = 0;
+       vmflush = ((kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
         err = hibernate_alloc_page_lists(&vars->page_list, 
                                         &vars->page_list_wired,
                                         &vars->page_list_pal);
-        if (KERN_SUCCESS != err)
-            break;
+        if (KERN_SUCCESS != err) break;
+
+       err = hibernate_pin_swap(TRUE);
+       if (KERN_SUCCESS != err) break;
+       swapPinned = true;
 
        if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode))
        {
@@ -1263,13 +503,13 @@ IOHibernateSystemSleep(void)
            IOService::getPlatform()->getConsoleInfo(&consoleInfo);
 
            // estimate: 6% increase in pages compressed
-           // screen preview 2 images compressed 50%
+           // screen preview 2 images compressed 0%
            setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
                                + vars->page_list->list_size
                                + (consoleInfo.v_width * consoleInfo.v_height * 8);
            enum { setFileRound = 1024*1024ULL };
            setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
-
+       
            HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n", 
                    pageCount, (100ULL * gIOHibernateCompression) >> 8,
                    setFileSize, vars->fileMinSize);
@@ -1281,11 +521,27 @@ IOHibernateSystemSleep(void)
            }
        }
     
-       // open & invalidate the image file
+       // Invalidate the image file
+    if (gDebugImageLock) {
+        IOLockLock(gDebugImageLock);
+        if (gDebugImageFileVars != 0) {
+            IOSetBootImageNVRAM(0);
+            IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
+        }
+        IOLockUnlock(gDebugImageLock);
+    }
+
+        err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, 0,
+                               gIOHibernateCurrentHeader, sizeof(gIOHibernateCurrentHeader),
+                                &vars->fileVars, &nvramData, 
+                                &vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
 
-        err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, vars->ioBuffer,
-                                &vars->fileVars, &vars->fileExtents, &data, 
-                                &vars->volumeCryptKey[0]);
+        if (KERN_SUCCESS != err)
+        {
+           IOLockLock(gFSLock);
+           if (kFSOpening != gFSState) err = kIOReturnTimeout;
+           IOLockUnlock(gFSLock);
+       }
 
         if (KERN_SUCCESS != err)
         {
@@ -1293,53 +549,60 @@ IOHibernateSystemSleep(void)
             break;
         }
 
+       // write extents for debug data usage in EFI
+        IOWriteExtentsToFile(vars->fileVars, kIOHibernateHeaderOpenSignature);
+
+        err = IOPolledFilePollersSetup(vars->fileVars, kIOPolledPreflightState);
+        if (KERN_SUCCESS != err) break;
+
         clock_get_uptime(&startTime);
         err = hibernate_setup(gIOHibernateCurrentHeader, 
-                                gIOHibernateFreeRatio, gIOHibernateFreeTime,
                                 vmflush,
                                 vars->page_list, vars->page_list_wired, vars->page_list_pal);
         clock_get_uptime(&endTime);
         SUB_ABSOLUTETIME(&endTime, &startTime);
         absolutetime_to_nanoseconds(endTime, &nsec);
-        HIBLOG("hibernate_setup(%d) took %qd ms\n", err, nsec / 1000000ULL);
 
-        dsSSD = ((0 != (kIOHibernateOptionSSD & vars->fileVars->flags))
+        boolean_t haveSwapPin, hibFileSSD;
+        haveSwapPin = vm_swap_files_pinned();
+
+        hibFileSSD = (kIOPolledFileSSD & vars->fileVars->flags);
+
+        HIBLOG("hibernate_setup(%d) took %qd ms, swapPin(%d) ssd(%d)\n",
+                err, nsec / 1000000ULL,
+                haveSwapPin, hibFileSSD);
+        if (KERN_SUCCESS != err) break;
+
+        gIOHibernateStandbyDisabled = ((!haveSwapPin || !hibFileSSD));
+
+        dsSSD = ((0 != (kIOPolledFileSSD & vars->fileVars->flags))
                 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
-        if (dsSSD)
-        {
-            gIOHibernateCurrentHeader->options |= 
-                                                kIOHibernateOptionSSD
-                                              | kIOHibernateOptionColor;
+
+        if (dsSSD) gIOHibernateCurrentHeader->options |= kIOHibernateOptionSSD | kIOHibernateOptionColor;
+        else       gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
+
 
 #if defined(__i386__) || defined(__x86_64__)
-            if (!uuid_is_null(vars->volumeCryptKey) &&
-                  (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey)))
-            {
-                uintptr_t smcVars[2];
-                smcVars[0] = sizeof(vars->volumeCryptKey);
-                smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
+       if (!uuid_is_null(vars->volumeCryptKey) &&
+             (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey)))
+       {
+           uintptr_t smcVars[2];
+           smcVars[0] = sizeof(vars->volumeCryptKey);
+           smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
 
-                IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
-                bzero(smcVars, sizeof(smcVars));
-            }
+           IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
+           bzero(smcVars, sizeof(smcVars));
+       }
 #endif
-        }
-        else
-        {
-            gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
-        }
 
 
-        if (KERN_SUCCESS != err)
-            break;
-
         if (encryptedswap || !uuid_is_null(vars->volumeCryptKey))
             gIOHibernateMode ^= kIOHibernateModeEncrypt; 
 
         if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
         {
             vars->videoAllocSize = kVideoMapSize;
-            if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
+            if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize, VM_KERN_MEMORY_IOKIT))
                 vars->videoMapping = 0;
         }
 
@@ -1351,28 +614,11 @@ IOHibernateSystemSleep(void)
 
        // set nvram
 
-        IORegistryEntry * regEntry;
-        if (!gIOOptionsEntry)
-        {
-            regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
-            gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
-            if (regEntry && !gIOOptionsEntry)
-                regEntry->release();
-        }
-
-       if (gIOOptionsEntry)
-       {
-            const OSSymbol *  sym;
-
-            sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
-            if (sym)
-            {
-                gIOOptionsEntry->setProperty(sym, data);
-                sym->release();
-            }
-            data->release();
+       IOSetBootImageNVRAM(nvramData);
+        nvramData->release();
 
 #if defined(__i386__) || defined(__x86_64__)
+       {
            struct AppleRTCHibernateVars
            {
                uint8_t     signature[4];
@@ -1381,6 +627,7 @@ IOHibernateSystemSleep(void)
                uint8_t     wiredCryptKey[16];
            };
            AppleRTCHibernateVars rtcVars;
+           OSData * data;
 
            rtcVars.signature[0] = 'A';
            rtcVars.signature[1] = 'A';
@@ -1396,167 +643,94 @@ IOHibernateSystemSleep(void)
                    (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
                    i++)
                {
-                   if (c >= 'a')
-                       c -= 'a' - 10;
-                   else if (c >= 'A')
-                       c -= 'A' - 10;
-                   else if (c >= '0')
-                       c -= '0';
-                   else
-                       continue;
+                   if (c >= 'a')      c -= 'a' - 10;
+                   else if (c >= 'A') c -= 'A' - 10;
+                   else if (c >= '0') c -= '0';
+                   else               continue;
                    value = (value << 4) | c;
-                   if (i & 1)
-                       rtcVars.booterSignature[i >> 1] = value;
+                   if (i & 1) rtcVars.booterSignature[i >> 1] = value;
                }
            }
            data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
            if (data)
            { 
-               if (!gIOHibernateRTCVariablesKey)
-                   gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
                if (gIOHibernateRTCVariablesKey)
                    IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
-       
-               if( gIOOptionsEntry )
-               {
-                   if( gIOHibernateMode & kIOHibernateModeSwitch )
-                   {
-                       const OSSymbol *sym;
-                       sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSwitchVarsKey);
-                       if( sym )
-                       {
-                           gIOOptionsEntry->setProperty(sym, data); /* intentional insecure backup of rtc boot vars */
-                           sym->release();
-                       }
-                   }   
-               }
-
                data->release();
            }
             if (gIOChosenEntry)
             {
                 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
-                if (data)
-                    gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
+                if (data) gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
+               // set BootNext
+               if (!gIOHibernateBoot0082Data)
                {
-                   // set BootNext
-
-                   if (!gIOHibernateBoot0082Data)
+                   data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
+                   if (data)
                    {
-                       data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
-                       if (data)
+                       // AppleNVRAM_EFI_LOAD_OPTION
+                       struct {
+                           uint32_t Attributes;
+                           uint16_t FilePathLength;
+                           uint16_t Desc;
+                       } loadOptionHeader;
+                       loadOptionHeader.Attributes     = 1;
+                       loadOptionHeader.FilePathLength = data->getLength();
+                       loadOptionHeader.Desc           = 0;
+                       gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
+                       if (gIOHibernateBoot0082Data)
                        {
-                           // AppleNVRAM_EFI_LOAD_OPTION
-                           struct {
-                               uint32_t Attributes;
-                               uint16_t FilePathLength;
-                               uint16_t Desc;
-                           } loadOptionHeader;
-                           loadOptionHeader.Attributes     = 1;
-                           loadOptionHeader.FilePathLength = data->getLength();
-                           loadOptionHeader.Desc           = 0;
-                           gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
-                           if (gIOHibernateBoot0082Data)
-                           {
-                               gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
-                               gIOHibernateBoot0082Data->appendBytes(data);
-                           }
+                           gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
+                           gIOHibernateBoot0082Data->appendBytes(data);
                        }
                    }
-                   if (!gIOHibernateBoot0082Key)
-                       gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
-                   if (!gIOHibernateBootNextKey)
-                       gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
-                   if (!gIOHibernateBootNextData)
-                   {
-                       uint16_t bits = 0x0082;
-                       gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
-                   }
-                   if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData)
-                   {
-                       gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
-                       gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
-                       gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
-                   }
-               }
-            }
-#else /* !i386 && !x86_64 */
-            if (kIOHibernateModeEncrypt & gIOHibernateMode)
-            {
-                data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
-                sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
-                if (sym && data)
-                    gIOOptionsEntry->setProperty(sym, data);
-                if (sym)
-                    sym->release();
-                if (data)
-                    data->release();
-                if (false && gIOHibernateBootSignature[0])
-                {
-                    data = OSData::withCapacity(16);
-                    sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
-                    if (sym && data)
-                    {
-                        char c;
-                        uint8_t value = 0;
-                        for (uint32_t i = 0; (c = gIOHibernateBootSignature[i]); i++)
-                        {
-                            if (c >= 'a')
-                                c -= 'a' - 10;
-                            else if (c >= 'A')
-                                c -= 'A' - 10;
-                            else if (c >= '0')
-                                c -= '0';
-                            else
-                                continue;
-                            value = (value << 4) | c;
-                            if (i & 1)
-                                data->appendBytes(&value, sizeof(value));
-                        }
-                        gIOOptionsEntry->setProperty(sym, data);
-                    }
-                    if (sym)
-                        sym->release();
-                    if (data)
-                        data->release();
-                }
-            }
-            if (!vars->haveFastBoot)
-            {
-                // set boot volume to zero
-                IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
-                if (platform && (kIOReturnSuccess == platform->readXPRAM(kXPRamAudioVolume, 
-                                            &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume))))
-                {
-                    uint8_t newVolume;
-                    newVolume = vars->saveBootAudioVolume & 0xf8;
-                    platform->writeXPRAM(kXPRamAudioVolume, 
-                                            &newVolume, sizeof(newVolume));
-                }
+               }
+               if (!gIOHibernateBootNextData)
+               {
+                   uint16_t bits = 0x0082;
+                   gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
+               }
+               if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData)
+               {
+                   gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
+                   gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
+                   gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
+               }
+               // BootNext
             }
-#endif /* !i386 && !x86_64 */
        }
-       // --
-
+#endif /* !i386 && !x86_64 */
     }
     while (false);
 
+    if (swapPinned) hibernate_pin_swap(FALSE);
+
     IOLockLock(gFSLock);
-    if ((kIOReturnSuccess == err) && (kFSOpening == gFSState))
+    if ((kIOReturnSuccess == err) && (kFSOpening != gFSState))
+    {
+       HIBLOG("hibernate file close due timeout\n");
+       err = kIOReturnTimeout;
+    }
+    if (kIOReturnSuccess == err)
     {
        gFSState = kFSOpened;
        gIOHibernateVars = *vars;
        gFileVars = *vars->fileVars;
+       gFileVars.allocated = false;
        gIOHibernateVars.fileVars = &gFileVars;
-       gIOHibernateFileRef = gFileVars.fileRef;
        gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
        gIOHibernateState = kIOHibernateStateHibernating;
     }
     else
     {
-       HIBLOG("hibernate file close due timeout\n");
-       if (vars->fileVars && vars->fileVars->fileRef) kern_close_file_for_direct_io(vars->fileVars->fileRef, 0, 0, 0, 0, 0);
+       IOPolledFileIOVars * fileVars = vars->fileVars;
        IOHibernateDone(vars);
+    IOPolledFileClose(&fileVars,
+#if DISABLE_TRIM
+                      0, NULL, 0, 0, 0);
+#else
+                      0, NULL, 0, sizeof(IOHibernateImageHeader), setFileSize);
+#endif
        gFSState = kFSIdle;
     }
     IOLockUnlock(gFSLock);
@@ -1569,6 +743,155 @@ IOHibernateSystemSleep(void)
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
+static void
+IOSetBootImageNVRAM(OSData * data)
+{
+    IORegistryEntry * regEntry;
+
+    if (!gIOOptionsEntry)
+    {
+        regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
+        gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
+        if (regEntry && !gIOOptionsEntry)
+            regEntry->release();
+    }
+    if (gIOOptionsEntry && gIOHibernateBootImageKey)
+    {
+       if (data) gIOOptionsEntry->setProperty(gIOHibernateBootImageKey, data);
+       else
+       {
+           gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
+           gIOOptionsEntry->sync();
+       }
+    }
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+/* 
+ * Writes header to disk with signature, block size and file extents data.
+ * If there are more than 2 extents, then they are written on second block.
+ */
+static IOReturn
+IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature)
+{
+    IOHibernateImageHeader hdr;
+    IOItemCount            count;
+    IOReturn               err = kIOReturnSuccess;
+    int                    rc;
+    IOPolledFileExtent *   fileExtents;
+
+    fileExtents = (typeof(fileExtents)) vars->fileExtents->getBytesNoCopy();
+
+    memset(&hdr, 0, sizeof(IOHibernateImageHeader));
+    count = vars->fileExtents->getLength();
+    if (count > sizeof(hdr.fileExtentMap))
+    {
+        hdr.fileExtentMapSize = count;
+        count = sizeof(hdr.fileExtentMap);
+    }
+    else
+        hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap);
+
+    bcopy(fileExtents, &hdr.fileExtentMap[0], count);
+
+    // copy file block extent list if larger than header
+    if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap))
+    {
+            count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap);
+            rc = kern_write_file(vars->fileRef, vars->blockSize, 
+                                 (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)), 
+                                 count, IO_SKIP_ENCRYPTION);
+            if (rc != 0) {
+                HIBLOG("kern_write_file returned %d\n", rc);
+                err = kIOReturnIOError;
+                goto exit;
+            }    
+    }
+    hdr.signature = signature;
+    hdr.deviceBlockSize = vars->blockSize;
+
+    rc = kern_write_file(vars->fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION);
+    if (rc != 0) {
+        HIBLOG("kern_write_file returned %d\n", rc);
+        err = kIOReturnIOError;
+        goto exit;
+    }
+
+exit:
+    return err;
+}
+
+extern "C" boolean_t root_is_CF_drive;
+
+void
+IOOpenDebugDataFile(const char *fname, uint64_t size)
+{
+    IOReturn   err;
+    OSData *   imagePath = NULL;
+    uint64_t   padding;
+
+    if (!gDebugImageLock) {
+        gDebugImageLock = IOLockAlloc();
+    }
+
+    if (root_is_CF_drive) return;
+
+    // Try to get a lock, but don't block for getting lock
+    if (!IOLockTryLock(gDebugImageLock)) {
+        HIBLOG("IOOpenDebugDataFile: Failed to get lock\n");
+        return;
+    }
+
+    if (gDebugImageFileVars ||  !fname || !size) {
+        HIBLOG("IOOpenDebugDataFile: conditions failed\n");
+        goto exit;
+    }
+
+    padding = (PAGE_SIZE*2);  // allocate couple more pages for header and fileextents
+    err = IOPolledFileOpen(fname, size+padding, 32ULL*1024*1024*1024,
+                           NULL, 0,
+                           &gDebugImageFileVars, &imagePath, NULL, 0);
+
+    if ((kIOReturnSuccess == err) && imagePath)
+    {
+        if ((gDebugImageFileVars->fileSize < (size+padding)) ||
+            (gDebugImageFileVars->fileExtents->getLength() > PAGE_SIZE)) {
+            // Can't use the file
+            IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
+            HIBLOG("IOOpenDebugDataFile: too many file extents\n");
+            goto exit;
+        }
+
+        // write extents for debug data usage in EFI
+        IOWriteExtentsToFile(gDebugImageFileVars, kIOHibernateHeaderOpenSignature);
+        IOSetBootImageNVRAM(imagePath);
+    }
+
+exit:
+    IOLockUnlock(gDebugImageLock);
+
+    if (imagePath) imagePath->release();
+    return;
+}
+
+void
+IOCloseDebugDataFile()
+{
+    IOSetBootImageNVRAM(0);
+
+    if (gDebugImageLock) {
+        IOLockLock(gDebugImageLock);
+        if (gDebugImageFileVars != 0) {
+            IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
+        }
+        IOLockUnlock(gDebugImageLock);
+    }
+
+
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
 DECLARE_IOHIBERNATEPROGRESSALPHA
 
 static void
@@ -1809,6 +1132,7 @@ IOHibernateSystemWake(void)
 {
     if (kFSOpened == gFSState)
     {
+       IOPolledFilePollersClose(gIOHibernateVars.fileVars, kIOPolledPostflightState);
        IOHibernateDone(&gIOHibernateVars);
     }
     else
@@ -1822,8 +1146,6 @@ IOHibernateSystemWake(void)
 static IOReturn
 IOHibernateDone(IOHibernateVars * vars)
 {
-    IORegistryEntry * next;
-
     hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
 
     if (vars->videoMapping)
@@ -1864,17 +1186,6 @@ IOHibernateDone(IOHibernateVars * vars)
         IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
     }
 
-    if (vars->fileVars)
-    {
-       if ((next = vars->fileVars->media)) do
-       {
-           next->removeProperty(kIOPolledInterfaceActiveKey);
-           next = next->getParentEntry(gIOServicePlane);
-       }
-       while (next);
-       IOPolledFileClose(vars->fileVars);
-    }
-
     // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
 
 #if defined(__i386__) || defined(__x86_64__)
@@ -1908,10 +1219,7 @@ IOHibernateDone(IOHibernateVars * vars)
        }
 #endif
 
-    if (vars->srcBuffer)
-       vars->srcBuffer->release();
-    if (vars->ioBuffer)
-       vars->ioBuffer->release();
+    if (vars->srcBuffer) vars->srcBuffer->release();
     bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
     if (vars->handoffBuffer)
     {
@@ -1956,8 +1264,6 @@ IOHibernateDone(IOHibernateVars * vars)
        }
        vars->handoffBuffer->release();
     }
-    if (vars->fileExtents)
-       vars->fileExtents->release();
 
     bzero(vars, sizeof(*vars));
 
@@ -1966,54 +1272,63 @@ IOHibernateDone(IOHibernateVars * vars)
     return (kIOReturnSuccess);
 }
 
-IOReturn
-IOHibernateSystemPostWake(void)
+void
+IOHibernateSystemPostWakeTrim(void * p1, void * p2)
 {
-    struct kern_direct_file_io_ref_t * fileRef;
-
-    if (kFSOpened == gFSState)
+    // invalidate & close the image file
+    if (p1) IOLockLock(gFSLock);
+    if (kFSTrimDelay == gFSState)
     {
-       // invalidate & close the image file
-       gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
-       if ((fileRef = gIOHibernateFileRef))
-       {
-           gIOHibernateFileRef = 0;
-           IOSleep(TRIM_DELAY);
-           kern_close_file_for_direct_io(fileRef,
+       IOPolledFileIOVars * vars = &gFileVars;
+       IOPolledFileClose(&vars,
 #if DISABLE_TRIM
-                                      0, 0, 0, 0, 0);
+                         0, NULL, 0, 0, 0);
 #else
-                                      0, (caddr_t) gIOHibernateCurrentHeader, 
-                                      sizeof(IOHibernateImageHeader),
-                                      0,
-                                      gIOHibernateCurrentHeader->imageSize);
+                         0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader),
+                         sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize);
 #endif
-       }
-       gFSState = kFSIdle;
+        gFSState = kFSIdle;
+    }
+    if (p1) IOLockUnlock(gFSLock);
+}
+
+IOReturn
+IOHibernateSystemPostWake(void)
+{
+    gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
+    IOLockLock(gFSLock);
+    if (kFSTrimDelay == gFSState) IOHibernateSystemPostWakeTrim(NULL, NULL);
+    else if (kFSOpened != gFSState) gFSState = kFSIdle;
+    else
+    {
+       AbsoluteTime deadline;
+
+        gFSState = kFSTrimDelay;
+       clock_interval_to_deadline(TRIM_DELAY, kMillisecondScale, &deadline );
+       thread_call_enter1_delayed(gIOHibernateTrimCalloutEntry, NULL, deadline);
     }
+    IOLockUnlock(gFSLock);
+
+    // IOCloseDebugDataFile() calls IOSetBootImageNVRAM() unconditionally
+    IOCloseDebugDataFile( );
     return (kIOReturnSuccess);
 }
 
-bool IOHibernateWasScreenLocked(void)
+uint32_t IOHibernateWasScreenLocked(void)
 {
-    bool ret = false;
+    uint32_t ret = 0;
     if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry)
     {
        OSData *
        data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
-       if (data) switch (*((uint32_t *)data->getBytesNoCopy()))
+       if (data)
        {
-           case kIOScreenLockLocked:
-           case kIOScreenLockFileVaultDialog:
-               ret = true;
-               break;
-           case kIOScreenLockNoLock:
-           case kIOScreenLockUnlocked:
-           default:
-               ret = false;
-               break;
-       }
+           ret = ((uint32_t *)data->getBytesNoCopy())[0];
+           gIOChosenEntry->setProperty(kIOBooterScreenLockStateKey, data);
+        }
     }
+    else gIOChosenEntry->removeProperty(kIOBooterScreenLockStateKey);
+
     return (ret);
 }
 
@@ -2030,25 +1345,34 @@ SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
                &gIOHibernateMode, 0, "");
 SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
                CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
-               gIOHibernateStats, hibernate_statistics_t, "");
+               &_hibernateStats, hibernate_statistics_t, "");
 
 SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
                CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
-               &gIOHibernateStats->graphicsReadyTime, 0, "");
+               &_hibernateStats.graphicsReadyTime, 0, "");
 SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
                CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
-               &gIOHibernateStats->wakeNotificationTime, 0, "");
+               &_hibernateStats.wakeNotificationTime, 0, "");
 SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
                CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
-               &gIOHibernateStats->lockScreenReadyTime, 0, "");
+               &_hibernateStats.lockScreenReadyTime, 0, "");
 SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
                CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
-               &gIOHibernateStats->hidReadyTime, 0, "");
+               &_hibernateStats.hidReadyTime, 0, "");
 
 
 void
 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
 {
+    gIOHibernateBootImageKey = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
+
+#if defined(__i386__) || defined(__x86_64__)
+    gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
+    gIOHibernateBoot0082Key     = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
+    gIOHibernateBootNextKey     = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
+    gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
+#endif /* defined(__i386__) || defined(__x86_64__) */
+
     OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
     if (data)
     {
@@ -2070,58 +1394,27 @@ IOHibernateSystemInit(IOPMrootDomain * rootDomain)
     sysctl_register_oid(&sysctl__kern_hibernatelockscreenready);
     sysctl_register_oid(&sysctl__kern_hibernatehidready);
 
-    gFSLock = IOLockAlloc();
-}
-
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+    gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
 
-static void
-hibernate_setup_for_wake(void)
-{
+    gFSLock = IOLockAlloc();
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
-#define C_ASSERT(e) typedef char    __C_ASSERT__[(e) ? 1 : -1]
-
-static bool
-no_encrypt_page(vm_offset_t ppnum)
-{
-    if (pmap_is_noencrypt((ppnum_t)ppnum) == TRUE)
-    {
-        return true;
-    }
-    return false;
-}
-
-static void
-hibernate_pal_callback(void *vars_arg, vm_offset_t addr)
-{
-       IOHibernateVars *vars = (IOHibernateVars *)vars_arg;
-       /* Make sure it's not in either of the save lists */
-       hibernate_set_page_state(vars->page_list, vars->page_list_wired, atop_64(addr), 1, kIOHibernatePageStateFree);
-
-       /* Set it in the bitmap of pages owned by the PAL */
-       hibernate_page_bitset(vars->page_list_pal, TRUE, atop_64(addr));
-}
-
-static struct hibernate_cryptvars_t *local_cryptvars;
-
-extern "C" int
-hibernate_pal_write(void *buffer, size_t size)
+static IOReturn 
+IOHibernatePolledFileWrite(IOPolledFileIOVars * vars,
+                          const uint8_t * bytes, IOByteCount size,
+                          IOPolledFileCryptVars * cryptvars)
 {
-    IOHibernateVars * vars  = &gIOHibernateVars;
+    IOReturn err;
 
-       IOReturn err = IOPolledFileWrite(vars->fileVars, (const uint8_t *)buffer, size, local_cryptvars);
-       if (kIOReturnSuccess != err) {
-               kprintf("epic hibernate fail! %d\n", err);
-               return err;
-       }
+    err = IOPolledFileWrite(vars, bytes, size, cryptvars);
+    if ((kIOReturnSuccess == err) && hibernate_should_abort()) err = kIOReturnAborted;
 
-       return 0;
+    return (err);
 }
 
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 extern "C" uint32_t
 hibernate_write_image(void)
@@ -2130,7 +1423,7 @@ hibernate_write_image(void)
     IOHibernateVars *        vars  = &gIOHibernateVars;
     IOPolledFileExtent *     fileExtents;
 
-    C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
+    _static_assert_1_arg(sizeof(IOHibernateImageHeader) == 512);
 
     uint32_t    pageCount, pagesDone;
     IOReturn     err;
@@ -2140,8 +1433,7 @@ hibernate_write_image(void)
     uint8_t *   data;
     uint8_t *   compressed;
     uint8_t *   scratch;
-    void *       zerosCompressed;
-    IOByteCount  pageCompressedSize, zerosCompressedLen;
+    IOByteCount  pageCompressedSize;
     uint64_t    compressedSize, uncompressedSize;
     uint64_t    image1Size = 0;
     uint32_t    bitmap_size;
@@ -2153,6 +1445,8 @@ hibernate_write_image(void)
     uint32_t    pageAndCount[2];
     addr64_t     phys64;
     IOByteCount  segLen;
+    uintptr_t    hibernateBase;
+    uintptr_t    hibernateEnd;
 
     AbsoluteTime startTime, endTime;
     AbsoluteTime allTime, compTime;
@@ -2165,18 +1459,21 @@ hibernate_write_image(void)
     uint32_t    wiredPagesEncrypted;
     uint32_t    dirtyPagesEncrypted;
     uint32_t    wiredPagesClear;
-    uint32_t    zeroPageCount;
+    uint32_t    svPageCount;
+    uint32_t    zvPageCount;
 
-    hibernate_cryptvars_t _cryptvars;
-    hibernate_cryptvars_t * cryptvars = 0;
+    IOPolledFileCryptVars _cryptvars;
+    IOPolledFileCryptVars * cryptvars = 0;
 
     wiredPagesEncrypted = 0;
     dirtyPagesEncrypted = 0;
     wiredPagesClear     = 0;
-    zeroPageCount       = 0;
+    svPageCount         = 0;
+    zvPageCount         = 0;
 
-    if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
-        return (false /* sleep */ );
+    if (!vars->fileVars
+    || !vars->fileVars->pollers
+    || !(kIOHibernateModeOn & gIOHibernateMode))      return (kIOHibernatePostWriteSleep);
 
     if (kIOHibernateModeSleep & gIOHibernateMode)
        kdebug_enable = save_kdebug_enable;
@@ -2186,8 +1483,6 @@ hibernate_write_image(void)
 
     restore1Sum = sum1 = sum2 = 0;
 
-    hibernate_pal_prepare();
-
 #if CRYPTO
     // encryption data. "iv" is the "initial vector".
     if (kIOHibernateModeEncrypt & gIOHibernateMode)
@@ -2197,7 +1492,7 @@ hibernate_write_image(void)
              0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
     
         cryptvars = &gIOHibernateCryptWakeContext;
-        bzero(cryptvars, sizeof(hibernate_cryptvars_t));
+        bzero(cryptvars, sizeof(IOPolledFileCryptVars));
         aes_encrypt_key(vars->cryptKey,
                         kIOHibernateAESKeySize,
                         &cryptvars->ctx.encrypt);
@@ -2206,7 +1501,7 @@ hibernate_write_image(void)
                         &cryptvars->ctx.decrypt);
 
         cryptvars = &_cryptvars;
-        bzero(cryptvars, sizeof(hibernate_cryptvars_t));
+        bzero(cryptvars, sizeof(IOPolledFileCryptVars));
         for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++)
             vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
         bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
@@ -2217,13 +1512,9 @@ hibernate_write_image(void)
         bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
         bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
         bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
-
-        local_cryptvars = cryptvars;
     }
 #endif /* CRYPTO */
 
-    hibernate_setup_for_wake();
-
     hibernate_page_list_setall(vars->page_list,
                                vars->page_list_wired,
                                vars->page_list_pal,
@@ -2235,7 +1526,7 @@ hibernate_write_image(void)
 
     HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
 
-    fileExtents = (IOPolledFileExtent *) vars->fileExtents->getBytesNoCopy();
+    fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy();
 
 #if 0
     count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
@@ -2258,13 +1549,16 @@ hibernate_write_image(void)
     {
         compressedSize   = 0;
         uncompressedSize = 0;
-        zeroPageCount    = 0;
+        svPageCount      = 0;
+        zvPageCount      = 0;
 
         IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
     
         HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n", 
                 ml_get_interrupts_enabled());
-        err = IOHibernatePollerOpen(vars->fileVars, kIOPolledBeforeSleepState, vars->ioBuffer);
+        err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState,
+                // abortable if not low battery
+                !IOService::getPMRootDomain()->mustHibernate());
         HIBLOG("IOHibernatePollerOpen(%x)\n", err);
         pollerOpen = (kIOReturnSuccess == err);
         if (!pollerOpen)
@@ -2272,21 +1566,17 @@ hibernate_write_image(void)
     
         // copy file block extent list if larger than header
     
-        count = vars->fileExtents->getLength();
+        count = vars->fileVars->fileExtents->getLength();
         if (count > sizeof(header->fileExtentMap))
         {
             count -= sizeof(header->fileExtentMap);
-            err = IOPolledFileWrite(vars->fileVars,
+            err = IOHibernatePolledFileWrite(vars->fileVars,
                                     ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
             if (kIOReturnSuccess != err)
                 break;
         }
 
-        uintptr_t hibernateBase;
-        uintptr_t hibernateEnd;
-
         hibernateBase = HIB_BASE; /* Defined in PAL headers */
-
         hibernateEnd = (segHIBB + segSizeHIB);
 
         // copy out restore1 code
@@ -2327,11 +1617,11 @@ hibernate_write_image(void)
         count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
         if (count)
         {
-            err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
+            err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
             if (kIOReturnSuccess != err)
                 break;
         }
-        err = IOPolledFileWrite(vars->fileVars, 
+        err = IOHibernatePolledFileWrite(vars->fileVars, 
                                         (uint8_t *) 0,
                                         &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
                                         cryptvars);
@@ -2341,7 +1631,7 @@ hibernate_write_image(void)
         count = round_page(hibernateEnd) - ((uintptr_t) src);
         if (count)
         {
-            err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
+            err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
             if (kIOReturnSuccess != err)
                 break;
         }
@@ -2364,7 +1654,7 @@ hibernate_write_image(void)
                 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
                 pageAndCount[0] = atop_64(phys64);
                 pageAndCount[1] = atop_32(segLen);
-                err = IOPolledFileWrite(vars->fileVars, 
+                err = IOHibernatePolledFileWrite(vars->fileVars, 
                                         (const uint8_t *) &pageAndCount, sizeof(pageAndCount), 
                                         cryptvars);
                 if (kIOReturnSuccess != err)
@@ -2390,15 +1680,16 @@ hibernate_write_image(void)
                 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
                 sum1 += hibernate_sum_page(src + page, atop_64(phys64));
             }
-            err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
+            err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
             if (kIOReturnSuccess != err)
                 break;
         }
 
         // mark areas for no save
-    
+        IOMemoryDescriptor * ioBuffer;
+        ioBuffer = IOPolledFileGetIOBuffer(vars->fileVars);
         for (count = 0;
-            (phys64 = vars->ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+            (phys64 = ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
             count += segLen)
         {
             hibernate_set_page_state(vars->page_list, vars->page_list_wired, 
@@ -2421,7 +1712,7 @@ hibernate_write_image(void)
     
         bitmap_size = vars->page_list_wired->list_size;
         src = (uint8_t *) vars->page_list_wired;
-        err = IOPolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
+        err = IOHibernatePolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
         if (kIOReturnSuccess != err)
             break;
 
@@ -2458,20 +1749,10 @@ hibernate_write_image(void)
             pageCount -= atop_32(segLen);
         }
 
-               (void)hibernate_pal_callback;
-
         src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
        compressed = src + page_size;
         scratch    = compressed + page_size;
 
-       // compress a zero page
-       bzero(src, page_size);
-       zerosCompressed    = vars->handoffBuffer->getBytesNoCopy();
-       zerosCompressedLen = WKdm_compress_new((WK_word*) src,
-                                              (WK_word*) zerosCompressed, 
-                                              (WK_word*) scratch,
-                                              page_size - 4);
-
         pagesDone  = 0;
         lastBlob   = 0;
     
@@ -2489,6 +1770,9 @@ hibernate_write_image(void)
             kUnwiredEncrypt = kEncrypt
         };
 
+        bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features()));
+#define _pmap_is_noencrypt(x) (cpuAES ? false : pmap_is_noencrypt((x)))
+
         for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)
         {
            if (kUnwiredEncrypt == pageType)
@@ -2512,13 +1796,13 @@ hibernate_write_image(void)
                                                         &ppnum);
 //              kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
                 iterDone = !count;
-    
+
                 if (count && (kWired & pageType) && needEncrypt)
                 {
                     uint32_t checkIndex;
                     for (checkIndex = 0;
                             (checkIndex < count) 
-                                && (((kEncrypt & pageType) == 0) == no_encrypt_page(ppnum + checkIndex)); 
+                                && (((kEncrypt & pageType) == 0) == _pmap_is_noencrypt(ppnum + checkIndex));
                             checkIndex++)
                     {}
                     if (!checkIndex)
@@ -2541,7 +1825,7 @@ hibernate_write_image(void)
                 {
                     pageAndCount[0] = ppnum;
                     pageAndCount[1] = count;
-                    err = IOPolledFileWrite(vars->fileVars, 
+                    err = IOHibernatePolledFileWrite(vars->fileVars, 
                                             (const uint8_t *) &pageAndCount, sizeof(pageAndCount), 
                                             cryptvars);
                     if (kIOReturnSuccess != err)
@@ -2564,7 +1848,7 @@ hibernate_write_image(void)
                         sum2 += sum;
        
                     clock_get_uptime(&startTime);
-                    wkresult = WKdm_compress_new((WK_word*) src,
+                    wkresult = WKdm_compress_new((const WK_word*) src,
                                                 (WK_word*) compressed, 
                                                 (WK_word*) scratch,
                                                 page_size - 4);
@@ -2576,27 +1860,30 @@ hibernate_write_image(void)
                     compBytes += page_size;
                     pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
 
-                   if ((pageCompressedSize == zerosCompressedLen) 
-                    && !bcmp(compressed, zerosCompressed, zerosCompressedLen))
+                   if (pageCompressedSize == 0) 
                    {
-                       pageCompressedSize = 0;
-                       zeroPageCount++;
-                   }
-
-                    if (kIOHibernateModeEncrypt & gIOHibernateMode)
-                        pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
-
-                    if (pageCompressedSize != page_size)
-                        data = compressed;
-                    else
+                       pageCompressedSize = 4;
                         data = src;
+
+                       if (*(uint32_t *)src)
+                               svPageCount++;
+                       else
+                               zvPageCount++;
+                   }
+                   else 
+                   {
+                       if (pageCompressedSize != page_size)
+                           data = compressed;
+                       else
+                           data = src;
+                   }
     
                     tag = pageCompressedSize | kIOHibernateTagSignature;
-                    err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
+                    err = IOHibernatePolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
                     if (kIOReturnSuccess != err)
                         break;
     
-                    err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
+                    err = IOHibernatePolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
                     if (kIOReturnSuccess != err)
                         break;
     
@@ -2643,14 +1930,14 @@ hibernate_write_image(void)
             if (kWiredEncrypt != pageType)
             {
                 // end of image1/2 - fill to next block
-                err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
+                err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
                 if (kIOReturnSuccess != err)
                     break;
             }
             if (kWiredClear == pageType)
             {
                // enlarge wired image for test
-//              err = IOPolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
+//              err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
 
                 // end wired image
                 header->encryptStart = vars->fileVars->encryptStart;
@@ -2664,9 +1951,12 @@ hibernate_write_image(void)
         {
             if (kIOReturnOverrun == err)
             {
-               // update actual compression ratio on not enough space
+                // update actual compression ratio on not enough space (for retry)
                 gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
             }
+
+            // update partial amount written (for IOPolledFileClose cleanup/unmap)
+            header->imageSize = vars->fileVars->position;
             break;
         }
 
@@ -2685,7 +1975,7 @@ hibernate_write_image(void)
        header->compression     = (compressedSize << 8) / uncompressedSize;
        gIOHibernateCompression = header->compression;
     
-        count = vars->fileExtents->getLength();
+        count = vars->fileVars->fileExtents->getLength();
         if (count > sizeof(header->fileExtentMap))
         {
             header->fileExtentMapSize = count;
@@ -2699,17 +1989,12 @@ hibernate_write_image(void)
         header->deviceBlockSize = vars->fileVars->blockSize;
     
         IOPolledFileSeek(vars->fileVars, 0);
-        err = IOPolledFileWrite(vars->fileVars,
+        err = IOHibernatePolledFileWrite(vars->fileVars,
                                     (uint8_t *) header, sizeof(IOHibernateImageHeader), 
                                     cryptvars);
         if (kIOReturnSuccess != err)
             break;
-        err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
-        if (kIOReturnSuccess != err)
-            break;
-        err = IOHibernatePollerIODone(vars->fileVars, true);
-        if (kIOReturnSuccess != err)
-            break;
+        err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
     }
     while (false);
     
@@ -2740,14 +2025,11 @@ hibernate_write_image(void)
                uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
                sum1, sum2);
 
-    HIBLOG("zeroPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n", 
-             zeroPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
-
-    if (vars->fileVars->io)
-        (void) IOHibernatePollerIODone(vars->fileVars, false);
+    HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n", 
+          svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
 
     if (pollerOpen)
-        IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
+        IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted );
 
     if (vars->consoleMapping)
         ProgressUpdate(gIOHibernateGraphicsInfo, 
@@ -2804,12 +2086,12 @@ hibernate_machine_init(void)
     uint64_t     compBytes;
     uint32_t     lastProgressStamp = 0;
     uint32_t     progressStamp;
-    hibernate_cryptvars_t * cryptvars = 0;
+    IOPolledFileCryptVars * cryptvars = 0;
 
     IOHibernateVars * vars  = &gIOHibernateVars;
     bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
 
-    if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
+    if (!vars->fileVars || !vars->fileVars->pollers)
        return;
 
     sum = gIOHibernateCurrentHeader->actualImage1Sum;
@@ -2828,7 +2110,7 @@ hibernate_machine_init(void)
 #define t40ms(x)       (tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)
 #define tStat(x, y)    gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
     tStat(booterStart, booterStart);
-    gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart,
+    gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart;
     tStat(booterDuration0, booterTime0);
     tStat(booterDuration1, booterTime1);
     tStat(booterDuration2, booterTime2);
@@ -2884,7 +2166,10 @@ hibernate_machine_init(void)
                break;
 
            case kIOHibernateHandoffTypeGraphicsInfo:
-               bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
+               if (handoff->bytecount == sizeof(*gIOHibernateGraphicsInfo))
+               {
+                   bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
+               }
                break;
 
            case kIOHibernateHandoffTypeCryptVars:
@@ -2929,7 +2214,7 @@ hibernate_machine_init(void)
     if (cryptvars && !foundCryptData)
        panic("hibernate handoff");
 
-    HIBPRINT("video %x %d %d %d status %x\n",
+    HIBPRINT("video 0x%llx %d %d %d status %x\n",
            gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth, 
            gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus); 
 
@@ -2959,18 +2244,17 @@ hibernate_machine_init(void)
     AbsoluteTime_to_scalar(&compTime) = 0;
     compBytes = 0;
 
-    HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
-    err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0);
+    HIBLOG("IOPolledFilePollersOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
+    err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledAfterSleepState, false);
     clock_get_uptime(&startIOTime);
     endTime = startIOTime;
     SUB_ABSOLUTETIME(&endTime, &allTime);
     absolutetime_to_nanoseconds(endTime, &nsec);
-    HIBLOG("IOHibernatePollerOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
+    HIBLOG("IOPolledFilePollersOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
 
     IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
 
     // kick off the read ahead
-    vars->fileVars->io          = false;
     vars->fileVars->bufferHalf   = 0;
     vars->fileVars->bufferLimit  = 0;
     vars->fileVars->lastRead     = 0;
@@ -3020,30 +2304,39 @@ hibernate_machine_init(void)
                break;
            }
 
-           if (!compressedSize) bzero_phys(ptoa_64(ppnum), page_size);
-           else
+           err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
+           if (kIOReturnSuccess != err) break;
+
+           if (compressedSize < page_size)
            {
-               err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
-               if (kIOReturnSuccess != err) break;
-               if (compressedSize < page_size)
-               {
-                   decoOffset = page_size;
-                   clock_get_uptime(&startTime);
-                   WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, page_size);
-                   clock_get_uptime(&endTime);
-                   ADD_ABSOLUTETIME(&compTime, &endTime);
-                   SUB_ABSOLUTETIME(&compTime, &startTime);
-                   compBytes += page_size;
+               decoOffset = page_size;
+               clock_get_uptime(&startTime);
+
+               if (compressedSize == 4) {
+                   int i;
+                   uint32_t *s, *d;
+                       
+                   s = (uint32_t *)src;
+                   d = (uint32_t *)(uintptr_t)compressed;
+
+                   for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++)
+                       *d++ = *s;
                }
-               else decoOffset = 0;
+               else 
+                   WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, compressedSize);
+               clock_get_uptime(&endTime);
+               ADD_ABSOLUTETIME(&compTime, &endTime);
+               SUB_ABSOLUTETIME(&compTime, &startTime);
+               compBytes += page_size;
+           }
+           else decoOffset = 0;
 
-               sum += hibernate_sum_page((src + decoOffset), ppnum);
-               err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
-               if (err)
-               {
+           sum += hibernate_sum_page((src + decoOffset), ppnum);
+           err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
+           if (err)
+           {
                    HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
                    break;
-               }
            }
 
            ppnum++;
@@ -3074,12 +2367,9 @@ hibernate_machine_init(void)
     gIOHibernateCurrentHeader->actualImage2Sum = sum;
     gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
 
-    if (vars->fileVars->io)
-        (void) IOHibernatePollerIODone(vars->fileVars, false);
-
     clock_get_uptime(&endIOTime);
 
-    err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
+    err = IOPolledFilePollersClose(vars->fileVars, kIOPolledAfterSleepState);
 
     clock_get_uptime(&endTime);
 
@@ -3128,7 +2418,6 @@ void IOHibernateSetWakeCapabilities(uint32_t capability)
        {
                vm_compressor_do_warmup();
        }
-
     }
 }