xnu-1699.22.81.tar.gz

[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp

diff --git a/iokit/Kernel/IOHibernateIO.cpp b/iokit/Kernel/IOHibernateIO.cpp
index d38300ad5fe092afcc1da548f294e9c116ac7f77..a4d7dbb4d08e62029d549932f4acb54ff7c51f78 100644 (file)
--- a/iokit/Kernel/IOHibernateIO.cpp
+++ b/iokit/Kernel/IOHibernateIO.cpp
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2008 Apple Computer, Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 


@@ -151,6 +151,7 @@ to restrict I/O ops.
 #include <IOKit/pwr_mgt/IOPowerConnection.h>
 #include "IOPMPowerStateQueue.h"
 #include <IOKit/IOBufferMemoryDescriptor.h>
 #include <IOKit/pwr_mgt/IOPowerConnection.h>
 #include "IOPMPowerStateQueue.h"
 #include <IOKit/IOBufferMemoryDescriptor.h>

+#include <IOKit/AppleKeyStoreInterface.h>

 #include <crypto/aes.h>
 
 #include <sys/uio.h>
 #include <crypto/aes.h>
 
 #include <sys/uio.h>


@@ -158,37 +159,24 @@ to restrict I/O ops.
 #include <sys/stat.h>
 #include <sys/fcntl.h>                       // (FWRITE, ...)
 #include <sys/sysctl.h>
 #include <sys/stat.h>
 #include <sys/fcntl.h>                       // (FWRITE, ...)
 #include <sys/sysctl.h>

+#include <sys/kdebug.h>

 
 #include <IOKit/IOHibernatePrivate.h>
 #include <IOKit/IOPolledInterface.h>
 #include <IOKit/IONVRAM.h>
 #include "IOHibernateInternal.h"
 
 #include <IOKit/IOHibernatePrivate.h>
 #include <IOKit/IOPolledInterface.h>
 #include <IOKit/IONVRAM.h>
 #include "IOHibernateInternal.h"

-#include "WKdm.h"
+#include <libkern/WKdm.h>

 #include "IOKitKernelInternal.h"
 #include "IOKitKernelInternal.h"

+#include <pexpert/device_tree.h>

 
 

-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+#include <machine/pal_routines.h>
+#include <machine/pal_hibernate.h>

 
 

-OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
-
-OSMetaClassDefineReservedUnused(IOPolledInterface, 0);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
+extern "C" addr64_t            kvtophys(vm_offset_t va);

 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 

+extern unsigned int            save_kdebug_enable;

 extern uint32_t                gIOHibernateState;
 uint32_t                       gIOHibernateMode;
 static char                    gIOHibernateBootSignature[256+1];
 extern uint32_t                gIOHibernateState;
 uint32_t                       gIOHibernateMode;
 static char                    gIOHibernateBootSignature[256+1];


@@ -198,14 +186,22 @@ uint32_t                  gIOHibernateFreeTime  = 0*1000; // max time to spend freeing pages (m
 
 static IODTNVRAM *             gIOOptionsEntry;
 static IORegistryEntry *       gIOChosenEntry;
 
 static IODTNVRAM *             gIOOptionsEntry;
 static IORegistryEntry *       gIOChosenEntry;

-#ifdef __i386__
+#if defined(__i386__) || defined(__x86_64__)

 static const OSSymbol *         gIOCreateEFIDevicePathSymbol;
 static const OSSymbol *         gIOCreateEFIDevicePathSymbol;

+static const OSSymbol *        gIOHibernateRTCVariablesKey;
+static const OSSymbol *         gIOHibernateBoot0082Key;
+static const OSSymbol *         gIOHibernateBootNextKey;
+static OSData *                        gIOHibernateBoot0082Data;
+static OSData *                        gIOHibernateBootNextData;
+static OSObject *              gIOHibernateBootNextSave;

 #endif
 
 static IOPolledFileIOVars                gFileVars;
 static IOHibernateVars                   gIOHibernateVars;
 static struct kern_direct_file_io_ref_t * gIOHibernateFileRef;
 static hibernate_cryptvars_t             gIOHibernateCryptWakeContext;
 #endif
 
 static IOPolledFileIOVars                gFileVars;
 static IOHibernateVars                   gIOHibernateVars;
 static struct kern_direct_file_io_ref_t * gIOHibernateFileRef;
 static hibernate_cryptvars_t             gIOHibernateCryptWakeContext;

+static hibernate_graphics_t              _hibernateGraphics;
+static hibernate_graphics_t *            gIOHibernateGraphicsInfo = &_hibernateGraphics;

 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 


@@ -243,7 +239,7 @@ IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
         addr64_t    dstAddr64;
         IOByteCount dstLen;
 
         addr64_t    dstAddr64;
         IOByteCount dstLen;
 

-        dstAddr64 = md->getPhysicalSegment64(offset, &dstLen);
+        dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);

         if (!dstAddr64)
             break;
 
         if (!dstAddr64)
             break;
 


@@ -281,7 +277,7 @@ IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
         addr64_t    srcAddr64;
         IOByteCount dstLen;
 
         addr64_t    srcAddr64;
         IOByteCount dstLen;
 

-        srcAddr64 = md->getPhysicalSegment64(offset, &dstLen);
+        srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);

         if (!srcAddr64)
             break;
 
         if (!srcAddr64)
             break;
 


@@ -468,28 +464,45 @@ IOHibernatePollerIO(IOPolledFileIOVars * vars,
 }
 
 static IOReturn
 }
 
 static IOReturn

-IOHibernatePollerIODone(IOPolledFileIOVars * vars)
+IOHibernatePollerIODone(IOPolledFileIOVars * vars, bool abortable)

 {
 {

-    IOReturn            err = kIOReturnError;
-    int32_t            idx;
+    IOReturn            err = kIOReturnSuccess;
+    int32_t            idx = 0;

     IOPolledInterface * poller;
 
     while (-1 == vars->ioStatus)
     {
     IOPolledInterface * poller;
 
     while (-1 == vars->ioStatus)
     {

-        for (idx = 0;
-             (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
+        for (idx = 0; 
+           (poller = (IOPolledInterface *) vars->pollers->getObject(idx));

              idx++)
         {
              idx++)
         {

-            err = poller->checkForWork();
-            if (err)
-                HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
+           IOReturn newErr;
+            newErr = poller->checkForWork();
+           if ((newErr == kIOReturnAborted) && !abortable)
+               newErr = kIOReturnSuccess;
+           if (kIOReturnSuccess == err)
+               err = newErr;

         }
     }
 
         }
     }
 

-    if (kIOReturnSuccess != vars->ioStatus)
-        HIBLOG("IOPolledInterface::ioStatus 0x%x\n", vars->ioStatus);
+    if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort())
+    {
+        err = kIOReturnAborted;
+       HIBLOG("IOPolledInterface::checkForWork sw abort\n");
+    }
+
+    if (err)
+    {
+       HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
+    }
+    else 
+    {
+       err = vars->ioStatus;
+       if (kIOReturnSuccess != err)
+           HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
+    }

 
 

-    return (vars->ioStatus);
+    return (err);

 }
 
 IOReturn
 }
 
 IOReturn


@@ -537,19 +550,59 @@ file_extent_callback(void * ref, uint64_t start, uint64_t length)
     ctx->size += length;
 }
 
     ctx->size += length;
 }
 

+static IOService * 
+IOCopyMediaForDev(dev_t device)
+{
+    OSDictionary * matching;
+    OSNumber *     num;
+    OSIterator *   iter;
+    IOService *    result = 0;
+
+    matching = IOService::serviceMatching("IOMedia");
+    if (!matching)
+        return (0);
+    do
+    {
+        num = OSNumber::withNumber(major(device), 32);
+        if (!num)
+            break;
+        matching->setObject(kIOBSDMajorKey, num);
+        num->release();
+        num = OSNumber::withNumber(minor(device), 32);
+        if (!num)
+            break;
+        matching->setObject(kIOBSDMinorKey, num);
+        num->release();
+        if (!num)
+            break;
+        iter = IOService::getMatchingServices(matching);
+        if (iter)
+        {
+            result = (IOService *) iter->getNextObject();
+            result->retain();
+            iter->release();
+        }
+    }
+    while (false);
+    matching->release();
+
+    return (result);
+}
+

 IOReturn
 IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
                            IOPolledFileIOVars ** fileVars, OSData ** fileExtents,
 IOReturn
 IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
                            IOPolledFileIOVars ** fileVars, OSData ** fileExtents,

-                           OSData ** imagePath)
+                           OSData ** imagePath, uint8_t * volumeCryptKey)

 {
     IOReturn                   err = kIOReturnError;
     IOPolledFileIOVars *       vars;
     _OpenFileContext           ctx;
     OSData *                   extentsData;
     OSNumber *                 num;
 {
     IOReturn                   err = kIOReturnError;
     IOPolledFileIOVars *       vars;
     _OpenFileContext           ctx;
     OSData *                   extentsData;
     OSNumber *                 num;

-    IORegistryEntry *          part = 0;
-    OSDictionary *             matching;
-    OSIterator *               iter;
+    IOService *                 part = 0;
+    OSString *                  keyUUID = 0;
+    OSString *                  keyStoreUUID = 0;
+    dev_t                      block_dev;

     dev_t                      hibernate_image_dev;
     uint64_t                   maxiobytes;
 
     dev_t                      hibernate_image_dev;
     uint64_t                   maxiobytes;
 


@@ -572,16 +625,25 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
        ctx.size    = 0;
        vars->fileRef = kern_open_file_for_direct_io(filename, 
                                                    &file_extent_callback, &ctx, 
        ctx.size    = 0;
        vars->fileRef = kern_open_file_for_direct_io(filename, 
                                                    &file_extent_callback, &ctx, 

+                                                   &block_dev,

                                                    &hibernate_image_dev,
                                                     &vars->block0,
                                                    &hibernate_image_dev,
                                                     &vars->block0,

-                                                    &maxiobytes);
+                                                    &maxiobytes,
+                                                    &vars->flags, 
+                                                    0, (caddr_t) gIOHibernateCurrentHeader, 
+                                                    sizeof(IOHibernateImageHeader));

        if (!vars->fileRef)
        {
            err = kIOReturnNoSpace;
            break;
        }
        if (!vars->fileRef)
        {
            err = kIOReturnNoSpace;
            break;
        }

-       HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx\n", filename, ctx.size, 
-                    vars->block0, maxiobytes);
+       gIOHibernateFileRef = vars->fileRef;
+
+        if (kIOHibernateModeSSDInvert & gIOHibernateMode)
+            vars->flags ^= kIOHibernateOptionSSD;
+
+       HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx ssd %d\n", filename, ctx.size, 
+                    vars->block0, maxiobytes, kIOHibernateOptionSSD & vars->flags);

        if (ctx.size < 1*1024*1024)             // check against image size estimate!
        {
            err = kIOReturnNoSpace;
        if (ctx.size < 1*1024*1024)             // check against image size estimate!
        {
            err = kIOReturnNoSpace;


@@ -592,39 +654,52 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
             vars->bufferSize = maxiobytes;
     
        vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
             vars->bufferSize = maxiobytes;
     
        vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();

-    
-       matching = IOService::serviceMatching("IOMedia");
-       num = OSNumber::withNumber(major(hibernate_image_dev), 32);
-       matching->setObject(kIOBSDMajorKey, num);
-       num->release();
-       num = OSNumber::withNumber(minor(hibernate_image_dev), 32);
-       matching->setObject(kIOBSDMinorKey, num);
-       num->release();
-       iter = IOService::getMatchingServices(matching);
-       matching->release();
-       if (iter)
-       {
-           part = (IORegistryEntry *) iter->getNextObject();
-           part->retain();
-           iter->release();
-       }
-    
-       int minor, major;
+
+        part = IOCopyMediaForDev(block_dev);
+        if (!part)
+            break;
+
+        err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false, 
+                                         (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
+        if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID)
+        {
+//            IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
+            uuid_t                  volumeKeyUUID;
+            aks_volume_key_t        vek;
+            static IOService *      sKeyStore;
+            static const OSSymbol * sAKSGetKey;
+
+            if (!sAKSGetKey)
+                sAKSGetKey = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY);
+            if (!sKeyStore)
+                sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
+            if (sKeyStore)
+                err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
+            else
+                err = kIOReturnNoResources;
+            if (kIOReturnSuccess == err)    
+                err = sKeyStore->callPlatformFunction(sAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
+            if (kIOReturnSuccess != err)    
+                IOLog("volume key err 0x%x\n", err);
+            else
+            {
+                size_t bytes = (kIOHibernateAESKeySize / 8);
+                if (vek.key.keybytecount < bytes)
+                     bytes = vek.key.keybytecount;
+                bcopy(&vek.key.keybytes[0], volumeCryptKey, bytes);
+            }
+            bzero(&vek, sizeof(vek));
+        }
+        part->release();
+
+        part = IOCopyMediaForDev(hibernate_image_dev);
+        if (!part)
+            break;
+

        IORegistryEntry * next;
        IORegistryEntry * child;
        OSData * data;
 
        IORegistryEntry * next;
        IORegistryEntry * child;
        OSData * data;
 

-       num = (OSNumber *) part->getProperty(kIOBSDMajorKey);
-       if (!num)
-           break;
-       major = num->unsigned32BitValue();
-       num = (OSNumber *) part->getProperty(kIOBSDMinorKey);
-       if (!num)
-           break;
-       minor = num->unsigned32BitValue();
-
-       hibernate_image_dev = makedev(major, minor);
-

         vars->pollers = OSArray::withCapacity(4);
        if (!vars->pollers)
            break;
         vars->pollers = OSArray::withCapacity(4);
        if (!vars->pollers)
            break;


@@ -652,7 +727,7 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
                 && child->isParent(next, gIOServicePlane, true));
 
        HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
                 && child->isParent(next, gIOServicePlane, true));
 
        HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",

-                   major, minor, vars->blockSize, vars->pollers->getCount());
+                   major(hibernate_image_dev), minor(hibernate_image_dev), (long)vars->blockSize, vars->pollers->getCount());

        if (vars->pollers->getCount() < kIOHibernateMinPollersNeeded)
            continue;
 
        if (vars->pollers->getCount() < kIOHibernateMinPollersNeeded)
            continue;
 


@@ -671,18 +746,22 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
 
        if ((extentsData->getLength() >= sizeof(IOPolledFileExtent)))
        {
 
        if ((extentsData->getLength() >= sizeof(IOPolledFileExtent)))
        {

-           char str2[24];
+           char str2[24 + sizeof(uuid_string_t) + 2];

 
 

-#if __i386__
+#if defined(__i386__) || defined(__x86_64__)

            if (!gIOCreateEFIDevicePathSymbol)
                gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
 
            if (!gIOCreateEFIDevicePathSymbol)
                gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
 

-           sprintf(str2, "%qx", vars->extentMap[0]);
+            if (keyUUID)
+                snprintf(str2, sizeof(str2), "%qx:%s", 
+                                vars->extentMap[0].start, keyUUID->getCStringNoCopy());
+            else
+                snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);

 
            err = IOService::getPlatform()->callPlatformFunction(
                                                gIOCreateEFIDevicePathSymbol, false,
 
            err = IOService::getPlatform()->callPlatformFunction(
                                                gIOCreateEFIDevicePathSymbol, false,

-                                               (void *) part, (void *) str2, (void *) true,
-                                               (void *) &data);
+                                               (void *) part, (void *) str2,
+                                               (void *) (uintptr_t) true, (void *) &data);

 #else
            char str1[256];
            int len = sizeof(str1);
 #else
            char str1[256];
            int len = sizeof(str1);


@@ -691,7 +770,7 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
                err = kIOReturnNotFound;
            else
            {
                err = kIOReturnNotFound;
            else
            {

-               sprintf(str2, ",%qx", vars->extentMap[0]);
+               snprintf(str2, sizeof(str2), ",%qx", vars->extentMap[0].start);

                // (strip the plane name)
                char * tail = strchr(str1, ':');
                if (!tail)
                // (strip the plane name)
                char * tail = strchr(str1, ':');
                if (!tail)


@@ -712,7 +791,10 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
     {
         HIBLOG("error 0x%x opening hibernation file\n", err);
        if (vars->fileRef)
     {
         HIBLOG("error 0x%x opening hibernation file\n", err);
        if (vars->fileRef)

-           kern_close_file_for_direct_io(vars->fileRef);
+       {
+           kern_close_file_for_direct_io(vars->fileRef, 0, 0, 0);
+           gIOHibernateFileRef = vars->fileRef = NULL;
+       }

     }
 
     if (part)
     }
 
     if (part)


@@ -730,8 +812,6 @@ IOPolledFileClose( IOPolledFileIOVars * vars )
         vars->pollers->release();
     }
 
         vars->pollers->release();
     }
 

-    gIOHibernateFileRef = vars->fileRef;
-

     bzero(vars, sizeof(IOPolledFileIOVars));
 
     return (kIOReturnSuccess);
     bzero(vars, sizeof(IOPolledFileIOVars));
 
     return (kIOReturnSuccess);


@@ -810,29 +890,45 @@ IOPolledFileWrite(IOPolledFileIOVars * vars,
                                - vars->extentPosition + vars->currentExtent->start);
            uint32_t length = (vars->bufferOffset);
 
                                - vars->extentPosition + vars->currentExtent->start);
            uint32_t length = (vars->bufferOffset);
 

-            if (cryptvars && vars->encryptStart && (vars->position > vars->encryptStart))
+#if CRYPTO
+            if (cryptvars && vars->encryptStart
+                && (vars->position > vars->encryptStart)
+                && ((vars->position - length) < vars->encryptEnd))

             {
             {

+                AbsoluteTime startTime, endTime;
+

                 uint32_t encryptLen, encryptStart;
                 encryptLen = vars->position - vars->encryptStart;
                 if (encryptLen > length)
                     encryptLen = length;
                 encryptStart = length - encryptLen;
                 uint32_t encryptLen, encryptStart;
                 encryptLen = vars->position - vars->encryptStart;
                 if (encryptLen > length)
                     encryptLen = length;
                 encryptStart = length - encryptLen;

-                
+                if (vars->position > vars->encryptEnd)
+                    encryptLen -= (vars->position - vars->encryptEnd);
+
+                clock_get_uptime(&startTime);
+

                 // encrypt the buffer
                 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
                                 &cryptvars->aes_iv[0],
                                 encryptLen / AES_BLOCK_SIZE,
                                 vars->buffer + vars->bufferHalf + encryptStart,
                                 &cryptvars->ctx.encrypt);
                 // encrypt the buffer
                 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
                                 &cryptvars->aes_iv[0],
                                 encryptLen / AES_BLOCK_SIZE,
                                 vars->buffer + vars->bufferHalf + encryptStart,
                                 &cryptvars->ctx.encrypt);

+    
+                clock_get_uptime(&endTime);
+                ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
+                SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
+                vars->cryptBytes += encryptLen;
+

                 // save initial vector for following encrypts
                 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
                         &cryptvars->aes_iv[0],
                         AES_BLOCK_SIZE);
             }
                 // save initial vector for following encrypts
                 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
                         &cryptvars->aes_iv[0],
                         AES_BLOCK_SIZE);
             }

+#endif /* CRYPTO */

 
            if (vars->io)
             {
 
            if (vars->io)
             {

-               err = IOHibernatePollerIODone(vars);
+               err = IOHibernatePollerIODone(vars, true);

                 if (kIOReturnSuccess != err)
                     break;
             }
                 if (kIOReturnSuccess != err)
                     break;
             }


@@ -898,11 +994,11 @@ IOPolledFileRead(IOPolledFileIOVars * vars,
        vars->bufferOffset += copy;
 //     vars->position += copy;
 
        vars->bufferOffset += copy;
 //     vars->position += copy;
 

-       if (vars->bufferOffset == vars->bufferLimit)
+       if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd))

        {
            if (vars->io)
             {
        {
            if (vars->io)
             {

-               err = IOHibernatePollerIODone(vars);
+               err = IOHibernatePollerIODone(vars, false);

                 if (kIOReturnSuccess != err)
                     break;
             }
                 if (kIOReturnSuccess != err)
                     break;
             }


@@ -911,9 +1007,9 @@ IOPolledFileRead(IOPolledFileIOVars * vars,
 
 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
 
 
 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
 

-           vars->position += vars->lastRead;
+           vars->position        += vars->lastRead;

            vars->extentRemaining -= vars->lastRead;
            vars->extentRemaining -= vars->lastRead;

-           vars->bufferLimit = vars->lastRead;
+           vars->bufferLimit      = vars->lastRead;

 
            if (!vars->extentRemaining)
            {
 
            if (!vars->extentRemaining)
            {


@@ -927,39 +1023,56 @@ if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n",
                 }
            }
 
                 }
            }
 

-           if (vars->extentRemaining <= vars->bufferSize)
-               vars->lastRead = vars->extentRemaining;
-           else
-               vars->lastRead = vars->bufferSize;
-
+           uint64_t length;
+           uint64_t lastReadLength = vars->lastRead;

            uint64_t offset = (vars->position 
                                - vars->extentPosition + vars->currentExtent->start);
            uint64_t offset = (vars->position 
                                - vars->extentPosition + vars->currentExtent->start);

-           uint64_t length = (vars->lastRead);
+           if (vars->extentRemaining <= vars->bufferSize)
+               length = vars->extentRemaining;
+           else
+               length = vars->bufferSize;
+           if ((length + vars->position) > vars->readEnd)
+               length = vars->readEnd - vars->position;

 
 

+           vars->lastRead = length;
+           if (length)
+           {

 //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
 //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);

-
-           err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
-            if (kIOReturnSuccess != err)
-                break;
-           vars->io = true;
+               err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
+               if (kIOReturnSuccess != err)
+                   break;
+               vars->io = true;
+           }

 
            vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
            vars->bufferOffset = 0;
 
 
            vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
            vars->bufferOffset = 0;
 

+#if CRYPTO

             if (cryptvars)
             {
                 uint8_t thisVector[AES_BLOCK_SIZE];
             if (cryptvars)
             {
                 uint8_t thisVector[AES_BLOCK_SIZE];

+                AbsoluteTime startTime, endTime;
+

                 // save initial vector for following decrypts
                 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
                 // save initial vector for following decrypts
                 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);

-                bcopy(vars->buffer + vars->bufferHalf + vars->lastRead - AES_BLOCK_SIZE, 
+                bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE, 

                         &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
                         &cryptvars->aes_iv[0], AES_BLOCK_SIZE);

+

                 // decrypt the buffer
                 // decrypt the buffer

+                clock_get_uptime(&startTime);
+

                 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
                                 &thisVector[0],
                 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
                                 &thisVector[0],

-                                vars->lastRead / AES_BLOCK_SIZE,
+                                lastReadLength / AES_BLOCK_SIZE,

                                 vars->buffer + vars->bufferHalf,
                                 &cryptvars->ctx.decrypt);
                                 vars->buffer + vars->bufferHalf,
                                 &cryptvars->ctx.decrypt);

+
+                clock_get_uptime(&endTime);
+                ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
+                SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
+                vars->cryptBytes += lastReadLength;

             }
             }

+#endif /* CRYPTO */

        }
     }
     while (size);
        }
     }
     while (size);


@@ -976,7 +1089,7 @@ IOHibernateSystemSleep(void)
     OSData *   data;
     OSObject * obj;
     OSString * str;
     OSData *   data;
     OSObject * obj;
     OSString * str;

-    OSNumber * num;
+    bool       dsSSD;

 
     IOHibernateVars * vars  = &gIOHibernateVars;
 
 
     IOHibernateVars * vars  = &gIOHibernateVars;
 


@@ -986,33 +1099,24 @@ IOHibernateSystemSleep(void)
 
     gIOHibernateState = kIOHibernateStateInactive;
 
 
     gIOHibernateState = kIOHibernateStateInactive;
 

-    if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateModeKey)))
+    gIOHibernateDebugFlags = 0;
+    if (kIOLogHibernate & gIOKitDebug)
+       gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
+
+    if (IOService::getPMRootDomain()->getHibernateSettings(
+        &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime))

     {
     {

-       if ((num = OSDynamicCast(OSNumber, obj)))
-           gIOHibernateMode = num->unsigned32BitValue();

         if (kIOHibernateModeSleep & gIOHibernateMode)
             // default to discard clean for safe sleep
             gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive 
                                 | kIOHibernateModeDiscardCleanActive);
         if (kIOHibernateModeSleep & gIOHibernateMode)
             // default to discard clean for safe sleep
             gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive 
                                 | kIOHibernateModeDiscardCleanActive);

-
-       obj->release();
-    }
-    if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeRatioKey)))
-    {
-       if ((num = OSDynamicCast(OSNumber, obj)))
-           gIOHibernateFreeRatio = num->unsigned32BitValue();
-       obj->release();
-    }
-    if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeTimeKey)))
-    {
-       if ((num = OSDynamicCast(OSNumber, obj)))
-           gIOHibernateFreeTime = num->unsigned32BitValue();
-       obj->release();

     }
     }

+

     if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
     {
        if ((str = OSDynamicCast(OSString, obj)))
     if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
     {
        if ((str = OSDynamicCast(OSString, obj)))

-           strcpy(gIOHibernateFilename, str->getCStringNoCopy());
+           strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
+                           sizeof(gIOHibernateFilename));

        obj->release();
     }
 
        obj->release();
     }
 


@@ -1021,6 +1125,7 @@ IOHibernateSystemSleep(void)
 
     HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
 
 
     HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
 

+

     do
     {
         vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
     do
     {
         vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,


@@ -1028,47 +1133,80 @@ IOHibernateSystemSleep(void)
         vars->ioBuffer  = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, 
                                    2 * kDefaultIOSize, page_size);
 
         vars->ioBuffer  = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, 
                                    2 * kDefaultIOSize, page_size);
 

-        if (!vars->srcBuffer || !vars->ioBuffer)
+       vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, 
+                                   ptoa_64(gIOHibernateHandoffPageCount), page_size);
+
+        if (!vars->srcBuffer || !vars->ioBuffer || !vars->handoffBuffer)

         {
             err = kIOReturnNoMemory;
             break;
         }
 
         {
             err = kIOReturnNoMemory;
             break;
         }
 

+       // open & invalidate the image file
+       gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;

         err = IOPolledFileOpen(gIOHibernateFilename, vars->ioBuffer,
         err = IOPolledFileOpen(gIOHibernateFilename, vars->ioBuffer,

-                                &vars->fileVars, &vars->fileExtents, &data);
+                                &vars->fileVars, &vars->fileExtents, &data, 
+                                &vars->volumeCryptKey[0]);

         if (KERN_SUCCESS != err)
         {
            HIBLOG("IOPolledFileOpen(%x)\n", err);
             break;
         }
         if (KERN_SUCCESS != err)
         {
            HIBLOG("IOPolledFileOpen(%x)\n", err);
             break;
         }

-       if (vars->fileVars->fileRef)
-       {
-           // invalidate the image file
-           gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
-           int err = kern_write_file(vars->fileVars->fileRef, 0,
-                                       (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
-            if (KERN_SUCCESS != err)
-                HIBLOG("kern_write_file(%d)\n", err);
-       }

 
        bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
 
        bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));

+       gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
+        dsSSD = ((0 != (kIOHibernateOptionSSD & vars->fileVars->flags))
+                && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
+        if (dsSSD)
+        {
+            gIOHibernateCurrentHeader->options |= 
+                                                kIOHibernateOptionSSD
+                                              | kIOHibernateOptionColor;
+
+#if defined(__i386__) || defined(__x86_64__)
+            if (!uuid_is_null(vars->volumeCryptKey) &&
+                  (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey)))
+            {
+                uintptr_t smcVars[2];
+                smcVars[0] = sizeof(vars->volumeCryptKey);
+                smcVars[1] = (uintptr_t)(void *) &vars->volumeCryptKey[0];
+
+                IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
+                bzero(smcVars, sizeof(smcVars));
+            }
+#endif
+        }
+        else
+        {
+            gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
+        }

 
         boolean_t encryptedswap;
 
         boolean_t encryptedswap;

+        AbsoluteTime startTime, endTime;
+        uint64_t nsec;
+
+        clock_get_uptime(&startTime);

         err = hibernate_setup(gIOHibernateCurrentHeader, 
                                 gIOHibernateFreeRatio, gIOHibernateFreeTime,
         err = hibernate_setup(gIOHibernateCurrentHeader, 
                                 gIOHibernateFreeRatio, gIOHibernateFreeTime,

-                                &vars->page_list, &vars->page_list_wired, &encryptedswap);
+                                dsSSD,
+                                &vars->page_list, &vars->page_list_wired, &vars->page_list_pal, &encryptedswap);
+        clock_get_uptime(&endTime);
+        SUB_ABSOLUTETIME(&endTime, &startTime);
+        absolutetime_to_nanoseconds(endTime, &nsec);
+        HIBLOG("hibernate_setup(%d) took %qd ms\n", err, nsec / 1000000ULL);
+

         if (KERN_SUCCESS != err)
         if (KERN_SUCCESS != err)

-        {
-           HIBLOG("hibernate_setup(%d)\n", err);

             break;
             break;

-        }

 
 

-        if (encryptedswap)
+        if (encryptedswap || !uuid_is_null(vars->volumeCryptKey))

             gIOHibernateMode ^= kIOHibernateModeEncrypt; 
 
             gIOHibernateMode ^= kIOHibernateModeEncrypt; 
 

-        vars->videoAllocSize = kVideoMapSize;
-        if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
-            vars->videoMapping = 0;
+        if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
+        {
+            vars->videoAllocSize = kVideoMapSize;
+            if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
+                vars->videoMapping = 0;
+        }

 
        // generate crypt keys
         for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
 
        // generate crypt keys
         for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)


@@ -1101,46 +1239,7 @@ IOHibernateSystemSleep(void)
             }
             data->release();
 
             }
             data->release();
 

-#ifdef __ppc__
-            size_t           len;
-            char              valueString[16];
-
-           vars->saveBootDevice = gIOOptionsEntry->copyProperty(kIOSelectedBootDeviceKey);
-            if (gIOChosenEntry)
-            {
-               OSData * bootDevice = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOBootPathKey));
-               if (bootDevice)
-               {
-                   sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
-                   OSString * str2 = OSString::withCStringNoCopy((const char *) bootDevice->getBytesNoCopy());
-                   if (sym && str2)
-                       gIOOptionsEntry->setProperty(sym, str2);
-                   if (sym)
-                       sym->release();
-                   if (str2)
-                       str2->release();
-               }
-                data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMemorySignatureKey));
-                if (data)
-                {
-                    vars->haveFastBoot = true;
-
-                    len = sprintf(valueString, "0x%lx", *((UInt32 *)data->getBytesNoCopy()));
-                    data = OSData::withBytes(valueString, len + 1);
-                    sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
-                    if (sym && data)
-                        gIOOptionsEntry->setProperty(sym, data);
-                    if (sym)
-                        sym->release();
-                    if (data)
-                        data->release();
-                }
-                data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
-                if (data)
-                    gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
-            }
-#endif /* __ppc__ */
-#ifdef __i386__
+#if defined(__i386__) || defined(__x86_64__)

            struct AppleRTCHibernateVars
            {
                uint8_t     signature[4];
            struct AppleRTCHibernateVars
            {
                uint8_t     signature[4];


@@ -1179,8 +1278,26 @@ IOHibernateSystemSleep(void)
            }
            data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
            if (data)
            }
            data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
            if (data)

-           {
-               IOService::getPMRootDomain()->setProperty(kIOHibernateRTCVariablesKey, data);
+           { 
+               if (!gIOHibernateRTCVariablesKey)
+                   gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
+               if (gIOHibernateRTCVariablesKey)
+                   IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
+       
+               if( gIOOptionsEntry )
+               {
+                   if( gIOHibernateMode & kIOHibernateModeSwitch )
+                   {
+                       const OSSymbol *sym;
+                       sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSwitchVarsKey);
+                       if( sym )
+                       {
+                           gIOOptionsEntry->setProperty(sym, data); /* intentional insecure backup of rtc boot vars */
+                           sym->release();
+                       }
+                   }   
+               }
+

                data->release();
            }
             if (gIOChosenEntry)
                data->release();
            }
             if (gIOChosenEntry)


@@ -1188,8 +1305,49 @@ IOHibernateSystemSleep(void)
                 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
                 if (data)
                     gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
                 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
                 if (data)
                     gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());

+               {
+                   // set BootNext
+
+                   if (!gIOHibernateBoot0082Data)
+                   {
+                       data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
+                       if (data)
+                       {
+                           // AppleNVRAM_EFI_LOAD_OPTION
+                           struct {
+                               uint32_t Attributes;
+                               uint16_t FilePathLength;
+                               uint16_t Desc;
+                           } loadOptionHeader;
+                           loadOptionHeader.Attributes     = 1;
+                           loadOptionHeader.FilePathLength = data->getLength();
+                           loadOptionHeader.Desc           = 0;
+                           gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
+                           if (gIOHibernateBoot0082Data)
+                           {
+                               gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
+                               gIOHibernateBoot0082Data->appendBytes(data);
+                           }
+                       }
+                   }
+                   if (!gIOHibernateBoot0082Key)
+                       gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
+                   if (!gIOHibernateBootNextKey)
+                       gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
+                   if (!gIOHibernateBootNextData)
+                   {
+                       uint16_t bits = 0x0082;
+                       gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
+                   }
+                   if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData)
+                   {
+                       gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
+                       gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
+                       gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
+                   }
+               }

             }
             }

-#else /* !__i386__ */
+#else /* !i386 && !x86_64 */

             if (kIOHibernateModeEncrypt & gIOHibernateMode)
             {
                 data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
             if (kIOHibernateModeEncrypt & gIOHibernateMode)
             {
                 data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));


@@ -1200,7 +1358,7 @@ IOHibernateSystemSleep(void)
                     sym->release();
                 if (data)
                     data->release();
                     sym->release();
                 if (data)
                     data->release();

-                if (gIOHibernateBootSignature[0])
+                if (false && gIOHibernateBootSignature[0])

                 {
                     data = OSData::withCapacity(16);
                     sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
                 {
                     data = OSData::withCapacity(16);
                     sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);


@@ -1243,7 +1401,7 @@ IOHibernateSystemSleep(void)
                                             &newVolume, sizeof(newVolume));
                 }
             }
                                             &newVolume, sizeof(newVolume));
                 }
             }

-#endif /* !__i386__ */
+#endif /* !i386 && !x86_64 */

        }
        // --
 
        }
        // --
 


@@ -1257,27 +1415,175 @@ IOHibernateSystemSleep(void)
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 

+DECLARE_IOHIBERNATEPROGRESSALPHA
+
+static void
+ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
+{
+    uint32_t   rowBytes, pixelShift;
+    uint32_t   x, y;
+    int32_t    blob;
+    uint32_t   alpha, in, color, result;
+    uint8_t *  out;
+    uint32_t   saveindex[kIOHibernateProgressCount] = { 0 };
+
+    rowBytes = display->rowBytes;
+    pixelShift = display->depth >> 4;
+    if (pixelShift < 1) return;
+    
+    screen += ((display->width
+                - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
+        + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
+    
+    for (y = 0; y < kIOHibernateProgressHeight; y++)
+    {
+        out = screen + y * rowBytes;
+        for (blob = 0; blob < kIOHibernateProgressCount; blob++)
+        {
+            color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
+            for (x = 0; x < kIOHibernateProgressWidth; x++)
+            {
+                alpha  = gIOHibernateProgressAlpha[y][x];
+                result = color;
+                if (alpha)
+                {
+                    if (0xff != alpha)
+                    {
+                        if (1 == pixelShift)
+                        {
+                            in = *((uint16_t *)out) & 0x1f;    // 16
+                            in = (in << 3) | (in >> 2);
+                        }
+                        else
+                            in = *((uint32_t *)out) & 0xff;    // 32
+                        saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
+                        result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
+                    }
+                    if (1 == pixelShift)
+                    {
+                        result >>= 3;
+                        *((uint16_t *)out) = (result << 10) | (result << 5) | result;  // 16
+                    }
+                    else
+                        *((uint32_t *)out) = (result << 16) | (result << 8) | result;  // 32
+                }
+                out += (1 << pixelShift);
+            }
+            out += (kIOHibernateProgressSpacing << pixelShift);
+        }
+    }
+}
+
+
+static void
+ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
+{
+    uint32_t  rowBytes, pixelShift;
+    uint32_t  x, y;
+    int32_t   blob, lastBlob;
+    uint32_t  alpha, in, color, result;
+    uint8_t * out;
+    uint32_t  saveindex[kIOHibernateProgressCount] = { 0 };
+
+    pixelShift = display->depth >> 4;
+    if (pixelShift < 1)
+        return;
+
+    rowBytes = display->rowBytes;
+
+    screen += ((display->width 
+            - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
+                + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
+
+    lastBlob  = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
+
+    screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
+
+    for (y = 0; y < kIOHibernateProgressHeight; y++)
+    {
+        out = screen + y * rowBytes;
+        for (blob = firstBlob; blob <= lastBlob; blob++)
+        {
+            color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
+            for (x = 0; x < kIOHibernateProgressWidth; x++)
+            {
+                alpha  = gIOHibernateProgressAlpha[y][x];
+                result = color;
+                if (alpha)
+                {
+                    if (0xff != alpha)
+                    {
+                        in = display->progressSaveUnder[blob][saveindex[blob]++];
+                        result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
+                    }
+                    if (1 == pixelShift)
+                    {
+                        result >>= 3;
+                        *((uint16_t *)out) = (result << 10) | (result << 5) | result;  // 16
+                    }
+                    else
+                        *((uint32_t *)out) = (result << 16) | (result << 8) | result;  // 32
+                }
+                out += (1 << pixelShift);
+            }
+            out += (kIOHibernateProgressSpacing << pixelShift);
+        }
+    }
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+

 IOReturn
 IOHibernateSystemHasSlept(void)
 {
     IOHibernateVars * vars  = &gIOHibernateVars;
 IOReturn
 IOHibernateSystemHasSlept(void)
 {
     IOHibernateVars * vars  = &gIOHibernateVars;

+    OSObject        * obj;
+    OSData          * data;
+
+    obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
+    vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
+    if (obj && !vars->previewBuffer)
+       obj->release();
+
+    vars->consoleMapping = NULL;
+    if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
+    {
+       vars->previewBuffer->release();
+       vars->previewBuffer = 0;
+    }

 
 

-    if ((vars->previewData = OSDynamicCast(OSData, 
-            IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewBufferKey))))
+    if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
+        && vars->previewBuffer 
+        && (data = OSDynamicCast(OSData, 
+       IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))

     {
     {

-        vars->previewBuffer = IOMemoryDescriptor::withAddress(
-                                    (void *) vars->previewData->getBytesNoCopy(), 
-                                    vars->previewData->getLength(), 
-                                    kIODirectionInOut);
+       UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
+       HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);

 
 

-        if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
-        {
-            vars->previewBuffer->release();
-            vars->previewBuffer = 0;
-        }
-        if (!vars->previewBuffer)
-            vars->previewData = 0;
+       IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
+
+       if (kIOHibernatePreviewUpdates & flags)
+       {
+           PE_Video           consoleInfo;
+           hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
+
+           IOService::getPlatform()->getConsoleInfo(&consoleInfo);
+
+           graphicsInfo->width    = consoleInfo.v_width;
+           graphicsInfo->height   = consoleInfo.v_height;
+           graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
+           graphicsInfo->depth    = consoleInfo.v_depth;
+           vars->consoleMapping   = (uint8_t *) consoleInfo.v_baseAddr;
+
+           HIBPRINT("video %p %d %d %d\n",
+                       vars->consoleMapping, graphicsInfo->depth, 
+                       graphicsInfo->width, graphicsInfo->height);
+           if (vars->consoleMapping)
+                       ProgressInit(graphicsInfo, vars->consoleMapping,
+                                       &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
+       }

     }
     }

+

     if (gIOOptionsEntry)
         gIOOptionsEntry->sync();
 
     if (gIOOptionsEntry)
         gIOOptionsEntry->sync();
 


@@ -1286,6 +1592,38 @@ IOHibernateSystemHasSlept(void)
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 

+static DeviceTreeNode *
+MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry)
+{
+    DeviceTreeNodeProperty * prop;
+    DeviceTreeNode *         child;
+    IORegistryEntry *        childRegEntry;
+    const char *             nameProp;
+    unsigned int             propLen, idx;
+
+    prop = (DeviceTreeNodeProperty *) (entry + 1);
+    for (idx = 0; idx < entry->nProperties; idx++)
+    {
+       if (regEntry && (0 != strcmp("name", prop->name)))
+       {
+           regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
+//         HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
+       }
+       prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
+    }
+
+    child = (DeviceTreeNode *) prop;
+    for (idx = 0; idx < entry->nChildren; idx++)
+    {
+       if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen))
+           panic("no name");
+       childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
+//     HIBPRINT("%s == %p\n", nameProp, childRegEntry);
+       child = MergeDeviceTree(child, childRegEntry);
+    }
+    return (child);
+}
+

 IOReturn
 IOHibernateSystemWake(void)
 {
 IOReturn
 IOHibernateSystemWake(void)
 {


@@ -1300,7 +1638,7 @@ IOHibernateSystemWake(void)
             IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
         if (vars->videoAllocSize)
             // dealloc range
             IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
         if (vars->videoAllocSize)
             // dealloc range

-            kmem_free(kernel_map, trunc_page_32(vars->videoMapping), vars->videoAllocSize);
+            kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);

     }
 
     if (vars->previewBuffer)
     }
 
     if (vars->previewBuffer)


@@ -1309,77 +1647,112 @@ IOHibernateSystemWake(void)
         vars->previewBuffer = 0;
     }
 
         vars->previewBuffer = 0;
     }
 

-    if (vars->fileVars)
+    if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)

     {
     {

-       IOPolledFileClose(vars->fileVars);
+        IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey, 
+                                            gIOHibernateCurrentHeader->options, 32);

     }
     }

-
-    // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
-
-#ifdef __ppc__
-    OSData * data = OSData::withCapacity(4);
-    if (gIOOptionsEntry && data)
+    else

     {
     {

-        const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
-        if (sym)
-        {
-            gIOOptionsEntry->setProperty(sym, data);
-            sym->release();
-        }
-        sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
-        if (sym)
-        {
-           if (vars->saveBootDevice)
-           {
-               gIOOptionsEntry->setProperty(sym, vars->saveBootDevice);
-               vars->saveBootDevice->release();
-           }
-            sym->release();
-        }
-        sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
-        if (sym)
-        {
-            gIOOptionsEntry->setProperty(sym, data);
-            sym->release();
-        }
-        sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
-        if (sym)
-        {
-            gIOOptionsEntry->removeProperty(sym);
-            sym->release();
-        }
+        IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);

     }
     }

-    if (data)
-        data->release();

 
 

-    if (gIOOptionsEntry)
+    if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
+      && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus))

     {
     {

-       if (!vars->haveFastBoot)
-       {
-           // reset boot audio volume
-           IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
-           if (platform)
-               platform->writeXPRAM(kXPRamAudioVolume, 
-                                       &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume));
-       }
+        IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey, 
+                                        &gIOHibernateGraphicsInfo->gfxStatus,
+                                        sizeof(gIOHibernateGraphicsInfo->gfxStatus));
+    }
+    else
+    {
+        IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
+    }

 
 

-       // sync now to hardware if the booter has not
-       if (kIOHibernateStateInactive == gIOHibernateState)
-           gIOOptionsEntry->sync();
-       else
-           // just sync the variables in case a later panic syncs nvram (it won't sync variables)
-           gIOOptionsEntry->syncOFVariables();
+
+    if (vars->fileVars)
+    {
+       IOPolledFileClose(vars->fileVars);

     }
     }

-#endif

 
 

-#ifdef __i386__
-    IOService::getPMRootDomain()->removeProperty(kIOHibernateRTCVariablesKey);
+    // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
+
+#if defined(__i386__) || defined(__x86_64__)
+       IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
+       IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
+
+       /*
+        * Hibernate variable is written to NVRAM on platforms in which RtcRam
+        * is not backed by coin cell.  Remove Hibernate data from NVRAM.
+        */
+       if (gIOOptionsEntry) {
+
+           if (gIOHibernateRTCVariablesKey) {
+               if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
+                   gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
+               }
+           }
+
+           if (gIOHibernateBootNextKey)
+           {
+               if (gIOHibernateBootNextSave)
+               {
+                   gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
+                   gIOHibernateBootNextSave->release();
+                   gIOHibernateBootNextSave = NULL;
+               }
+               else
+                   gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
+           }
+           gIOOptionsEntry->sync();
+       }

 #endif
 
     if (vars->srcBuffer)
        vars->srcBuffer->release();
     if (vars->ioBuffer)
        vars->ioBuffer->release();
 #endif
 
     if (vars->srcBuffer)
        vars->srcBuffer->release();
     if (vars->ioBuffer)
        vars->ioBuffer->release();

+    bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
+    if (vars->handoffBuffer)
+    {
+       IOHibernateHandoff * handoff;
+       bool done = false;
+       for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
+            !done;
+            handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
+       {
+//         HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
+           uint8_t * data = &handoff->data[0];
+           switch (handoff->type)
+           {
+               case kIOHibernateHandoffTypeEnd:
+                   done = true;
+                   break;
+
+           case kIOHibernateHandoffTypeDeviceTree:
+                   MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
+                   break;
+    
+               case kIOHibernateHandoffTypeKeyStore:
+#if defined(__i386__) || defined(__x86_64__)
+                   {
+                       IOBufferMemoryDescriptor *
+                       md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
+                       if (md)
+                       {
+                           IOSetKeyStoreData(md);
+                       }
+                   }
+#endif
+                   break;
+    
+               default:
+                   done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
+                   break;
+           }    
+       }
+       vars->handoffBuffer->release();
+    }

     if (vars->fileExtents)
        vars->fileExtents->release();
 
     if (vars->fileExtents)
        vars->fileExtents->release();
 


@@ -1395,14 +1768,11 @@ IOHibernateSystemPostWake(void)
 {
     if (gIOHibernateFileRef)
     {
 {
     if (gIOHibernateFileRef)
     {

-       // invalidate the image file
+       // invalidate & close the image file

        gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
        gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;

-       int err = kern_write_file(gIOHibernateFileRef, 0,
-                                   (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
-       if (KERN_SUCCESS != err)
-           HIBLOG("kern_write_file(%d)\n", err);
-
-       kern_close_file_for_direct_io(gIOHibernateFileRef);
+       kern_close_file_for_direct_io(gIOHibernateFileRef,
+                                      0, (caddr_t) gIOHibernateCurrentHeader, 
+                                      sizeof(IOHibernateImageHeader));

         gIOHibernateFileRef = 0;
     }
     return (kIOReturnSuccess);
         gIOHibernateFileRef = 0;
     }
     return (kIOReturnSuccess);


@@ -1410,6 +1780,16 @@ IOHibernateSystemPostWake(void)
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 

+SYSCTL_STRING(_kern, OID_AUTO, hibernatefile, 
+               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
+               gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
+SYSCTL_STRING(_kern, OID_AUTO, bootsignature, 
+               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
+               gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatemode, 
+               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
+               &gIOHibernateMode, 0, "");
+

 void
 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
 {
 void
 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
 {


@@ -1420,43 +1800,71 @@ IOHibernateSystemInit(IOPMrootDomain * rootDomain)
        data->release();
     }
 
        data->release();
     }
 

-    if (PE_parse_boot_arg("hfile", gIOHibernateFilename))
+    if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))

        gIOHibernateMode = kIOHibernateModeOn;
     else
        gIOHibernateFilename[0] = 0;
 
        gIOHibernateMode = kIOHibernateModeOn;
     else
        gIOHibernateFilename[0] = 0;
 

-    static SYSCTL_STRING(_kern, OID_AUTO, hibernatefile, 
-                               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, 
-                               gIOHibernateFilename, sizeof(gIOHibernateFilename), "");

     sysctl_register_oid(&sysctl__kern_hibernatefile);
     sysctl_register_oid(&sysctl__kern_hibernatefile);

-
-    static SYSCTL_STRING(_kern, OID_AUTO, bootsignature, 
-                               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, 
-                               gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");

     sysctl_register_oid(&sysctl__kern_bootsignature);
     sysctl_register_oid(&sysctl__kern_bootsignature);

-
-    static SYSCTL_UINT(_kern, OID_AUTO, hibernatemode, 
-                               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, 
-                               &gIOHibernateMode, 0, "");

     sysctl_register_oid(&sysctl__kern_hibernatemode);
 }
 
     sysctl_register_oid(&sysctl__kern_hibernatemode);
 }
 

+

 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 static void
 hibernate_setup_for_wake(void)
 {
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 static void
 hibernate_setup_for_wake(void)
 {

-#if __ppc__
-    // go slow (state needed for wake)
-    ml_set_processor_speed(1);
-#endif

 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 #define C_ASSERT(e) typedef char    __C_ASSERT__[(e) ? 1 : -1]
 
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 #define C_ASSERT(e) typedef char    __C_ASSERT__[(e) ? 1 : -1]
 

-extern "C" boolean_t
+static bool
+no_encrypt_page(vm_offset_t ppnum)
+{
+    if (pmap_is_noencrypt((ppnum_t)ppnum) == TRUE)
+    {
+        return true;
+    }
+    return false;
+}
+
+uint32_t       wired_pages_encrypted = 0;
+uint32_t       dirty_pages_encrypted = 0;
+uint32_t       wired_pages_clear = 0;
+
+static void
+hibernate_pal_callback(void *vars_arg, vm_offset_t addr)
+{
+       IOHibernateVars *vars = (IOHibernateVars *)vars_arg;
+       /* Make sure it's not in either of the save lists */
+       hibernate_set_page_state(vars->page_list, vars->page_list_wired, atop_64(addr), 1, kIOHibernatePageStateFree);
+
+       /* Set it in the bitmap of pages owned by the PAL */
+       hibernate_page_bitset(vars->page_list_pal, TRUE, atop_64(addr));
+}
+
+static struct hibernate_cryptvars_t *local_cryptvars;
+
+extern "C" int
+hibernate_pal_write(void *buffer, size_t size)
+{
+    IOHibernateVars * vars  = &gIOHibernateVars;
+
+       IOReturn err = IOPolledFileWrite(vars->fileVars, (const uint8_t *)buffer, size, local_cryptvars);
+       if (kIOReturnSuccess != err) {
+               kprintf("epic hibernate fail! %d\n", err);
+               return err;
+       }
+
+       return 0;
+}
+
+
+extern "C" uint32_t

 hibernate_write_image(void)
 {
     IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
 hibernate_write_image(void)
 {
     IOHibernateImageHeader * header = gIOHibernateCurrentHeader;


@@ -1467,34 +1875,51 @@ hibernate_write_image(void)
 
     uint32_t    pageCount, pagesDone;
     IOReturn     err;
 
     uint32_t    pageCount, pagesDone;
     IOReturn     err;

-    vm_offset_t  ppnum;
-    IOItemCount  page, count;
+    vm_offset_t  ppnum, page;
+    IOItemCount  count;

     uint8_t *   src;
     uint8_t *   data;
     IOByteCount  pageCompressedSize;
     uint64_t    compressedSize, uncompressedSize;
     uint64_t    image1Size = 0;
     uint32_t    bitmap_size;
     uint8_t *   src;
     uint8_t *   data;
     IOByteCount  pageCompressedSize;
     uint64_t    compressedSize, uncompressedSize;
     uint64_t    image1Size = 0;
     uint32_t    bitmap_size;

-    bool        iterDone, pollerOpen, needEncryptStart;
+    bool        iterDone, pollerOpen, needEncrypt;

     uint32_t    restore1Sum, sum, sum1, sum2;
     uint32_t    tag;
     uint32_t    pageType;
     uint32_t    pageAndCount[2];
     uint32_t    restore1Sum, sum, sum1, sum2;
     uint32_t    tag;
     uint32_t    pageType;
     uint32_t    pageAndCount[2];

+    addr64_t     phys64;
+    IOByteCount  segLen;

 
     AbsoluteTime startTime, endTime;
 
     AbsoluteTime startTime, endTime;

-    AbsoluteTime allTime, compTime, decoTime;
+    AbsoluteTime allTime, compTime;
+    uint64_t     compBytes;

     uint64_t     nsec;
     uint32_t     lastProgressStamp = 0;
     uint32_t     progressStamp;
     uint64_t     nsec;
     uint32_t     lastProgressStamp = 0;
     uint32_t     progressStamp;

+    uint32_t    blob, lastBlob = (uint32_t) -1L;

 
     hibernate_cryptvars_t _cryptvars;
     hibernate_cryptvars_t * cryptvars = 0;
 
 
     hibernate_cryptvars_t _cryptvars;
     hibernate_cryptvars_t * cryptvars = 0;
 

+    wired_pages_encrypted = 0;
+    dirty_pages_encrypted = 0;
+    wired_pages_clear = 0;
+

     if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
         return (false /* sleep */ );
 
     if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
         return (false /* sleep */ );
 

+    if (kIOHibernateModeSleep & gIOHibernateMode)
+       kdebug_enable = save_kdebug_enable;
+
+    KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START, 0, 0, 0, 0, 0);
+    IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
+

     restore1Sum = sum1 = sum2 = 0;
 
     restore1Sum = sum1 = sum2 = 0;
 

+    hibernate_pal_prepare();
+
+#if CRYPTO

     // encryption data. "iv" is the "initial vector".
     if (kIOHibernateModeEncrypt & gIOHibernateMode)
     {
     // encryption data. "iv" is the "initial vector".
     if (kIOHibernateModeEncrypt & gIOHibernateMode)
     {


@@ -1513,6 +1938,9 @@ hibernate_write_image(void)
 
         cryptvars = &_cryptvars;
         bzero(cryptvars, sizeof(hibernate_cryptvars_t));
 
         cryptvars = &_cryptvars;
         bzero(cryptvars, sizeof(hibernate_cryptvars_t));

+        for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++)
+            vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
+        bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));

         aes_encrypt_key(vars->wiredCryptKey,
                         kIOHibernateAESKeySize,
                         &cryptvars->ctx.encrypt);
         aes_encrypt_key(vars->wiredCryptKey,
                         kIOHibernateAESKeySize,
                         &cryptvars->ctx.encrypt);


@@ -1520,13 +1948,16 @@ hibernate_write_image(void)
         bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
         bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
         bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
         bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
         bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
         bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));

-        bzero(gIOHibernateCryptWakeVars, sizeof(hibernate_cryptwakevars_t));
+
+        local_cryptvars = cryptvars;

     }
     }

+#endif /* CRYPTO */

 
     hibernate_setup_for_wake();
 
     hibernate_page_list_setall(vars->page_list,
                                vars->page_list_wired,
 
     hibernate_setup_for_wake();
 
     hibernate_page_list_setall(vars->page_list,
                                vars->page_list_wired,

+                                                          vars->page_list_pal,

                                &pageCount);
 
     HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
                                &pageCount);
 
     HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);


@@ -1543,19 +1974,18 @@ hibernate_write_image(void)
     }
 #endif
 
     }
 #endif
 

-    needEncryptStart = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
-
+    needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));

     AbsoluteTime_to_scalar(&compTime) = 0;
     AbsoluteTime_to_scalar(&compTime) = 0;

-    AbsoluteTime_to_scalar(&decoTime) = 0;
+    compBytes = 0;

 
     clock_get_uptime(&allTime);
 
     clock_get_uptime(&allTime);

+    IOService::getPMRootDomain()->pmStatsRecordEvent( 
+                        kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);

 
     do 
     {
         compressedSize   = 0;
         uncompressedSize = 0;
 
     do 
     {
         compressedSize   = 0;
         uncompressedSize = 0;

-        iterDone         = false;
-        pageType         = 0;          // wired pages first

 
         IOPolledFileSeek(vars->fileVars, sizeof(IOHibernateImageHeader));
     
 
         IOPolledFileSeek(vars->fileVars, sizeof(IOHibernateImageHeader));
     


@@ -1579,31 +2009,49 @@ hibernate_write_image(void)
                 break;
         }
 
                 break;
         }
 

+        uintptr_t hibernateBase;
+        uintptr_t hibernateEnd;
+
+        hibernateBase = HIB_BASE; /* Defined in PAL headers */
+
+        hibernateEnd = (sectHIBB + sectSizeHIB);
+

         // copy out restore1 code
         // copy out restore1 code

-    
-        page = atop_32(sectHIBB);
-        count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
-        header->restore1CodePage = page;
+
+        for (count = 0;
+            (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+            count += segLen)
+        {
+           for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++)
+           {
+               gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
+           }
+        }
+
+        page = atop_32(kvtophys(hibernateBase));
+        count = atop_32(round_page(hibernateEnd) - hibernateBase);
+        header->restore1CodePhysPage = page;
+        header->restore1CodeVirt = hibernateBase;

         header->restore1PageCount = count;
         header->restore1PageCount = count;

-        header->restore1CodeOffset = ((uint32_t) &hibernate_machine_entrypoint)      - sectHIBB;
-        header->restore1StackOffset = ((uint32_t) &gIOHibernateRestoreStackEnd[0]) - 64 - sectHIBB;
+        header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint)      - hibernateBase;
+        header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;

 
         // sum __HIB sect, with zeros for the stack
 
         // sum __HIB sect, with zeros for the stack

-        src = (uint8_t *) trunc_page(sectHIBB);
+        src = (uint8_t *) trunc_page(hibernateBase);

         for (page = 0; page < count; page++)
         {
             if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
         for (page = 0; page < count; page++)
         {
             if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))

-                restore1Sum += hibernate_sum(src, page_size);
+                restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);

             else
             else

-                restore1Sum += 0x10000001;
+                restore1Sum += 0x00000000;

             src += page_size;
         }
         sum1 = restore1Sum;
     
         // write the __HIB sect, with zeros for the stack
 
             src += page_size;
         }
         sum1 = restore1Sum;
     
         // write the __HIB sect, with zeros for the stack
 

-        src = (uint8_t *) trunc_page(sectHIBB);
-        count = ((uint32_t) &gIOHibernateRestoreStack[0]) - trunc_page(sectHIBB);
+        src = (uint8_t *) trunc_page(hibernateBase);
+        count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);

         if (count)
         {
             err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
         if (count)
         {
             err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);


@@ -1617,7 +2065,7 @@ hibernate_write_image(void)
         if (kIOReturnSuccess != err)
             break;
         src = &gIOHibernateRestoreStackEnd[0];
         if (kIOReturnSuccess != err)
             break;
         src = &gIOHibernateRestoreStackEnd[0];

-        count = round_page(sectHIBB + sectSizeHIB) - ((uint32_t) src);
+        count = round_page(hibernateEnd) - ((uintptr_t) src);

         if (count)
         {
             err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
         if (count)
         {
             err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);


@@ -1627,16 +2075,13 @@ hibernate_write_image(void)
 
         // write the preview buffer
 
 
         // write the preview buffer
 

-        addr64_t phys64;
-        IOByteCount segLen;
-
-        if (vars->previewData)
+        if (vars->previewBuffer)

         {
             ppnum = 0;
             count = 0;
             do
             {
         {
             ppnum = 0;
             count = 0;
             do
             {

-                phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen);
+                phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);

                 pageAndCount[0] = atop_64(phys64);
                 pageAndCount[1] = atop_32(segLen);
                 err = IOPolledFileWrite(vars->fileVars, 
                 pageAndCount[0] = atop_64(phys64);
                 pageAndCount[1] = atop_32(segLen);
                 err = IOPolledFileWrite(vars->fileVars, 


@@ -1651,15 +2096,17 @@ hibernate_write_image(void)
             if (kIOReturnSuccess != err)
                 break;
 
             if (kIOReturnSuccess != err)
                 break;
 

-            src = (uint8_t *) vars->previewData->getBytesNoCopy();
-            count = vars->previewData->getLength();
+            src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
+            count = vars->previewBuffer->getLength();

 
             header->previewPageListSize = ppnum;
             header->previewSize = count + ppnum;
 
             for (page = 0; page < count; page += page_size)
 
             header->previewPageListSize = ppnum;
             header->previewSize = count + ppnum;
 
             for (page = 0; page < count; page += page_size)

-                sum1 += hibernate_sum(src + page, page_size);
-
+            {
+                phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
+                sum1 += hibernate_sum_page(src + page, atop_64(phys64));
+            }

             err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
             if (kIOReturnSuccess != err)
                 break;
             err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
             if (kIOReturnSuccess != err)
                 break;


@@ -1668,7 +2115,7 @@ hibernate_write_image(void)
         // mark areas for no save
     
         for (count = 0;
         // mark areas for no save
     
         for (count = 0;

-            (phys64 = vars->ioBuffer->getPhysicalSegment64(count, &segLen));
+            (phys64 = vars->ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));

             count += segLen)
         {
             hibernate_set_page_state(vars->page_list, vars->page_list_wired, 
             count += segLen)
         {
             hibernate_set_page_state(vars->page_list, vars->page_list_wired, 


@@ -1678,7 +2125,7 @@ hibernate_write_image(void)
         }
     
         for (count = 0;
         }
     
         for (count = 0;

-            (phys64 = vars->srcBuffer->getPhysicalSegment64(count, &segLen));
+            (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));

             count += segLen)
         {
             hibernate_set_page_state(vars->page_list, vars->page_list_wired, 
             count += segLen)
         {
             hibernate_set_page_state(vars->page_list, vars->page_list_wired, 


@@ -1700,15 +2147,16 @@ hibernate_write_image(void)
 
        hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
     
 
        hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
     

-        page = atop_32(sectHIBB);
-        count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
+
+        page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
+        count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;

         hibernate_set_page_state(vars->page_list, vars->page_list_wired,
                                         page, count,
                                         kIOHibernatePageStateFree);
         pageCount -= count;
 
         if (vars->previewBuffer) for (count = 0;
         hibernate_set_page_state(vars->page_list, vars->page_list_wired,
                                         page, count,
                                         kIOHibernatePageStateFree);
         pageCount -= count;
 
         if (vars->previewBuffer) for (count = 0;

-                                        (phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen));
+                                        (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));

                                         count += segLen)
         {
             hibernate_set_page_state(vars->page_list, vars->page_list_wired, 
                                         count += segLen)
         {
             hibernate_set_page_state(vars->page_list, vars->page_list_wired, 


@@ -1717,130 +2165,201 @@ hibernate_write_image(void)
             pageCount -= atop_32(segLen);
         }
 
             pageCount -= atop_32(segLen);
         }
 

+        for (count = 0;
+            (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+            count += segLen)
+        {
+            hibernate_set_page_state(vars->page_list, vars->page_list_wired, 
+                                        atop_64(phys64), atop_32(segLen),
+                                        kIOHibernatePageStateFree);
+            pageCount -= atop_32(segLen);
+        }
+
+               (void)hibernate_pal_callback;
+

         src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
     
         src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
     

-        ppnum     = 0;
-        pagesDone = 0;
+        pagesDone  = 0;
+        lastBlob   = 0;

     
         HIBLOG("writing %d pages\n", pageCount);
 
     
         HIBLOG("writing %d pages\n", pageCount);
 

-        do
+        enum
+        // pageType
+        { 
+            kWired          = 0x02,
+            kEncrypt        = 0x01,
+            kWiredEncrypt   = kWired | kEncrypt,
+            kWiredClear     = kWired,
+            kUnwiredEncrypt = kEncrypt
+        };
+
+        for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)

         {
         {

-            count = hibernate_page_list_iterate(pageType ? vars->page_list : vars->page_list_wired,
-                                                    &ppnum);
-//          kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
-    
-            iterDone = !count;
-
-            pageAndCount[0] = ppnum;
-            pageAndCount[1] = count;
-            err = IOPolledFileWrite(vars->fileVars, 
-                                    (const uint8_t *) &pageAndCount, sizeof(pageAndCount), 
-                                    cryptvars);
-            if (kIOReturnSuccess != err)
-                break;
-
-            for (page = 0; page < count; page++)
+            if (needEncrypt && (kEncrypt & pageType))

             {
             {

-                err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(ppnum), page_size);
-                if (err)
+                vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
+                vars->fileVars->encryptEnd   = UINT64_MAX;
+                HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
+
+                if (kUnwiredEncrypt == pageType)

                 {
                 {

-                    HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%d] %x\n", __LINE__, ppnum, err);
-                    break;
+                    // start unwired image
+                    bcopy(&cryptvars->aes_iv[0], 
+                            &gIOHibernateCryptWakeContext.aes_iv[0], 
+                            sizeof(cryptvars->aes_iv));
+                    cryptvars = &gIOHibernateCryptWakeContext;

                 }
                 }

+            }
+            for (iterDone = false, ppnum = 0; !iterDone; )
+            {
+                count = hibernate_page_list_iterate((kWired & pageType) 
+                                                            ? vars->page_list_wired : vars->page_list,
+                                                        &ppnum);
+//              kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
+                iterDone = !count;

     
     

-                sum = hibernate_sum(src, page_size);
-   
-                clock_get_uptime(&startTime);
-
-                pageCompressedSize = WKdm_compress ((WK_word*) src, (WK_word*) (src + page_size), PAGE_SIZE_IN_WORDS);
-    
-                clock_get_uptime(&endTime);
-                ADD_ABSOLUTETIME(&compTime, &endTime);
-                SUB_ABSOLUTETIME(&compTime, &startTime);
-    
-                if (kIOHibernateModeEncrypt & gIOHibernateMode)
-                    pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
-
-                if (pageCompressedSize > page_size)
+                if (count && (kWired & pageType) && needEncrypt)

                 {
                 {

-//                  HIBLOG("------------lose: %d\n", pageCompressedSize);
-                    pageCompressedSize = page_size;
+                    uint32_t checkIndex;
+                    for (checkIndex = 0;
+                            (checkIndex < count) 
+                                && (((kEncrypt & pageType) == 0) == no_encrypt_page(ppnum + checkIndex)); 
+                            checkIndex++)
+                    {}
+                    if (!checkIndex)
+                    {
+                        ppnum++;
+                        continue;
+                    }
+                    count = checkIndex;

                 }
 
                 }
 

-                if (pageCompressedSize != page_size)
-                    data = (src + page_size);
-                else
-                    data = src;
-
-                tag = pageCompressedSize | kIOHibernateTagSignature;
-
-                if (pageType)
-                    sum2 += sum;
+                switch (pageType)
+                {
+                    case kWiredEncrypt:   wired_pages_encrypted += count; break;
+                    case kWiredClear:     wired_pages_clear     += count; break;
+                    case kUnwiredEncrypt: dirty_pages_encrypted += count; break;
+                }
+    
+                if (iterDone && (kWiredEncrypt == pageType))   {/* not yet end of wired list */}

                 else
                 else

-                    sum1 += sum;
-
-                if (needEncryptStart && (ppnum >= atop_32(sectDATAB)))

                 {
                 {

-                    // start encrypting partway into the data about to be written
-                    vars->fileVars->encryptStart = (vars->fileVars->position + AES_BLOCK_SIZE - 1) 
-                                                    & ~(AES_BLOCK_SIZE - 1);
-                    needEncryptStart = false;
+                    pageAndCount[0] = ppnum;
+                    pageAndCount[1] = count;
+                    err = IOPolledFileWrite(vars->fileVars, 
+                                            (const uint8_t *) &pageAndCount, sizeof(pageAndCount), 
+                                            cryptvars);
+                    if (kIOReturnSuccess != err)
+                        break;

                 }
                 }

-
-                err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
-                if (kIOReturnSuccess != err)
-                    break;
-
-                err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
-                if (kIOReturnSuccess != err)
-                    break;
-
-                compressedSize += pageCompressedSize;
-                if (pageCompressedSize)
-                    uncompressedSize += page_size;
-                ppnum++;
-                pagesDone++;

     
     

-                if (0 == (8191 & pagesDone))
+                for (page = ppnum; page < (ppnum + count); page++)

                 {
                 {

+                    err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
+                    if (err)
+                    {
+                        HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
+                        break;
+                    }
+        
+                    sum = hibernate_sum_page(src, page);
+                    if (kWired & pageType)
+                        sum1 += sum;
+                    else
+                        sum2 += sum;
+       
+                    clock_get_uptime(&startTime);
+
+                    pageCompressedSize = WKdm_compress ((WK_word*) src, (WK_word*) (src + page_size), PAGE_SIZE_IN_WORDS);
+        

                     clock_get_uptime(&endTime);
                     clock_get_uptime(&endTime);

-                    SUB_ABSOLUTETIME(&endTime, &allTime);
-                    absolutetime_to_nanoseconds(endTime, &nsec);
-                    progressStamp = nsec / 750000000ULL;
-                    if (progressStamp != lastProgressStamp)
+                    ADD_ABSOLUTETIME(&compTime, &endTime);
+                    SUB_ABSOLUTETIME(&compTime, &startTime);
+                    compBytes += page_size;
+        
+                    if (kIOHibernateModeEncrypt & gIOHibernateMode)
+                        pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
+    
+                    if (pageCompressedSize > page_size)

                     {
                     {

-                        lastProgressStamp = progressStamp;
-                        HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
+//                      HIBLOG("------------lose: %d\n", pageCompressedSize);
+                        pageCompressedSize = page_size;
+                    }
+    
+                    if (pageCompressedSize != page_size)
+                        data = (src + page_size);
+                    else
+                        data = src;
+    
+                    tag = pageCompressedSize | kIOHibernateTagSignature;
+                    err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
+                    if (kIOReturnSuccess != err)
+                        break;
+    
+                    err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
+                    if (kIOReturnSuccess != err)
+                        break;
+    
+                    compressedSize += pageCompressedSize;
+                    if (pageCompressedSize)
+                        uncompressedSize += page_size;
+                    pagesDone++;
+    
+                    if (vars->consoleMapping && (0 == (1023 & pagesDone)))
+                    {
+                        blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
+                        if (blob != lastBlob)
+                        {
+                            ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
+                            lastBlob = blob;
+                        }
+                    }
+                    if (0 == (8191 & pagesDone))
+                    {
+                        clock_get_uptime(&endTime);
+                        SUB_ABSOLUTETIME(&endTime, &allTime);
+                        absolutetime_to_nanoseconds(endTime, &nsec);
+                        progressStamp = nsec / 750000000ULL;
+                        if (progressStamp != lastProgressStamp)
+                        {
+                            lastProgressStamp = progressStamp;
+                            HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
+                        }

                     }
                 }
                     }
                 }

+                if (kIOReturnSuccess != err)
+                    break;
+                ppnum = page;

             }
             }

+

             if (kIOReturnSuccess != err)
                 break;
             if (kIOReturnSuccess != err)
                 break;

-            if (iterDone && !pageType)
+
+            if ((kEncrypt & pageType))
+            {
+                vars->fileVars->encryptEnd = (vars->fileVars->position + AES_BLOCK_SIZE - 1) 
+                                              & ~(AES_BLOCK_SIZE - 1);
+                HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
+            }
+
+            if (kWiredEncrypt != pageType)

             {
             {

+                // end of image1/2 - fill to next block

                 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
                 if (kIOReturnSuccess != err)
                     break;
                 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
                 if (kIOReturnSuccess != err)
                     break;

-
-                iterDone = false;
-                pageType = 1;
-                ppnum = 0;
+            }
+            if (kWiredClear == pageType)
+            {
+                // end wired image
+                header->encryptStart = vars->fileVars->encryptStart;
+                header->encryptEnd   = vars->fileVars->encryptEnd;

                 image1Size = vars->fileVars->position;
                 image1Size = vars->fileVars->position;

-                if (cryptvars)
-                {
-                    bcopy(&cryptvars->aes_iv[0], 
-                            &gIOHibernateCryptWakeContext.aes_iv[0], 
-                            sizeof(cryptvars->aes_iv));
-                    cryptvars = &gIOHibernateCryptWakeContext;
-                }
-                HIBLOG("image1Size %qd\n", image1Size);
+                HIBLOG("image1Size %qd, encryptStart1 %qx, End1 %qx\n",
+                        image1Size, header->encryptStart, header->encryptEnd);

             }
         }
             }
         }

-        while (!iterDone);
-        if (kIOReturnSuccess != err)
-            break;
-        err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);

         if (kIOReturnSuccess != err)
             break;
 
         if (kIOReturnSuccess != err)
             break;
 


@@ -1850,7 +2369,6 @@ hibernate_write_image(void)
         header->image1Size   = image1Size;
         header->bitmapSize   = bitmap_size;
         header->pageCount    = pageCount;
         header->image1Size   = image1Size;
         header->bitmapSize   = bitmap_size;
         header->pageCount    = pageCount;

-        header->encryptStart = vars->fileVars->encryptStart;

     
         header->restore1Sum  = restore1Sum;
         header->image1Sum    = sum1;
     
         header->restore1Sum  = restore1Sum;
         header->image1Sum    = sum1;


@@ -1865,6 +2383,8 @@ hibernate_write_image(void)
         else
             header->fileExtentMapSize = sizeof(header->fileExtentMap);
         bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
         else
             header->fileExtentMapSize = sizeof(header->fileExtentMap);
         bcopy(&fileExtents[0], &header->fileExtentMap[0], count);

+
+        header->deviceBase = vars->fileVars->block0;

     
         IOPolledFileSeek(vars->fileVars, 0);
         err = IOPolledFileWrite(vars->fileVars,
     
         IOPolledFileSeek(vars->fileVars, 0);
         err = IOPolledFileWrite(vars->fileVars,


@@ -1875,25 +2395,33 @@ hibernate_write_image(void)
         err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
         if (kIOReturnSuccess != err)
             break;
         err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
         if (kIOReturnSuccess != err)
             break;

-        err = IOHibernatePollerIODone(vars->fileVars);
+        err = IOHibernatePollerIODone(vars->fileVars, true);

         if (kIOReturnSuccess != err)
             break;
     }
     while (false);
     
     clock_get_uptime(&endTime);
         if (kIOReturnSuccess != err)
             break;
     }
     while (false);
     
     clock_get_uptime(&endTime);

+
+    IOService::getPMRootDomain()->pmStatsRecordEvent( 
+                        kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
+

     SUB_ABSOLUTETIME(&endTime, &allTime);
     absolutetime_to_nanoseconds(endTime, &nsec);
     HIBLOG("all time: %qd ms, ", 
                nsec / 1000000ULL);
 
     absolutetime_to_nanoseconds(compTime, &nsec);
     SUB_ABSOLUTETIME(&endTime, &allTime);
     absolutetime_to_nanoseconds(endTime, &nsec);
     HIBLOG("all time: %qd ms, ", 
                nsec / 1000000ULL);
 
     absolutetime_to_nanoseconds(compTime, &nsec);

-    HIBLOG("comp time: %qd ms, ", 
-               nsec / 1000000ULL);
+    HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ", 
+               compBytes, 
+               nsec / 1000000ULL,
+               nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);

 
 

-    absolutetime_to_nanoseconds(decoTime, &nsec);
-    HIBLOG("deco time: %qd ms, ", 
-               nsec / 1000000ULL);
+    absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
+    HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ", 
+               vars->fileVars->cryptBytes, 
+               nsec / 1000000ULL, 
+               nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);

 
     HIBLOG("\nimage %qd, uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n", 
                header->imageSize,
 
     HIBLOG("\nimage %qd, uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n", 
                header->imageSize,


@@ -1901,77 +2429,51 @@ hibernate_write_image(void)
                uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
                sum1, sum2);
 
                uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
                sum1, sum2);
 

+    HIBLOG("wired_pages_encrypted %d, wired_pages_clear %d, dirty_pages_encrypted %d\n", 
+             wired_pages_encrypted, wired_pages_clear, dirty_pages_encrypted);
+
+    if (vars->fileVars->io)
+        (void) IOHibernatePollerIODone(vars->fileVars, false);
+

     if (pollerOpen)
         IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
 
     if (pollerOpen)
         IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
 

+    if (vars->consoleMapping)
+        ProgressUpdate(gIOHibernateGraphicsInfo, 
+                        vars->consoleMapping, 0, kIOHibernateProgressCount);
+

     HIBLOG("hibernate_write_image done(%x)\n", err);
 
     // should we come back via regular wake, set the state in memory.
     gIOHibernateState = kIOHibernateStateInactive;
 
     HIBLOG("hibernate_write_image done(%x)\n", err);
 
     // should we come back via regular wake, set the state in memory.
     gIOHibernateState = kIOHibernateStateInactive;
 

-    if ((kIOReturnSuccess == err) && !(kIOHibernateModeSleep & gIOHibernateMode))
-        return (true  /* power down */ );
-    else
-        return (false /* sleep */ );
-}
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+    KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END,
+                         wired_pages_encrypted, wired_pages_clear, dirty_pages_encrypted, 0, 0);

 
 

-DECLARE_IOHIBERNATEPROGRESSALPHA
-
-static void
-ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
-{
-    uint32_t  rowBytes, pixelShift;
-    uint32_t  x, y;
-    int32_t   blob, lastBlob;
-    uint32_t  alpha, in, color, result;
-    uint8_t * out;
-    uint32_t  saveindex[kIOHibernateProgressCount] = { 0 };
-
-    pixelShift = display->depth >> 4;
-    if (pixelShift < 1)
-        return;
-
-    rowBytes = display->rowBytes;
-
-    screen += ((display->width 
-            - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
-                + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
-
-    lastBlob  = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
-
-    screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
-
-    for (y = 0; y < kIOHibernateProgressHeight; y++)
+    if (kIOReturnSuccess == err)

     {
     {

-        out = screen + y * rowBytes;
-        for (blob = firstBlob; blob <= lastBlob; blob++)
-        {
-            color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
-            for (x = 0; x < kIOHibernateProgressWidth; x++)
-            {
-                alpha  = gIOHibernateProgressAlpha[y][x];
-                result = color;
-                if (alpha)
-                {
-                    if (0xff != alpha)
-                    {
-                        in = display->progressSaveUnder[blob][saveindex[blob]++];
-                        result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
-                    }
-                    if (1 == pixelShift)
-                    {
-                        result >>= 3;
-                        *((uint16_t *)out) = (result << 10) | (result << 5) | result;  // 16
-                    }
-                    else
-                        *((uint32_t *)out) = (result << 16) | (result << 8) | result;  // 32
-                }
-                out += (1 << pixelShift);
-            }
-            out += (kIOHibernateProgressSpacing << pixelShift);
-        }
+       if (kIOHibernateModeSleep & gIOHibernateMode)
+       {
+           return (kIOHibernatePostWriteSleep);
+       }
+       else if(kIOHibernateModeRestart & gIOHibernateMode)
+       {
+           return (kIOHibernatePostWriteRestart);
+       }
+       else
+       {
+           /* by default, power down */
+           return (kIOHibernatePostWriteHalt);
+       }
+    }
+    else if (kIOReturnAborted == err)
+    {
+       return (kIOHibernatePostWriteWake);
+    }
+    else
+    {
+       /* on error, sleep */
+       return (kIOHibernatePostWriteSleep);

     }
 }
 
     }
 }
 


@@ -1983,7 +2485,10 @@ hibernate_machine_init(void)
     IOReturn     err;
     uint32_t     sum;
     uint32_t     pagesDone;
     IOReturn     err;
     uint32_t     sum;
     uint32_t     pagesDone;

+    uint32_t     pagesRead = 0;
+    AbsoluteTime startTime, compTime;

     AbsoluteTime allTime, endTime;
     AbsoluteTime allTime, endTime;

+    uint64_t     compBytes;

     uint64_t     nsec;
     uint32_t     lastProgressStamp = 0;
     uint32_t     progressStamp;
     uint64_t     nsec;
     uint32_t     lastProgressStamp = 0;
     uint32_t     progressStamp;


@@ -2011,17 +2516,73 @@ hibernate_machine_init(void)
 
     HIBPRINT("diag %x %x %x %x\n",
            gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1], 
 
     HIBPRINT("diag %x %x %x %x\n",
            gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1], 

-           gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]); 
+           gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);

 
 

-    HIBPRINT("video %x %d %d %d\n",
+    HIBPRINT("video %x %d %d %d status %x\n",

            gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth, 
            gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth, 

-           gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height); 
+           gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus); 

 
     if ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)
         hibernate_page_list_discard(vars->page_list);
 
     boot_args *args = (boot_args *) PE_state.bootArgs;
 
 
     if ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)
         hibernate_page_list_discard(vars->page_list);
 
     boot_args *args = (boot_args *) PE_state.bootArgs;
 

+    cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
+
+    if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount)
+       panic("handoff overflow");
+
+    IOHibernateHandoff * handoff;
+    bool                 done           = false;
+    bool                 foundCryptData = false;
+
+    for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
+        !done;
+        handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
+    {
+//     HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
+       uint8_t * data = &handoff->data[0];
+       switch (handoff->type)
+       {
+           case kIOHibernateHandoffTypeEnd:
+               done = true;
+               break;
+
+           case kIOHibernateHandoffTypeGraphicsInfo:
+               bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
+               break;
+
+           case kIOHibernateHandoffTypeCryptVars:
+               if (cryptvars)
+               {
+                   hibernate_cryptwakevars_t *
+                   wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
+                   bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
+               }
+               foundCryptData = true;
+               bzero(data, handoff->bytecount);
+               break;
+
+           case kIOHibernateHandoffTypeMemoryMap:
+               hibernate_newruntime_map(data, handoff->bytecount, 
+                                        gIOHibernateCurrentHeader->systemTableOffset);
+               break;
+
+           case kIOHibernateHandoffTypeDeviceTree:
+               {
+//                 DTEntry chosen = NULL;
+//                 HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen));
+               }
+               break;
+
+           default:
+               done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
+               break;
+       }    
+    }
+    if (cryptvars && !foundCryptData)
+       panic("hibernate handoff");
+

     if (vars->videoMapping 
        && gIOHibernateGraphicsInfo->physicalAddress
        && (args->Video.v_baseAddr == gIOHibernateGraphicsInfo->physicalAddress))
     if (vars->videoMapping 
        && gIOHibernateGraphicsInfo->physicalAddress
        && (args->Video.v_baseAddr == gIOHibernateGraphicsInfo->physicalAddress))


@@ -2034,21 +2595,11 @@ hibernate_machine_init(void)
     }
 
     uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
     }
 
     uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();

-
-    if (gIOHibernateWakeMapSize)
-    {
-       err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(gIOHibernateWakeMap), 
-                                                   gIOHibernateWakeMapSize);
-       if (kIOReturnSuccess == err)
-           hibernate_newruntime_map(src, gIOHibernateWakeMapSize, 
-                                    gIOHibernateCurrentHeader->systemTableOffset);
-       gIOHibernateWakeMap = 0;
-       gIOHibernateWakeMapSize = 0;
-    }
-

     uint32_t decoOffset;
 
     clock_get_uptime(&allTime);
     uint32_t decoOffset;
 
     clock_get_uptime(&allTime);

+    AbsoluteTime_to_scalar(&compTime) = 0;
+    compBytes = 0;

 
     HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
     err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0);
 
     HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
     err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0);


@@ -2062,30 +2613,24 @@ hibernate_machine_init(void)
 
     IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
 
 
     IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
 

-    if (vars->videoMapping)
+    if (vars->videoMapSize)

     {
         lastBlob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
                         / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
         ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, 0, lastBlob);
     }
 
     {
         lastBlob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
                         / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
         ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, 0, lastBlob);
     }
 

-    cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
-    if (kIOHibernateModeEncrypt & gIOHibernateMode)
-    {
-        cryptvars = &gIOHibernateCryptWakeContext;
-        bcopy(&gIOHibernateCryptWakeVars->aes_iv[0], 
-                &cryptvars->aes_iv[0], 
-                sizeof(cryptvars->aes_iv));
-    }
-

     // kick off the read ahead
     vars->fileVars->io          = false;
     vars->fileVars->bufferHalf   = 0;
     vars->fileVars->bufferLimit  = 0;
     vars->fileVars->lastRead     = 0;
     // kick off the read ahead
     vars->fileVars->io          = false;
     vars->fileVars->bufferHalf   = 0;
     vars->fileVars->bufferLimit  = 0;
     vars->fileVars->lastRead     = 0;

+    vars->fileVars->readEnd      = gIOHibernateCurrentHeader->imageSize;

     vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
     vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;

+    vars->fileVars->cryptBytes   = 0;
+    AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;

 
 

-    IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
+    err = IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);

     vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
     // --
 
     vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
     // --
 


@@ -2094,14 +2639,16 @@ hibernate_machine_init(void)
     uint32_t * header = (uint32_t *) src;
     sum = 0;
 
     uint32_t * header = (uint32_t *) src;
     sum = 0;
 

-    do
+    while (kIOReturnSuccess == err)

     {
        unsigned int count;
        unsigned int page;
         uint32_t     tag;
        vm_offset_t  ppnum, compressedSize;
 
     {
        unsigned int count;
        unsigned int page;
         uint32_t     tag;
        vm_offset_t  ppnum, compressedSize;
 

-       IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
+       err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
+       if (kIOReturnSuccess != err)
+           break;

 
        ppnum = header[0];
        count = header[1];
 
        ppnum = header[0];
        count = header[1];


@@ -2113,9 +2660,17 @@ hibernate_machine_init(void)
 
        for (page = 0; page < count; page++)
        {
 
        for (page = 0; page < count; page++)
        {

-           IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
+           err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
+           if (kIOReturnSuccess != err)
+               break;

 
            compressedSize = kIOHibernateTagLength & tag;
 
            compressedSize = kIOHibernateTagLength & tag;

+           if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength))
+           {
+               err = kIOReturnIPCError;
+               break;
+           }
+

            if (!compressedSize)
            {
                ppnum++;
            if (!compressedSize)
            {
                ppnum++;


@@ -2123,26 +2678,39 @@ hibernate_machine_init(void)
                continue;
            }
 
                continue;
            }
 

-           IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
-   
-           if (compressedSize != page_size)
+           err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
+           if (kIOReturnSuccess != err)
+               break;
+
+           if (compressedSize < page_size)

            {
                decoOffset = page_size;
            {
                decoOffset = page_size;

+
+                clock_get_uptime(&startTime);

                WKdm_decompress((WK_word*) src, (WK_word*) (src + decoOffset), PAGE_SIZE_IN_WORDS);
                WKdm_decompress((WK_word*) src, (WK_word*) (src + decoOffset), PAGE_SIZE_IN_WORDS);

+                clock_get_uptime(&endTime);
+                ADD_ABSOLUTETIME(&compTime, &endTime);
+                SUB_ABSOLUTETIME(&compTime, &startTime);
+
+                compBytes += page_size;

            }
            else
                decoOffset = 0;
 
            }
            else
                decoOffset = 0;
 

-           sum += hibernate_sum((src + decoOffset), page_size);
+           sum += hibernate_sum_page((src + decoOffset), ppnum);

 
            err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
            if (err)
 
            err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
            if (err)

-               HIBLOG("IOMemoryDescriptorReadToPhysical [%d] %x\n", ppnum, err);
+           {
+               HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
+               break;
+           }

 
            ppnum++;
            pagesDone++;
 
            ppnum++;
            pagesDone++;

+           pagesRead++;

 
 

-            if (vars->videoMapping && (0 == (255 & pagesDone)))
+            if (vars->videoMapSize && (0 == (1023 & pagesDone)))

             {
                 blob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
                         / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
             {
                 blob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
                         / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);


@@ -2168,25 +2736,49 @@ hibernate_machine_init(void)
            }
        }
     }
            }
        }
     }

-    while (true);
+    if (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages)
+       err = kIOReturnLockedRead;
+
+    if (kIOReturnSuccess != err)
+       panic("Hibernate restore error %x", err);

 
     gIOHibernateCurrentHeader->actualImage2Sum = sum;
 
     if (vars->fileVars->io)
 
     gIOHibernateCurrentHeader->actualImage2Sum = sum;
 
     if (vars->fileVars->io)

-        (void) IOHibernatePollerIODone(vars->fileVars);
+        (void) IOHibernatePollerIODone(vars->fileVars, false);

 
     err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
 
 
     err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
 

-    if (vars->videoMapping)
+    if (vars->videoMapSize)

         ProgressUpdate(gIOHibernateGraphicsInfo, 
                         (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
 
     clock_get_uptime(&endTime);
         ProgressUpdate(gIOHibernateGraphicsInfo, 
                         (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
 
     clock_get_uptime(&endTime);

+
+    IOService::getPMRootDomain()->pmStatsRecordEvent( 
+                        kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
+    IOService::getPMRootDomain()->pmStatsRecordEvent( 
+                        kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
+

     SUB_ABSOLUTETIME(&endTime, &allTime);
     absolutetime_to_nanoseconds(endTime, &nsec);
 
     SUB_ABSOLUTETIME(&endTime, &allTime);
     absolutetime_to_nanoseconds(endTime, &nsec);
 

-    HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %qd ms\n", 
+    HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %qd ms, ", 

                pagesDone, sum, nsec / 1000000ULL);
                pagesDone, sum, nsec / 1000000ULL);

+
+    absolutetime_to_nanoseconds(compTime, &nsec);
+    HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ", 
+               compBytes, 
+               nsec / 1000000ULL,
+               nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+    absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
+    HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n", 
+               vars->fileVars->cryptBytes, 
+               nsec / 1000000ULL, 
+               nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+    KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_NONE, pagesRead, pagesDone, 0, 0, 0);

 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */