xnu-2782.10.72.tar.gz

[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp

diff --git a/iokit/Kernel/IOHibernateIO.cpp b/iokit/Kernel/IOHibernateIO.cpp
index 9abb3f6ca4374452362a5e6b8f057ebbdfbffb6c..f85f2ab0e65d5e75b7419cfc4f45f97d858f41ee 100644 (file)
--- a/iokit/Kernel/IOHibernateIO.cpp
+++ b/iokit/Kernel/IOHibernateIO.cpp
@@ -58,12 +58,12 @@ Sleep:
   by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
   The image header and block list are written. The header includes the second file extent so
   only the header block is needed to read the file, regardless of filesystem.
   by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
   The image header and block list are written. The header includes the second file extent so
   only the header block is needed to read the file, regardless of filesystem.

-  The kernel section "__HIB" is written uncompressed to the image. This section of code and data 
+  The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data 

   (only) is used to decompress the image during wake/boot.
   Some additional pages are removed from the bitmaps - the buffers used for hibernation.
   The bitmaps are written to the image.
   More areas are removed from the bitmaps (after they have been written to the image) - the 
   (only) is used to decompress the image during wake/boot.
   Some additional pages are removed from the bitmaps - the buffers used for hibernation.
   The bitmaps are written to the image.
   More areas are removed from the bitmaps (after they have been written to the image) - the 

-  section "__HIB" pages and interrupt stack.
+  segment "__HIB" pages and interrupt stack.

   Each wired page is compressed and written and then each non-wired page. Compression and 
   disk writes are in parallel.
   The image header is written to the start of the file and the polling driver closed.
   Each wired page is compressed and written and then each non-wired page. Compression and 
   disk writes are in parallel.
   The image header is written to the start of the file and the polling driver closed.


@@ -151,67 +151,85 @@ to restrict I/O ops.
 #include <IOKit/pwr_mgt/IOPowerConnection.h>
 #include "IOPMPowerStateQueue.h"
 #include <IOKit/IOBufferMemoryDescriptor.h>
 #include <IOKit/pwr_mgt/IOPowerConnection.h>
 #include "IOPMPowerStateQueue.h"
 #include <IOKit/IOBufferMemoryDescriptor.h>

-#include <crypto/aes.h>
+#include <IOKit/AppleKeyStoreInterface.h>
+#include <libkern/crypto/aes.h>

 
 #include <sys/uio.h>
 #include <sys/conf.h>
 #include <sys/stat.h>
 #include <sys/fcntl.h>                       // (FWRITE, ...)
 #include <sys/sysctl.h>
 
 #include <sys/uio.h>
 #include <sys/conf.h>
 #include <sys/stat.h>
 #include <sys/fcntl.h>                       // (FWRITE, ...)
 #include <sys/sysctl.h>

+#include <sys/kdebug.h>
+#include <stdint.h>

 
 #include <IOKit/IOHibernatePrivate.h>
 #include <IOKit/IOPolledInterface.h>
 #include <IOKit/IONVRAM.h>
 #include "IOHibernateInternal.h"
 
 #include <IOKit/IOHibernatePrivate.h>
 #include <IOKit/IOPolledInterface.h>
 #include <IOKit/IONVRAM.h>
 #include "IOHibernateInternal.h"

-#include "WKdm.h"
+#include <vm/WKdm_new.h>

 #include "IOKitKernelInternal.h"
 #include "IOKitKernelInternal.h"

+#include <pexpert/device_tree.h>

 
 

-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+#include <machine/pal_routines.h>
+#include <machine/pal_hibernate.h>
+#include <i386/tsc.h>

 
 

-OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
-
-OSMetaClassDefineReservedUnused(IOPolledInterface, 0);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
+extern "C" addr64_t            kvtophys(vm_offset_t va);
+extern "C" ppnum_t             pmap_find_phys(pmap_t pmap, addr64_t va);

 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 

+#define        DISABLE_TRIM            0
+#define TRIM_DELAY             5000
+
+extern unsigned int            save_kdebug_enable;

 extern uint32_t                gIOHibernateState;
 uint32_t                       gIOHibernateMode;
 static char                    gIOHibernateBootSignature[256+1];
 static char                    gIOHibernateFilename[MAXPATHLEN+1];
 extern uint32_t                gIOHibernateState;
 uint32_t                       gIOHibernateMode;
 static char                    gIOHibernateBootSignature[256+1];
 static char                    gIOHibernateFilename[MAXPATHLEN+1];

-static uint32_t                        gIOHibernateFreeRatio = 0;              // free page target (percent)
-uint32_t                       gIOHibernateFreeTime  = 0*1000; // max time to spend freeing pages (ms)
+static uint32_t                        gIOHibernateFreeRatio = 0;       // free page target (percent)
+uint32_t                       gIOHibernateFreeTime  = 0*1000;  // max time to spend freeing pages (ms)
+static uint64_t                        gIOHibernateCompression = 0x80;  // default compression 50%

 
 static IODTNVRAM *             gIOOptionsEntry;
 static IORegistryEntry *       gIOChosenEntry;
 #if defined(__i386__) || defined(__x86_64__)
 static const OSSymbol *         gIOCreateEFIDevicePathSymbol;
 
 static IODTNVRAM *             gIOOptionsEntry;
 static IORegistryEntry *       gIOChosenEntry;
 #if defined(__i386__) || defined(__x86_64__)
 static const OSSymbol *         gIOCreateEFIDevicePathSymbol;

+static const OSSymbol *        gIOHibernateRTCVariablesKey;
+static const OSSymbol *         gIOHibernateBoot0082Key;
+static const OSSymbol *         gIOHibernateBootNextKey;
+static OSData *                        gIOHibernateBoot0082Data;
+static OSData *                        gIOHibernateBootNextData;
+static OSObject *              gIOHibernateBootNextSave;
+static struct kern_direct_file_io_ref_t * gDebugImageFileRef;

 #endif
 
 #endif
 

+static IOLock *                           gFSLock;
+static uint32_t                           gFSState;

 static IOPolledFileIOVars                gFileVars;
 static IOHibernateVars                   gIOHibernateVars;
 static struct kern_direct_file_io_ref_t * gIOHibernateFileRef;
 static hibernate_cryptvars_t             gIOHibernateCryptWakeContext;
 static IOPolledFileIOVars                gFileVars;
 static IOHibernateVars                   gIOHibernateVars;
 static struct kern_direct_file_io_ref_t * gIOHibernateFileRef;
 static hibernate_cryptvars_t             gIOHibernateCryptWakeContext;

+static hibernate_graphics_t              _hibernateGraphics;
+static hibernate_graphics_t *            gIOHibernateGraphicsInfo = &_hibernateGraphics;
+static hibernate_statistics_t            _hibernateStats;
+static hibernate_statistics_t *                  gIOHibernateStats = &_hibernateStats;
+
+enum 
+{
+    kFSIdle     = 0,
+    kFSOpening  = 2,
+    kFSOpened   = 3,
+    kFSTimedOut = 4,
+};
+
+static IOReturn IOHibernateDone(IOHibernateVars * vars);

 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 enum { kXPRamAudioVolume = 8 };
 enum { kDefaultIOSize = 128 * 1024 };
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 enum { kXPRamAudioVolume = 8 };
 enum { kDefaultIOSize = 128 * 1024 };

-enum { kVideoMapSize  = 32 * 1024 * 1024 };
+enum { kVideoMapSize  = 80 * 1024 * 1024 };

 
 #ifndef kIOMediaPreferredBlockSizeKey
 #define kIOMediaPreferredBlockSizeKey  "Preferred Block Size"
 
 #ifndef kIOMediaPreferredBlockSizeKey
 #define kIOMediaPreferredBlockSizeKey  "Preferred Block Size"


@@ -224,9 +242,6 @@ enum { kVideoMapSize  = 32 * 1024 * 1024 };
 #define kIOSelectedBootDeviceKey       "boot-device"
 #endif
 
 #define kIOSelectedBootDeviceKey       "boot-device"
 #endif
 

-
-enum { kIOHibernateMinPollersNeeded = 2 };
-

 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 // copy from phys addr to MD
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 // copy from phys addr to MD


@@ -489,6 +504,12 @@ IOHibernatePollerIODone(IOPolledFileIOVars * vars, bool abortable)
         }
     }
 
         }
     }
 

+    if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort())
+    {
+        err = kIOReturnAborted;
+       HIBLOG("IOPolledInterface::checkForWork sw abort\n");
+    }
+

     if (err)
     {
        HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
     if (err)
     {
        HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);


@@ -544,33 +565,177 @@ file_extent_callback(void * ref, uint64_t start, uint64_t length)
     extent.start  = start;
     extent.length = length;
 
     extent.start  = start;
     extent.length = length;
 

+    HIBLOG("[0x%qx, 0x%qx]\n", start, length);
+

     ctx->extents->appendBytes(&extent, sizeof(extent));
     ctx->size += length;
 }
 
     ctx->extents->appendBytes(&extent, sizeof(extent));
     ctx->size += length;
 }
 

+static IOService * 
+IOCopyMediaForDev(dev_t device)
+{
+    OSDictionary * matching;
+    OSNumber *     num;
+    OSIterator *   iter;
+    IOService *    result = 0;
+
+    matching = IOService::serviceMatching("IOMedia");
+    if (!matching)
+        return (0);
+    do
+    {
+        num = OSNumber::withNumber(major(device), 32);
+        if (!num)
+            break;
+        matching->setObject(kIOBSDMajorKey, num);
+        num->release();
+        num = OSNumber::withNumber(minor(device), 32);
+        if (!num)
+            break;
+        matching->setObject(kIOBSDMinorKey, num);
+        num->release();
+        if (!num)
+            break;
+        iter = IOService::getMatchingServices(matching);
+        if (iter)
+        {
+            result = (IOService *) iter->getNextObject();
+            result->retain();
+            iter->release();
+        }
+    }
+    while (false);
+    matching->release();
+
+    return (result);
+}
+
+/* 
+ * Writes header to disk with signature, block size and file extents data.
+ * If there are more than 2 extents, then they are written on second block.
+ */
+static IOReturn
+WriteExtentsToFile(struct kern_direct_file_io_ref_t * fileRef,
+                   uint32_t signature, uint32_t blockSize,
+                   IOPolledFileExtent *fileExtents,
+                   IOByteCount size)
+{
+    IOHibernateImageHeader  hdr;
+    IOItemCount  count;
+    IOReturn     err = kIOReturnSuccess;
+    int         rc;
+
+    memset(&hdr, 0, sizeof(IOHibernateImageHeader));
+    count = size;
+    if (count > sizeof(hdr.fileExtentMap))
+    {
+        hdr.fileExtentMapSize = count;
+        count = sizeof(hdr.fileExtentMap);
+    }
+    else
+        hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap);
+
+    bcopy(fileExtents, &hdr.fileExtentMap[0], count);
+
+    // copy file block extent list if larger than header
+    if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap))
+    {
+            count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap);
+            rc = kern_write_file(fileRef, blockSize, 
+                                 (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)), 
+                                 count, IO_SKIP_ENCRYPTION);
+            if (rc != 0) {
+                HIBLOG("kern_write_file returned %d\n", rc);
+                err = kIOReturnIOError;
+                goto exit;
+            }    
+    }
+    hdr.signature = signature;
+    hdr.deviceBlockSize = blockSize;
+
+    rc = kern_write_file(fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION);
+    if (rc != 0) {
+        HIBLOG("kern_write_file returned %d\n", rc);
+        err = kIOReturnIOError;
+        goto exit;
+    }
+
+exit:
+    return err;
+}
+
+static IOReturn
+GetImageBlockSize(IOService *part, OSArray *pollers, IOByteCount *blockSize)
+{
+    IOService       * service;
+    IORegistryEntry * next;
+    IORegistryEntry * child;
+
+    IOReturn        err = kIOReturnSuccess;
+
+
+    next = part;
+    do
+    {
+        IOPolledInterface * poller;
+        OSObject *          obj;
+        OSNumber        * num;
+
+        obj = next->getProperty(kIOPolledInterfaceSupportKey);
+        if (kOSBooleanFalse == obj)
+        {
+            pollers->flushCollection();
+            break;
+        }
+        else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
+            pollers->setObject(poller);
+
+        if ((service = OSDynamicCast(IOService, next)) 
+            && service->getDeviceMemory()
+            && !pollers->getCount())   break;
+
+        if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey))))
+            *blockSize = num->unsigned32BitValue();
+        child = next;
+    }
+    while ((next = child->getParentEntry(gIOServicePlane)) 
+           && child->isParent(next, gIOServicePlane, true));
+
+    if (*blockSize < 4096) *blockSize = 4096;
+
+    if (!pollers->getCount())
+        err = kIOReturnUnsupported;
+
+    return err;
+}
+

 IOReturn
 IOReturn

-IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
-                           IOPolledFileIOVars ** fileVars, OSData ** fileExtents,
-                           OSData ** imagePath)
+IOPolledFileOpen( const char * filename, uint64_t setFileSize,
+                 IOBufferMemoryDescriptor * ioBuffer, 
+                 IOPolledFileIOVars ** fileVars, OSData ** fileExtents,
+                 OSData ** imagePath, uint8_t * volumeCryptKey)

 {
 {

-    IOReturn                   err = kIOReturnError;
+    IOReturn                   err = kIOReturnSuccess;

     IOPolledFileIOVars *       vars;
     _OpenFileContext           ctx;
     OSData *                   extentsData;
     IOPolledFileIOVars *       vars;
     _OpenFileContext           ctx;
     OSData *                   extentsData;

-    OSNumber *                 num;
-    IORegistryEntry *          part = 0;
-    OSDictionary *             matching;
-    OSIterator *               iter;
+    IOService *                 part = 0;
+    OSString *                  keyUUID = 0;
+    OSString *                  keyStoreUUID = 0;
+    dev_t                      block_dev;

     dev_t                      hibernate_image_dev;
     uint64_t                   maxiobytes;
     dev_t                      hibernate_image_dev;
     uint64_t                   maxiobytes;

+    AbsoluteTime                startTime, endTime;
+    uint64_t                    nsec;
+    caddr_t         write_file_addr = NULL;
+    vm_size_t       write_file_len = 0;
+
+    vars = IONew(IOPolledFileIOVars, 1);
+    if (!vars) return (kIOReturnNoMemory);
+    bzero(vars, sizeof(*vars));

 
 

-    vars = &gFileVars;

     do
     {
     do
     {

-       HIBLOG("sizeof(IOHibernateImageHeader) == %ld\n", sizeof(IOHibernateImageHeader));
-       if (sizeof(IOHibernateImageHeader) != 512)
-           continue;
-    

        vars->io           = false;
        vars->buffer       = (uint8_t *) ioBuffer->getBytesNoCopy();
        vars->bufferHalf   = 0;
        vars->io           = false;
        vars->buffer       = (uint8_t *) ioBuffer->getBytesNoCopy();
        vars->bufferHalf   = 0;


@@ -578,105 +743,146 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
        vars->bufferSize   = ioBuffer->getLength() >> 1;
     
        extentsData = OSData::withCapacity(32);
        vars->bufferSize   = ioBuffer->getLength() >> 1;
     
        extentsData = OSData::withCapacity(32);

-    
-       ctx.extents = extentsData;
+       ctx.extents = extentsData;

        ctx.size    = 0;
        ctx.size    = 0;

+       clock_get_uptime(&startTime);
+    if (!gDebugImageFileRef) 
+    {
+        // Avoid writing the header if it is written when file is prep'd for debug data
+        // Image is locked during prep for debug data. So, write may fail.
+        write_file_addr = (caddr_t)gIOHibernateCurrentHeader;
+        write_file_len = sizeof(IOHibernateImageHeader);
+    }

        vars->fileRef = kern_open_file_for_direct_io(filename, 
        vars->fileRef = kern_open_file_for_direct_io(filename, 

+                                                 true,

                                                    &file_extent_callback, &ctx, 
                                                    &file_extent_callback, &ctx, 

+                                                   setFileSize,
+                                                   // write file:
+                                                    0, write_file_addr,
+                                                    write_file_len,
+                                                    // results
+                                                   &block_dev,

                                                    &hibernate_image_dev,
                                                     &vars->block0,
                                                    &hibernate_image_dev,
                                                     &vars->block0,

-                                                    &maxiobytes);
-       if (!vars->fileRef)
-       {
-           err = kIOReturnNoSpace;
-           break;
-       }
-       gIOHibernateFileRef = vars->fileRef;
-       HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx\n", filename, ctx.size, 
-                    vars->block0, maxiobytes);
+                                                    &maxiobytes,
+                                                    &vars->flags);
+#if 0
+       uint32_t msDelay = (131071 & random());
+       HIBLOG("sleep %d\n", msDelay);
+       IOSleep(msDelay);
+#endif
+        clock_get_uptime(&endTime);
+        SUB_ABSOLUTETIME(&endTime, &startTime);
+        absolutetime_to_nanoseconds(endTime, &nsec);
+
+       if (!vars->fileRef) err = kIOReturnNoSpace;
+
+       IOLockLock(gFSLock);
+       if (kFSOpening != gFSState) err = kIOReturnTimeout;
+       IOLockUnlock(gFSLock);
+
+        HIBLOG("kern_open_file_for_direct_io(%d) took %qd ms\n", err, nsec / 1000000ULL);
+       if (kIOReturnSuccess != err) break;
+
+        if (kIOHibernateModeSSDInvert & gIOHibernateMode)
+            vars->flags ^= kIOHibernateOptionSSD;
+
+       HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx ssd %d\n", filename, ctx.size, 
+                    vars->block0, maxiobytes, kIOHibernateOptionSSD & vars->flags);

        if (ctx.size < 1*1024*1024)             // check against image size estimate!
        {
            err = kIOReturnNoSpace;
            break;
        }
 
        if (ctx.size < 1*1024*1024)             // check against image size estimate!
        {
            err = kIOReturnNoSpace;
            break;
        }
 

-        if (maxiobytes < vars->bufferSize)
-            vars->bufferSize = maxiobytes;
+       vars->fileSize = ctx.size;
+        if (maxiobytes < vars->bufferSize) vars->bufferSize = maxiobytes;

     
        vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
     
        vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();

-    
-       matching = IOService::serviceMatching("IOMedia");
-       num = OSNumber::withNumber(major(hibernate_image_dev), 32);
-       matching->setObject(kIOBSDMajorKey, num);
-       num->release();
-       num = OSNumber::withNumber(minor(hibernate_image_dev), 32);
-       matching->setObject(kIOBSDMinorKey, num);
-       num->release();
-       iter = IOService::getMatchingServices(matching);
-       matching->release();
-       if (iter)
-       {
-           part = (IORegistryEntry *) iter->getNextObject();
-           part->retain();
-           iter->release();
-       }
-       if (!part)
-           break;

 
 

-       int minor, major;
-       IORegistryEntry * next;
-       IORegistryEntry * child;
-       OSData * data;
-
-       num = (OSNumber *) part->getProperty(kIOBSDMajorKey);
-       if (!num)
-           break;
-       major = num->unsigned32BitValue();
-       num = (OSNumber *) part->getProperty(kIOBSDMinorKey);
-       if (!num)
-           break;
-       minor = num->unsigned32BitValue();
+        part = IOCopyMediaForDev(block_dev);
+        if (!part)
+        {
+            err = kIOReturnNotFound;
+            break;
+       }
+        err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false, 
+                                         (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
+        if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID)
+        {
+//            IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
+            uuid_t                  volumeKeyUUID;
+            aks_volume_key_t        vek;
+            static IOService *      sKeyStore;
+            static const OSSymbol * sAKSGetKey;
+
+            if (!sAKSGetKey)
+                sAKSGetKey = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY);
+            if (!sKeyStore)
+                sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
+            if (sKeyStore)
+                err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
+            else
+                err = kIOReturnNoResources;
+            if (kIOReturnSuccess == err)    
+                err = sKeyStore->callPlatformFunction(sAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
+            if (kIOReturnSuccess != err)    
+                IOLog("volume key err 0x%x\n", err);
+            else
+            {
+                size_t bytes = (kIOHibernateAESKeySize / 8);
+                if (vek.key.keybytecount < bytes)
+                     bytes = vek.key.keybytecount;
+                bcopy(&vek.key.keybytes[0], volumeCryptKey, bytes);
+            }
+            bzero(&vek, sizeof(vek));
+        }
+        part->release();

 
 

-       hibernate_image_dev = makedev(major, minor);
+        part = IOCopyMediaForDev(hibernate_image_dev);
+        if (!part)
+        {
+            err = kIOReturnNotFound;
+            break;
+       }

 
         vars->pollers = OSArray::withCapacity(4);
 
         vars->pollers = OSArray::withCapacity(4);

-       if (!vars->pollers)
-           break;
+        if (!vars->pollers)
+        {
+            err = kIOReturnNoMemory;
+            break;
+        }

 
 

-       vars->blockSize = 512;
-       next = part;
-       do
-       {
-            IOPolledInterface * poller;
-           OSObject *          obj;
+        err = GetImageBlockSize(part, vars->pollers, &vars->blockSize);

 
 

-           obj = next->getProperty(kIOPolledInterfaceSupportKey);
-           if (kOSBooleanFalse == obj)
-           {
-               vars->pollers->flushCollection();
-               break;
-           }
-            else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
-                vars->pollers->setObject(poller);
-           if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey))))
-               vars->blockSize = num->unsigned32BitValue();
-            child = next;
-       }
-       while ((next = child->getParentEntry(gIOServicePlane)) 
-                && child->isParent(next, gIOServicePlane, true));
+        HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
+               major(hibernate_image_dev), minor(hibernate_image_dev), (long)vars->blockSize, 
+               vars->pollers->getCount());
+
+        if (err != kIOReturnSuccess)
+            break;

 
 

-       HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
-                   major, minor, (long)vars->blockSize, vars->pollers->getCount());
-       if (vars->pollers->getCount() < kIOHibernateMinPollersNeeded)
+       IORegistryEntry * next;
+    OSData          * data;
+       if (vars->blockSize < sizeof(IOHibernateImageHeader))
+       {
+           err = kIOReturnError;

            continue;
            continue;

+       }

 
        err = IOHibernatePollerProbe(vars, (IOService *) part);
 
        err = IOHibernatePollerProbe(vars, (IOService *) part);

-       if (kIOReturnSuccess != err)
-           break;
+       if (kIOReturnSuccess != err) break;

 
        err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer);
 
        err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer);

-       if (kIOReturnSuccess != err)
-           break;
+       if (kIOReturnSuccess != err) break;
+
+       vars->media = part;
+        next = part;
+       while (next)
+       {
+           next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue);
+           next = next->getParentEntry(gIOServicePlane);
+       }

 
        *fileVars    = vars;
        *fileExtents = extentsData;
 
        *fileVars    = vars;
        *fileExtents = extentsData;


@@ -685,18 +891,22 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
 
        if ((extentsData->getLength() >= sizeof(IOPolledFileExtent)))
        {
 
        if ((extentsData->getLength() >= sizeof(IOPolledFileExtent)))
        {

-           char str2[24];
+           char str2[24 + sizeof(uuid_string_t) + 2];

 
 #if defined(__i386__) || defined(__x86_64__)
            if (!gIOCreateEFIDevicePathSymbol)
                gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
 
 
 #if defined(__i386__) || defined(__x86_64__)
            if (!gIOCreateEFIDevicePathSymbol)
                gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
 

-           snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
+            if (keyUUID)
+                snprintf(str2, sizeof(str2), "%qx:%s", 
+                                vars->extentMap[0].start, keyUUID->getCStringNoCopy());
+            else
+                snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);

 
            err = IOService::getPlatform()->callPlatformFunction(
                                                gIOCreateEFIDevicePathSymbol, false,
 
            err = IOService::getPlatform()->callPlatformFunction(
                                                gIOCreateEFIDevicePathSymbol, false,

-                                               (void *) part, (void *) str2, (void *) true,
-                                               (void *) &data);
+                                               (void *) part, (void *) str2,
+                                               (void *) (uintptr_t) true, (void *) &data);

 #else
            char str1[256];
            int len = sizeof(str1);
 #else
            char str1[256];
            int len = sizeof(str1);


@@ -727,10 +937,16 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
         HIBLOG("error 0x%x opening hibernation file\n", err);
        if (vars->fileRef)
        {
         HIBLOG("error 0x%x opening hibernation file\n", err);
        if (vars->fileRef)
        {

-           kern_close_file_for_direct_io(vars->fileRef);
-           gIOHibernateFileRef = vars->fileRef = NULL;
+           kern_close_file_for_direct_io(vars->fileRef, 0, 0, 0, 0, 0);
+           vars->fileRef = NULL;

        }
     }
        }
     }

+    else
+    {
+        WriteExtentsToFile(vars->fileRef, kIOHibernateHeaderOpenSignature, vars->blockSize, 
+                           (IOPolledFileExtent *)extentsData->getBytesNoCopy(),
+                           extentsData->getLength());
+    }

 
     if (part)
        part->release();
 
     if (part)
        part->release();


@@ -826,20 +1042,34 @@ IOPolledFileWrite(IOPolledFileIOVars * vars,
            uint32_t length = (vars->bufferOffset);
 
 #if CRYPTO
            uint32_t length = (vars->bufferOffset);
 
 #if CRYPTO

-            if (cryptvars && vars->encryptStart && (vars->position > vars->encryptStart))
+            if (cryptvars && vars->encryptStart
+                && (vars->position > vars->encryptStart)
+                && ((vars->position - length) < vars->encryptEnd))

             {
             {

-                uint32_t encryptLen, encryptStart;
+                AbsoluteTime startTime, endTime;
+
+                uint64_t encryptLen, encryptStart;

                 encryptLen = vars->position - vars->encryptStart;
                 if (encryptLen > length)
                     encryptLen = length;
                 encryptStart = length - encryptLen;
                 encryptLen = vars->position - vars->encryptStart;
                 if (encryptLen > length)
                     encryptLen = length;
                 encryptStart = length - encryptLen;

-                
+                if (vars->position > vars->encryptEnd)
+                    encryptLen -= (vars->position - vars->encryptEnd);
+
+                clock_get_uptime(&startTime);
+

                 // encrypt the buffer
                 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
                                 &cryptvars->aes_iv[0],
                                 encryptLen / AES_BLOCK_SIZE,
                                 vars->buffer + vars->bufferHalf + encryptStart,
                                 &cryptvars->ctx.encrypt);
                 // encrypt the buffer
                 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
                                 &cryptvars->aes_iv[0],
                                 encryptLen / AES_BLOCK_SIZE,
                                 vars->buffer + vars->bufferHalf + encryptStart,
                                 &cryptvars->ctx.encrypt);

+    
+                clock_get_uptime(&endTime);
+                ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
+                SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
+                vars->cryptBytes += encryptLen;
+

                 // save initial vector for following encrypts
                 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
                         &cryptvars->aes_iv[0],
                 // save initial vector for following encrypts
                 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
                         &cryptvars->aes_iv[0],


@@ -915,7 +1145,7 @@ IOPolledFileRead(IOPolledFileIOVars * vars,
        vars->bufferOffset += copy;
 //     vars->position += copy;
 
        vars->bufferOffset += copy;
 //     vars->position += copy;
 

-       if (vars->bufferOffset == vars->bufferLimit)
+       if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd))

        {
            if (vars->io)
             {
        {
            if (vars->io)
             {


@@ -928,9 +1158,9 @@ IOPolledFileRead(IOPolledFileIOVars * vars,
 
 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
 
 
 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
 

-           vars->position += vars->lastRead;
+           vars->position        += vars->lastRead;

            vars->extentRemaining -= vars->lastRead;
            vars->extentRemaining -= vars->lastRead;

-           vars->bufferLimit = vars->lastRead;
+           vars->bufferLimit      = vars->lastRead;

 
            if (!vars->extentRemaining)
            {
 
            if (!vars->extentRemaining)
            {


@@ -952,14 +1182,18 @@ if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n",
                length = vars->extentRemaining;
            else
                length = vars->bufferSize;
                length = vars->extentRemaining;
            else
                length = vars->bufferSize;

-           vars->lastRead = length;
+           if ((length + vars->position) > vars->readEnd)
+               length = vars->readEnd - vars->position;

 
 

+           vars->lastRead = length;
+           if (length)
+           {

 //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
 //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);

-
-           err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
-            if (kIOReturnSuccess != err)
-                break;
-           vars->io = true;
+               err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
+               if (kIOReturnSuccess != err)
+                   break;
+               vars->io = true;
+           }

 
            vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
            vars->bufferOffset = 0;
 
            vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
            vars->bufferOffset = 0;


@@ -968,16 +1202,26 @@ if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n",
             if (cryptvars)
             {
                 uint8_t thisVector[AES_BLOCK_SIZE];
             if (cryptvars)
             {
                 uint8_t thisVector[AES_BLOCK_SIZE];

+                AbsoluteTime startTime, endTime;
+

                 // save initial vector for following decrypts
                 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
                 bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE, 
                         &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
                 // save initial vector for following decrypts
                 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
                 bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE, 
                         &cryptvars->aes_iv[0], AES_BLOCK_SIZE);

+

                 // decrypt the buffer
                 // decrypt the buffer

+                clock_get_uptime(&startTime);
+

                 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
                                 &thisVector[0],
                                 lastReadLength / AES_BLOCK_SIZE,
                                 vars->buffer + vars->bufferHalf,
                                 &cryptvars->ctx.decrypt);
                 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
                                 &thisVector[0],
                                 lastReadLength / AES_BLOCK_SIZE,
                                 vars->buffer + vars->bufferHalf,
                                 &cryptvars->ctx.decrypt);

+
+                clock_get_uptime(&endTime);
+                ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
+                SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
+                vars->cryptBytes += lastReadLength;

             }
 #endif /* CRYPTO */
        }
             }
 #endif /* CRYPTO */
        }


@@ -988,82 +1232,178 @@ if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n",
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

-               
+
+#if HIBERNATION

 IOReturn
 IOReturn

-IOHibernateSystemSleep(void)
+IOHibernateOpenForDebugData( )

 {
 {

-    IOReturn   err;
-    OSData *   data;
-    OSObject * obj;
-    OSString * str;
-    OSNumber * num;
-    OSDictionary *sleepOverrideOptions;
+    dev_t       image_dev;
+    OSData      *extentsData = NULL;
+    OSObject    *obj;
+    OSString    *str;
+    IOByteCount blockSize = 0;
+    IOByteCount size;
+    IOService *                 part = 0;
+    OSData *   data = NULL;

 
 

-    IOHibernateVars * vars  = &gIOHibernateVars;
+    IOPolledFileExtent *     fileExtents;
+    IOReturn            err = kIOReturnSuccess;
+    IORegistryEntry * regEntry;
+    OSArray     * pollers = NULL;

 
 

-    if (vars->fileVars && vars->fileVars->fileRef)
-       // already on the way down
-       return (kIOReturnSuccess);
+    _OpenFileContext                       ctx;

 
 

-    gIOHibernateState = kIOHibernateStateInactive;
+    if (gDebugImageFileRef != NULL)
+        return kIOReturnError;

 
 

-    gIOHibernateDebugFlags = 0;
-    if (kIOLogHibernate & gIOKitDebug)
-       gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
+    if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
+    {
+        if ((str = OSDynamicCast(OSString, obj)))
+            strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
+                    sizeof(gIOHibernateFilename));
+        obj->release();
+    }

 
 

-    /* The invocation of IOPMSleepSystemWithOptions() may override
-     * existing hibernation settings.
-     */
-    sleepOverrideOptions = (OSDictionary *)OSDynamicCast( OSDictionary,
-        IOService::getPMRootDomain()->copyProperty(kRootDomainSleepOptionsKey));
+    if (!gIOHibernateFilename[0]) {
+        HIBLOG("Failed to get hibernate image filename\n");
+        return (kIOReturnUnsupported);
+    }

 
 

+    extentsData = OSData::withCapacity(32);
+    ctx.extents = extentsData;
+    ctx.size    = 0;

 
 

-    /* Hibernate mode overriden by sleep otions ? */
-    obj = NULL;
-    
-    if (sleepOverrideOptions) {
-        obj = sleepOverrideOptions->getObject(kIOHibernateModeKey);
-        if (obj) obj->retain();
+       bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
+       gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
+       gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
+
+    gDebugImageFileRef = kern_open_file_for_direct_io(gIOHibernateFilename, 
+                                                      false,
+                                                      &file_extent_callback, &ctx, 
+                                                      0, 0, 
+                                                      (caddr_t)gIOHibernateCurrentHeader, 
+                                                      sizeof(IOHibernateImageHeader),
+                                                      NULL, &image_dev, NULL, NULL, NULL);
+
+    if (gDebugImageFileRef == NULL) 
+    {
+        HIBLOG("Failed to open the file \n");
+        err = kIOReturnError;
+        goto exit;

     }
     }

+    fileExtents = (IOPolledFileExtent *)extentsData->getBytesNoCopy();
+    size = extentsData->getLength();

 
 

-    if(!obj) {
-        obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateModeKey);
+    part = IOCopyMediaForDev(image_dev);
+    if (!part)
+    {
+        HIBLOG("Failed to get the media device\n");
+        err = kIOReturnNotFound;
+        goto exit;

     }
 
     }
 

-    if (obj && (num = OSDynamicCast(OSNumber, obj)) )
+
+    pollers = OSArray::withCapacity(4);
+    if (!pollers)

     {
     {

-           gIOHibernateMode = num->unsigned32BitValue();
-        if (kIOHibernateModeSleep & gIOHibernateMode)
-            // default to discard clean for safe sleep
-            gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive 
-                                | kIOHibernateModeDiscardCleanActive);
+        err = kIOReturnNoMemory;
+        goto exit;

     }
 
     }
 

-    if (obj) obj->release();
-    
-    /* Hibernate free rotio overriden by sleep options ? */
-    obj = NULL;
-    
-    if (sleepOverrideOptions) {
-        obj = sleepOverrideOptions->getObject(kIOHibernateFreeRatioKey);
-        if (obj) obj->retain();
+    err = GetImageBlockSize(part, pollers, &blockSize);
+    if (err != kIOReturnSuccess)
+    {
+        HIBLOG("Failed to get block size\n");
+        goto exit;

     }
     }

-    
-    if(!obj) {
-        obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeRatioKey);
+    if (blockSize < sizeof(IOHibernateImageHeader))
+    {
+        HIBLOG("block size %llu is less than the size of the header\n", blockSize);
+        err = kIOReturnError;
+        goto exit;

     }
     }

-    if (obj && (num = OSDynamicCast(OSNumber, obj)))
+
+    WriteExtentsToFile(gDebugImageFileRef, kIOHibernateHeaderOpenSignature,
+                       blockSize, fileExtents, size);
+
+    char str2[24 + sizeof(uuid_string_t) + 2];
+
+    if (!gIOCreateEFIDevicePathSymbol)
+        gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
+
+    snprintf(str2, sizeof(str2), "%qx", fileExtents[0].start);
+
+    err = IOService::getPlatform()->callPlatformFunction(
+                                                         gIOCreateEFIDevicePathSymbol, false,
+                                                         (void *) part, (void *) str2,
+                                                         (void *) (uintptr_t) true, (void *) &data);
+
+    if (!gIOOptionsEntry)

     {
     {

-           gIOHibernateFreeRatio = num->unsigned32BitValue();
+        regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
+        gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
+        if (regEntry && !gIOOptionsEntry)
+            regEntry->release();

     }
     }

-    if (obj) obj->release();
-    
-    if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeTimeKey)))
+    if (gIOOptionsEntry)

     {
     {

-       if ((num = OSDynamicCast(OSNumber, obj)))
-           gIOHibernateFreeTime = num->unsigned32BitValue();
-       obj->release();
+        const OSSymbol *  sym;
+
+        sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
+        if (sym)
+        {
+            gIOOptionsEntry->setProperty(sym, data);
+            sym->release();
+        }
+    }
+
+
+exit:
+
+    if ( (err != kIOReturnSuccess) && gDebugImageFileRef) {
+        kern_close_file_for_direct_io(gDebugImageFileRef, 0, 0, 0, 0, 0);
+        gDebugImageFileRef = NULL;
+    }
+    if (extentsData) extentsData->release();
+    if (part) part->release();
+    if (pollers) pollers->release();
+    if (data) data->release();
+
+    return err;
+}
+#endif
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+IOReturn
+IOHibernateSystemSleep(void)
+{
+    IOReturn   err;
+    OSData *   data;
+    OSObject * obj;
+    OSString * str;
+    OSNumber * num;
+    bool       dsSSD, vmflush;
+    IOHibernateVars * vars;
+
+    gIOHibernateState = kIOHibernateStateInactive;
+
+    if (!gIOChosenEntry)
+       gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
+
+    gIOHibernateDebugFlags = 0;
+    if (kIOLogHibernate & gIOKitDebug)
+       gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
+
+    if (IOService::getPMRootDomain()->getHibernateSettings(
+        &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime))
+    {
+        if (kIOHibernateModeSleep & gIOHibernateMode)
+            // default to discard clean for safe sleep
+            gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive 
+                                | kIOHibernateModeDiscardCleanActive);

     }
     }

+

     if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
     {
        if ((str = OSDynamicCast(OSString, obj)))
     if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
     {
        if ((str = OSDynamicCast(OSString, obj)))


@@ -1072,67 +1412,170 @@ IOHibernateSystemSleep(void)
        obj->release();
     }
 
        obj->release();
     }
 

-    if (sleepOverrideOptions)
-        sleepOverrideOptions->release();
-

     if (!gIOHibernateMode || !gIOHibernateFilename[0])
        return (kIOReturnUnsupported);
 
     HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
 
     if (!gIOHibernateMode || !gIOHibernateFilename[0])
        return (kIOReturnUnsupported);
 
     HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
 

+    vars = IONew(IOHibernateVars, 1);
+    if (!vars) return (kIOReturnNoMemory);
+    bzero(vars, sizeof(*vars));
+
+    IOLockLock(gFSLock);
+    if (kFSIdle != gFSState)
+    {
+       HIBLOG("hibernate file busy\n");
+       IOLockUnlock(gFSLock);
+       IODelete(vars, IOHibernateVars, 1);
+        return (kIOReturnBusy);
+    }
+    gFSState = kFSOpening;
+    IOLockUnlock(gFSLock);

 
     do
     {
         vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
 
     do
     {
         vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,

-                                   4 * page_size, page_size);
+                                   2 * page_size + WKdm_SCRATCH_BUF_SIZE, page_size);

         vars->ioBuffer  = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, 
                                    2 * kDefaultIOSize, page_size);
 
         vars->ioBuffer  = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, 
                                    2 * kDefaultIOSize, page_size);
 

-        if (!vars->srcBuffer || !vars->ioBuffer)
+       vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, 
+                                   ptoa_64(gIOHibernateHandoffPageCount), page_size);
+
+        if (!vars->srcBuffer || !vars->ioBuffer || !vars->handoffBuffer)

         {
             err = kIOReturnNoMemory;
             break;
         }
 
         {
             err = kIOReturnNoMemory;
             break;
         }
 

-        err = IOPolledFileOpen(gIOHibernateFilename, vars->ioBuffer,
-                                &vars->fileVars, &vars->fileExtents, &data);
-        if (KERN_SUCCESS != err)
-        {
-           HIBLOG("IOPolledFileOpen(%x)\n", err);
-            break;
-        }
-       if (vars->fileVars->fileRef)
+       if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey)))
+       {
+           if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMinSize = num->unsigned64BitValue();
+           obj->release();
+       }
+       if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey)))

        {
        {

-           // invalidate the image file
-           gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
-           int err = kern_write_file(vars->fileVars->fileRef, 0,
-                                       (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
-            if (KERN_SUCCESS != err)
-                HIBLOG("kern_write_file(%d)\n", err);
+           if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMaxSize = num->unsigned64BitValue();
+           obj->release();

        }
 
        }
 

+        boolean_t encryptedswap = true;
+        uint32_t pageCount;
+        AbsoluteTime startTime, endTime;
+        uint64_t nsec;
+

        bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
        gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
        bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
        gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;

+       gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;

 
 

-        boolean_t encryptedswap;
-        err = hibernate_setup(gIOHibernateCurrentHeader, 
-                                gIOHibernateFreeRatio, gIOHibernateFreeTime,
-                                &vars->page_list, &vars->page_list_wired, &encryptedswap);
+       vmflush = (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey));
+       uint64_t setFileSize = 0;
+        err = hibernate_alloc_page_lists(&vars->page_list, 
+                                        &vars->page_list_wired,
+                                        &vars->page_list_pal);

         if (KERN_SUCCESS != err)
         if (KERN_SUCCESS != err)

-        {
-           HIBLOG("hibernate_setup(%d)\n", err);

             break;
             break;

-        }

 
 

-        if (encryptedswap)
-            gIOHibernateMode ^= kIOHibernateModeEncrypt; 
+       if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode))
+       {
+           hibernate_page_list_setall(vars->page_list,
+                                      vars->page_list_wired,
+                                      vars->page_list_pal,
+                                      true /* preflight */,
+                                      vmflush /* discard */,
+                                      &pageCount);
+           PE_Video consoleInfo;
+           bzero(&consoleInfo, sizeof(consoleInfo));
+           IOService::getPlatform()->getConsoleInfo(&consoleInfo);

 
 

-        vars->videoAllocSize = kVideoMapSize;
-        if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
-            vars->videoMapping = 0;
+           // estimate: 6% increase in pages compressed
+           // screen preview 2 images compressed 0%
+           setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
+                               + vars->page_list->list_size
+                               + (consoleInfo.v_width * consoleInfo.v_height * 8);
+           enum { setFileRound = 1024*1024ULL };
+           setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
+       
+           HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n", 
+                   pageCount, (100ULL * gIOHibernateCompression) >> 8,
+                   setFileSize, vars->fileMinSize);
+
+           if (!(kIOHibernateModeFileResize & gIOHibernateMode)
+            && (setFileSize < vars->fileMinSize))
+           { 
+               setFileSize = vars->fileMinSize;
+           }
+       }
+    
+       // open & invalidate the image file

 
 

-       // generate crypt keys
-        for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
+       if (gDebugImageFileRef) {
+           kern_close_file_for_direct_io(gDebugImageFileRef, 0, 0, 0, 0, 0);
+           gDebugImageFileRef = NULL;
+       }
+
+        err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, vars->ioBuffer,
+                                &vars->fileVars, &vars->fileExtents, &data, 
+                                &vars->volumeCryptKey[0]);
+
+        if (KERN_SUCCESS != err)
+        {
+           HIBLOG("IOPolledFileOpen(%x)\n", err);
+            break;
+        }
+
+        clock_get_uptime(&startTime);
+        err = hibernate_setup(gIOHibernateCurrentHeader, 
+                                gIOHibernateFreeRatio, gIOHibernateFreeTime,
+                                vmflush,
+                                vars->page_list, vars->page_list_wired, vars->page_list_pal);
+        clock_get_uptime(&endTime);
+        SUB_ABSOLUTETIME(&endTime, &startTime);
+        absolutetime_to_nanoseconds(endTime, &nsec);
+        HIBLOG("hibernate_setup(%d) took %qd ms\n", err, nsec / 1000000ULL);
+
+        dsSSD = ((0 != (kIOHibernateOptionSSD & vars->fileVars->flags))
+                && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
+        if (dsSSD)
+        {
+            gIOHibernateCurrentHeader->options |= 
+                                                kIOHibernateOptionSSD
+                                              | kIOHibernateOptionColor;
+
+#if defined(__i386__) || defined(__x86_64__)
+            if (!uuid_is_null(vars->volumeCryptKey) &&
+                  (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey)))
+            {
+                uintptr_t smcVars[2];
+                smcVars[0] = sizeof(vars->volumeCryptKey);
+                smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
+
+                IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
+                bzero(smcVars, sizeof(smcVars));
+            }
+#endif
+        }
+        else
+        {
+            gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
+        }
+
+
+        if (KERN_SUCCESS != err)
+            break;
+
+        if (encryptedswap || !uuid_is_null(vars->volumeCryptKey))
+            gIOHibernateMode ^= kIOHibernateModeEncrypt; 
+
+        if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
+        {
+            vars->videoAllocSize = kVideoMapSize;
+            if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
+                vars->videoMapping = 0;
+        }
+
+       // generate crypt keys
+        for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)

             vars->wiredCryptKey[i] = random();
         for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
             vars->cryptKey[i] = random();
             vars->wiredCryptKey[i] = random();
         for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
             vars->cryptKey[i] = random();


@@ -1147,8 +1590,6 @@ IOHibernateSystemSleep(void)
             if (regEntry && !gIOOptionsEntry)
                 regEntry->release();
         }
             if (regEntry && !gIOOptionsEntry)
                 regEntry->release();
         }

-        if (!gIOChosenEntry)
-            gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);

 
        if (gIOOptionsEntry)
        {
 
        if (gIOOptionsEntry)
        {


@@ -1162,45 +1603,6 @@ IOHibernateSystemSleep(void)
             }
             data->release();
 
             }
             data->release();
 

-#if defined(__ppc__)
-            size_t           len;
-            char              valueString[16];
-
-           vars->saveBootDevice = gIOOptionsEntry->copyProperty(kIOSelectedBootDeviceKey);
-            if (gIOChosenEntry)
-            {
-               OSData * bootDevice = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOBootPathKey));
-               if (bootDevice)
-               {
-                   sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
-                   OSString * str2 = OSString::withCStringNoCopy((const char *) bootDevice->getBytesNoCopy());
-                   if (sym && str2)
-                       gIOOptionsEntry->setProperty(sym, str2);
-                   if (sym)
-                       sym->release();
-                   if (str2)
-                       str2->release();
-               }
-                data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMemorySignatureKey));
-                if (data)
-                {
-                    vars->haveFastBoot = true;
-
-                    len = snprintf(valueString, sizeof(valueString), "0x%lx", *((UInt32 *)data->getBytesNoCopy()));
-                    data = OSData::withBytes(valueString, len + 1);
-                    sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
-                    if (sym && data)
-                        gIOOptionsEntry->setProperty(sym, data);
-                    if (sym)
-                        sym->release();
-                    if (data)
-                        data->release();
-                }
-                data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
-                if (data)
-                    gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
-            }
-#endif /* __ppc__ */

 #if defined(__i386__) || defined(__x86_64__)
            struct AppleRTCHibernateVars
            {
 #if defined(__i386__) || defined(__x86_64__)
            struct AppleRTCHibernateVars
            {


@@ -1241,29 +1643,73 @@ IOHibernateSystemSleep(void)
            data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
            if (data)
            { 
            data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
            if (data)
            { 

-                       IOService::getPMRootDomain()->setProperty(kIOHibernateRTCVariablesKey, data);
-               
-                       if( gIOOptionsEntry )
+               if (!gIOHibernateRTCVariablesKey)
+                   gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
+               if (gIOHibernateRTCVariablesKey)
+                   IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
+       
+               if( gIOOptionsEntry )
+               {
+                   if( gIOHibernateMode & kIOHibernateModeSwitch )
+                   {
+                       const OSSymbol *sym;
+                       sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSwitchVarsKey);
+                       if( sym )

                        {
                        {

-                               if( gIOHibernateMode & kIOHibernateModeSwitch )
-                               {
-                                       const OSSymbol *sym;
-                                       sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSwitchVarsKey);
-                                       if( sym )
-                                       {
-                                               gIOOptionsEntry->setProperty(sym, data); /* intentional insecure backup of rtc boot vars */
-                                               sym->release();
-                                       }
-                               }       
+                           gIOOptionsEntry->setProperty(sym, data); /* intentional insecure backup of rtc boot vars */
+                           sym->release();

                        }
                        }

+                   }   
+               }

 
 

-                       data->release();
+               data->release();

            }
             if (gIOChosenEntry)
             {
                 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
                 if (data)
                     gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
            }
             if (gIOChosenEntry)
             {
                 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
                 if (data)
                     gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());

+               {
+                   // set BootNext
+
+                   if (!gIOHibernateBoot0082Data)
+                   {
+                       data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
+                       if (data)
+                       {
+                           // AppleNVRAM_EFI_LOAD_OPTION
+                           struct {
+                               uint32_t Attributes;
+                               uint16_t FilePathLength;
+                               uint16_t Desc;
+                           } loadOptionHeader;
+                           loadOptionHeader.Attributes     = 1;
+                           loadOptionHeader.FilePathLength = data->getLength();
+                           loadOptionHeader.Desc           = 0;
+                           gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
+                           if (gIOHibernateBoot0082Data)
+                           {
+                               gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
+                               gIOHibernateBoot0082Data->appendBytes(data);
+                           }
+                       }
+                   }
+                   if (!gIOHibernateBoot0082Key)
+                       gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
+                   if (!gIOHibernateBootNextKey)
+                       gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
+                   if (!gIOHibernateBootNextData)
+                   {
+                       uint16_t bits = 0x0082;
+                       gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
+                   }
+                   if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData)
+                   {
+                       gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
+                       gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
+                       gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
+                   }
+               }

             }
 #else /* !i386 && !x86_64 */
             if (kIOHibernateModeEncrypt & gIOHibernateMode)
             }
 #else /* !i386 && !x86_64 */
             if (kIOHibernateModeEncrypt & gIOHibernateMode)


@@ -1323,10 +1769,31 @@ IOHibernateSystemSleep(void)
        }
        // --
 
        }
        // --
 

+    }
+    while (false);
+
+    IOLockLock(gFSLock);
+    if ((kIOReturnSuccess == err) && (kFSOpening == gFSState))
+    {
+       gFSState = kFSOpened;
+       gIOHibernateVars = *vars;
+       gFileVars = *vars->fileVars;
+       gIOHibernateVars.fileVars = &gFileVars;
+       gIOHibernateFileRef = gFileVars.fileRef;

        gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
        gIOHibernateState = kIOHibernateStateHibernating;
     }
        gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
        gIOHibernateState = kIOHibernateStateHibernating;
     }

-    while (false);
+    else
+    {
+       HIBLOG("hibernate file close due timeout\n");
+       if (vars->fileVars && vars->fileVars->fileRef) kern_close_file_for_direct_io(vars->fileVars->fileRef, 0, 0, 0, 0, 0);
+       IOHibernateDone(vars);
+       gFSState = kFSIdle;
+    }
+    IOLockUnlock(gFSLock);
+
+    if (vars->fileVars) IODelete(vars->fileVars, IOPolledFileIOVars, 1);
+    IODelete(vars, IOHibernateVars, 1);

 
     return (err);
 }
 
     return (err);
 }


@@ -1451,14 +1918,40 @@ ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBl
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 

+IOReturn
+IOHibernateIOKitSleep(void)
+{
+    IOReturn ret = kIOReturnSuccess;
+    IOLockLock(gFSLock);
+    if (kFSOpening == gFSState)
+    {
+       gFSState = kFSTimedOut;
+       HIBLOG("hibernate file open timed out\n");
+       ret = kIOReturnTimeout;
+    }
+    IOLockUnlock(gFSLock);
+    return (ret);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+

 IOReturn
 IOHibernateSystemHasSlept(void)
 {
 IOReturn
 IOHibernateSystemHasSlept(void)
 {

+    IOReturn          ret = kIOReturnSuccess;

     IOHibernateVars * vars  = &gIOHibernateVars;
     IOHibernateVars * vars  = &gIOHibernateVars;

-    OSObject        * obj;
+    OSObject        * obj = 0;

     OSData          * data;
 
     OSData          * data;
 

-    obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
+    IOLockLock(gFSLock);
+    if ((kFSOpened != gFSState) && gIOHibernateMode)
+    {
+       ret = kIOReturnTimeout;
+    }
+    IOLockUnlock(gFSLock);
+    if (kIOReturnSuccess != ret) return (ret);
+
+    if (gIOHibernateMode) obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);

     vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
     if (obj && !vars->previewBuffer)
        obj->release();
     vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
     if (obj && !vars->previewBuffer)
        obj->release();


@@ -1470,7 +1963,9 @@ IOHibernateSystemHasSlept(void)
        vars->previewBuffer = 0;
     }
 
        vars->previewBuffer = 0;
     }
 

-    if (vars->previewBuffer && (data = OSDynamicCast(OSData, 
+    if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
+        && vars->previewBuffer 
+        && (data = OSDynamicCast(OSData, 

        IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
     {
        UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
        IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
     {
        UInt32 flags = *((UInt32 *)data->getBytesNoCopy());


@@ -1492,8 +1987,8 @@ IOHibernateSystemHasSlept(void)
            vars->consoleMapping   = (uint8_t *) consoleInfo.v_baseAddr;
 
            HIBPRINT("video %p %d %d %d\n",
            vars->consoleMapping   = (uint8_t *) consoleInfo.v_baseAddr;
 
            HIBPRINT("video %p %d %d %d\n",

-                       vars->consoleMapping, gIOHibernateGraphicsInfo->depth, 
-                       gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height);
+                       vars->consoleMapping, graphicsInfo->depth, 
+                       graphicsInfo->width, graphicsInfo->height);

            if (vars->consoleMapping)
                        ProgressInit(graphicsInfo, vars->consoleMapping,
                                        &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
            if (vars->consoleMapping)
                        ProgressInit(graphicsInfo, vars->consoleMapping,
                                        &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));


@@ -1503,17 +1998,64 @@ IOHibernateSystemHasSlept(void)
     if (gIOOptionsEntry)
         gIOOptionsEntry->sync();
 
     if (gIOOptionsEntry)
         gIOOptionsEntry->sync();
 

-    return (kIOReturnSuccess);
+    return (ret);

 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 

+static DeviceTreeNode *
+MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry)
+{
+    DeviceTreeNodeProperty * prop;
+    DeviceTreeNode *         child;
+    IORegistryEntry *        childRegEntry;
+    const char *             nameProp;
+    unsigned int             propLen, idx;
+
+    prop = (DeviceTreeNodeProperty *) (entry + 1);
+    for (idx = 0; idx < entry->nProperties; idx++)
+    {
+       if (regEntry && (0 != strcmp("name", prop->name)))
+       {
+           regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
+//         HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
+       }
+       prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
+    }
+
+    child = (DeviceTreeNode *) prop;
+    for (idx = 0; idx < entry->nChildren; idx++)
+    {
+       if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen))
+           panic("no name");
+       childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
+//     HIBPRINT("%s == %p\n", nameProp, childRegEntry);
+       child = MergeDeviceTree(child, childRegEntry);
+    }
+    return (child);
+}
+

 IOReturn
 IOHibernateSystemWake(void)
 {
 IOReturn
 IOHibernateSystemWake(void)
 {

-    IOHibernateVars * vars  = &gIOHibernateVars;
+    if (kFSOpened == gFSState)
+    {
+       IOHibernateDone(&gIOHibernateVars);
+    }
+    else
+    {
+        IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
+        IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
+    }
+    return (kIOReturnSuccess);
+}
+
+static IOReturn
+IOHibernateDone(IOHibernateVars * vars)
+{
+    IORegistryEntry * next;

 
 

-    hibernate_teardown(vars->page_list, vars->page_list_wired);
+    hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);

 
     if (vars->videoMapping)
     {
 
     if (vars->videoMapping)
     {


@@ -1531,86 +2073,69 @@ IOHibernateSystemWake(void)
         vars->previewBuffer = 0;
     }
 
         vars->previewBuffer = 0;
     }
 

-    if (vars->fileVars)
+    if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)

     {
     {

-       IOPolledFileClose(vars->fileVars);
+        IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey, 
+                                            gIOHibernateCurrentHeader->options, 32);
+    }
+    else
+    {
+        IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);

     }
 
     }
 

-    // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
-
-#ifdef __ppc__
-    OSData * data = OSData::withCapacity(4);
-    if (gIOOptionsEntry && data)
+    if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
+      && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus))

     {
     {

-        const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
-        if (sym)
-        {
-            gIOOptionsEntry->setProperty(sym, data);
-            sym->release();
-        }
-        sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
-        if (sym)
-        {
-           if (vars->saveBootDevice)
-           {
-               gIOOptionsEntry->setProperty(sym, vars->saveBootDevice);
-               vars->saveBootDevice->release();
-           }
-            sym->release();
-        }
-        sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
-        if (sym)
-        {
-            gIOOptionsEntry->setProperty(sym, data);
-            sym->release();
-        }
-        sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
-        if (sym)
-        {
-            gIOOptionsEntry->removeProperty(sym);
-            sym->release();
-        }
+        IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey, 
+                                        &gIOHibernateGraphicsInfo->gfxStatus,
+                                        sizeof(gIOHibernateGraphicsInfo->gfxStatus));
+    }
+    else
+    {
+        IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);

     }
     }

-    if (data)
-        data->release();

 
 

-    if (gIOOptionsEntry)
+    if (vars->fileVars)

     {
     {

-       if (!vars->haveFastBoot)
+       if ((next = vars->fileVars->media)) do

        {
        {

-           // reset boot audio volume
-           IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
-           if (platform)
-               platform->writeXPRAM(kXPRamAudioVolume, 
-                                       &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume));
+           next->removeProperty(kIOPolledInterfaceActiveKey);
+           next = next->getParentEntry(gIOServicePlane);

        }
        }

-
-       // sync now to hardware if the booter has not
-       if (kIOHibernateStateInactive == gIOHibernateState)
-           gIOOptionsEntry->sync();
-       else
-           // just sync the variables in case a later panic syncs nvram (it won't sync variables)
-           gIOOptionsEntry->syncOFVariables();
+       while (next);
+       IOPolledFileClose(vars->fileVars);

     }
     }

-#endif
+
+    // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched

 
 #if defined(__i386__) || defined(__x86_64__)
 
 #if defined(__i386__) || defined(__x86_64__)

-       IOService::getPMRootDomain()->removeProperty(kIOHibernateRTCVariablesKey);
+       IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
+       IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);

 
        /*
         * Hibernate variable is written to NVRAM on platforms in which RtcRam
         * is not backed by coin cell.  Remove Hibernate data from NVRAM.
         */
        if (gIOOptionsEntry) {
 
        /*
         * Hibernate variable is written to NVRAM on platforms in which RtcRam
         * is not backed by coin cell.  Remove Hibernate data from NVRAM.
         */
        if (gIOOptionsEntry) {

-               const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);

 
 

-               if (sym) {
-                       if (gIOOptionsEntry->getProperty(sym)) {
-                               gIOOptionsEntry->removeProperty(sym);
-                               gIOOptionsEntry->sync();
-                       }
-                       sym->release();
+           if (gIOHibernateRTCVariablesKey) {
+               if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
+                   gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);

                }
                }

+           }
+
+           if (gIOHibernateBootNextKey)
+           {
+               if (gIOHibernateBootNextSave)
+               {
+                   gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
+                   gIOHibernateBootNextSave->release();
+                   gIOHibernateBootNextSave = NULL;
+               }
+               else
+                   gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
+           }
+           if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) gIOOptionsEntry->sync();

        }
 #endif
 
        }
 #endif
 


@@ -1618,6 +2143,50 @@ IOHibernateSystemWake(void)
        vars->srcBuffer->release();
     if (vars->ioBuffer)
        vars->ioBuffer->release();
        vars->srcBuffer->release();
     if (vars->ioBuffer)
        vars->ioBuffer->release();

+    bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
+    if (vars->handoffBuffer)
+    {
+       if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
+       {
+           IOHibernateHandoff * handoff;
+           bool done = false;
+           for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
+                !done;
+                handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
+           {
+               HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
+               uint8_t * data = &handoff->data[0];
+               switch (handoff->type)
+               {
+                   case kIOHibernateHandoffTypeEnd:
+                       done = true;
+                       break;
+
+                   case kIOHibernateHandoffTypeDeviceTree:
+                       MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
+                       break;
+       
+                   case kIOHibernateHandoffTypeKeyStore:
+#if defined(__i386__) || defined(__x86_64__)
+                       {
+                           IOBufferMemoryDescriptor *
+                           md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
+                           if (md)
+                           {
+                               IOSetKeyStoreData(md);
+                           }
+                       }
+#endif
+                       break;
+       
+                   default:
+                       done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
+                       break;
+               }    
+           }
+       }
+       vars->handoffBuffer->release();
+    }

     if (vars->fileExtents)
        vars->fileExtents->release();
 
     if (vars->fileExtents)
        vars->fileExtents->release();
 


@@ -1631,32 +2200,109 @@ IOHibernateSystemWake(void)
 IOReturn
 IOHibernateSystemPostWake(void)
 {
 IOReturn
 IOHibernateSystemPostWake(void)
 {

-    if (gIOHibernateFileRef)
+    struct kern_direct_file_io_ref_t * fileRef;
+
+    if (kFSOpened == gFSState)

     {
     {

-       // invalidate the image file
+       // invalidate & close the image file

        gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
        gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;

-       int err = kern_write_file(gIOHibernateFileRef, 0,
-                                   (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
-       if (KERN_SUCCESS != err)
-           HIBLOG("kern_write_file(%d)\n", err);
+       if ((fileRef = gIOHibernateFileRef))
+       {
+           gIOHibernateFileRef = 0;
+           IOSleep(TRIM_DELAY);
+           kern_close_file_for_direct_io(fileRef,
+#if DISABLE_TRIM
+                                      0, 0, 0, 0, 0);
+#else
+                                      0, (caddr_t) gIOHibernateCurrentHeader, 
+                                      sizeof(IOHibernateImageHeader),
+                                      0,
+                                      gIOHibernateCurrentHeader->imageSize);
+#endif
+       }
+       gFSState = kFSIdle;
+    }
+
+    if (gDebugImageFileRef) {
+        kern_close_file_for_direct_io(gDebugImageFileRef, 0, 0, 0, 0, 0);
+        gDebugImageFileRef = NULL;
+    }
+
+    if (!gIOOptionsEntry)
+    {
+        IORegistryEntry * regEntry;
+        regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
+        gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
+        if (regEntry && !gIOOptionsEntry)
+            regEntry->release();
+    }
+    if (gIOOptionsEntry)
+    {
+        const OSSymbol *  sym;

 
 

-       kern_close_file_for_direct_io(gIOHibernateFileRef);
-        gIOHibernateFileRef = 0;
+        sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
+        if (sym)
+        {
+            gIOOptionsEntry->removeProperty(sym);
+            gIOOptionsEntry->sync();
+            sym->release();
+        }

     }
     }

+

     return (kIOReturnSuccess);
 }
 
     return (kIOReturnSuccess);
 }
 

+bool IOHibernateWasScreenLocked(void)
+{
+    bool ret = false;
+    if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry)
+    {
+       OSData *
+       data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
+       if (data) switch (*((uint32_t *)data->getBytesNoCopy()))
+       {
+           case kIOScreenLockLocked:
+           case kIOScreenLockFileVaultDialog:
+               ret = true;
+               break;
+           case kIOScreenLockNoLock:
+           case kIOScreenLockUnlocked:
+           default:
+               ret = false;
+               break;
+       }
+    }
+    return (ret);
+}
+

 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 SYSCTL_STRING(_kern, OID_AUTO, hibernatefile, 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 SYSCTL_STRING(_kern, OID_AUTO, hibernatefile, 

-               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, 
+               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,

                gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
 SYSCTL_STRING(_kern, OID_AUTO, bootsignature, 
                gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
 SYSCTL_STRING(_kern, OID_AUTO, bootsignature, 

-               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, 
+               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,

                gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
 SYSCTL_UINT(_kern, OID_AUTO, hibernatemode, 
                gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
 SYSCTL_UINT(_kern, OID_AUTO, hibernatemode, 

-               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, 
+               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,

                &gIOHibernateMode, 0, "");
                &gIOHibernateMode, 0, "");

+SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
+               CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
+               gIOHibernateStats, hibernate_statistics_t, "");
+
+SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
+               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+               &gIOHibernateStats->graphicsReadyTime, 0, "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
+               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+               &gIOHibernateStats->wakeNotificationTime, 0, "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
+               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+               &gIOHibernateStats->lockScreenReadyTime, 0, "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
+               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+               &gIOHibernateStats->hidReadyTime, 0, "");
+

 
 void
 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
 
 void
 IOHibernateSystemInit(IOPMrootDomain * rootDomain)


@@ -1676,6 +2322,13 @@ IOHibernateSystemInit(IOPMrootDomain * rootDomain)
     sysctl_register_oid(&sysctl__kern_hibernatefile);
     sysctl_register_oid(&sysctl__kern_bootsignature);
     sysctl_register_oid(&sysctl__kern_hibernatemode);
     sysctl_register_oid(&sysctl__kern_hibernatefile);
     sysctl_register_oid(&sysctl__kern_bootsignature);
     sysctl_register_oid(&sysctl__kern_hibernatemode);

+    sysctl_register_oid(&sysctl__kern_hibernatestatistics);
+    sysctl_register_oid(&sysctl__kern_hibernategraphicsready);
+    sysctl_register_oid(&sysctl__kern_hibernatewakenotification);
+    sysctl_register_oid(&sysctl__kern_hibernatelockscreenready);
+    sysctl_register_oid(&sysctl__kern_hibernatehidready);
+
+    gFSLock = IOLockAlloc();

 }
 
 
 }
 
 


@@ -1684,16 +2337,50 @@ IOHibernateSystemInit(IOPMrootDomain * rootDomain)
 static void
 hibernate_setup_for_wake(void)
 {
 static void
 hibernate_setup_for_wake(void)
 {

-#if __ppc__
-    // go slow (state needed for wake)
-    ml_set_processor_speed(1);
-#endif

 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 #define C_ASSERT(e) typedef char    __C_ASSERT__[(e) ? 1 : -1]
 
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 #define C_ASSERT(e) typedef char    __C_ASSERT__[(e) ? 1 : -1]
 

+static bool
+no_encrypt_page(vm_offset_t ppnum)
+{
+    if (pmap_is_noencrypt((ppnum_t)ppnum) == TRUE)
+    {
+        return true;
+    }
+    return false;
+}
+
+static void
+hibernate_pal_callback(void *vars_arg, vm_offset_t addr)
+{
+       IOHibernateVars *vars = (IOHibernateVars *)vars_arg;
+       /* Make sure it's not in either of the save lists */
+       hibernate_set_page_state(vars->page_list, vars->page_list_wired, atop_64(addr), 1, kIOHibernatePageStateFree);
+
+       /* Set it in the bitmap of pages owned by the PAL */
+       hibernate_page_bitset(vars->page_list_pal, TRUE, atop_64(addr));
+}
+
+static struct hibernate_cryptvars_t *local_cryptvars;
+
+extern "C" int
+hibernate_pal_write(void *buffer, size_t size)
+{
+    IOHibernateVars * vars  = &gIOHibernateVars;
+
+       IOReturn err = IOPolledFileWrite(vars->fileVars, (const uint8_t *)buffer, size, local_cryptvars);
+       if (kIOReturnSuccess != err) {
+               kprintf("epic hibernate fail! %d\n", err);
+               return err;
+       }
+
+       return 0;
+}
+
+

 extern "C" uint32_t
 hibernate_write_image(void)
 {
 extern "C" uint32_t
 hibernate_write_image(void)
 {


@@ -1705,35 +2392,60 @@ hibernate_write_image(void)
 
     uint32_t    pageCount, pagesDone;
     IOReturn     err;
 
     uint32_t    pageCount, pagesDone;
     IOReturn     err;

-    vm_offset_t  ppnum;
-    IOItemCount  page, count;
+    vm_offset_t  ppnum, page;
+    IOItemCount  count;

     uint8_t *   src;
     uint8_t *   data;
     uint8_t *   src;
     uint8_t *   data;

-    IOByteCount  pageCompressedSize;
+    uint8_t *   compressed;
+    uint8_t *   scratch;
+    void *       zerosCompressed;
+    IOByteCount  pageCompressedSize, zerosCompressedLen;

     uint64_t    compressedSize, uncompressedSize;
     uint64_t    image1Size = 0;
     uint32_t    bitmap_size;
     uint64_t    compressedSize, uncompressedSize;
     uint64_t    image1Size = 0;
     uint32_t    bitmap_size;

-    bool        iterDone, pollerOpen, needEncryptStart;
+    bool        iterDone, pollerOpen, needEncrypt;

     uint32_t    restore1Sum, sum, sum1, sum2;
     uint32_t    restore1Sum, sum, sum1, sum2;

+    int          wkresult;

     uint32_t    tag;
     uint32_t    pageType;
     uint32_t    pageAndCount[2];
     uint32_t    tag;
     uint32_t    pageType;
     uint32_t    pageAndCount[2];

+    addr64_t     phys64;
+    IOByteCount  segLen;

 
     AbsoluteTime startTime, endTime;
 
     AbsoluteTime startTime, endTime;

-    AbsoluteTime allTime, compTime, decoTime;
+    AbsoluteTime allTime, compTime;
+    uint64_t     compBytes;

     uint64_t     nsec;
     uint32_t     lastProgressStamp = 0;
     uint32_t     progressStamp;
     uint32_t    blob, lastBlob = (uint32_t) -1L;
 
     uint64_t     nsec;
     uint32_t     lastProgressStamp = 0;
     uint32_t     progressStamp;
     uint32_t    blob, lastBlob = (uint32_t) -1L;
 

+    uint32_t    wiredPagesEncrypted;
+    uint32_t    dirtyPagesEncrypted;
+    uint32_t    wiredPagesClear;
+    uint32_t    zeroPageCount;
+

     hibernate_cryptvars_t _cryptvars;
     hibernate_cryptvars_t * cryptvars = 0;
 
     hibernate_cryptvars_t _cryptvars;
     hibernate_cryptvars_t * cryptvars = 0;
 

+    wiredPagesEncrypted = 0;
+    dirtyPagesEncrypted = 0;
+    wiredPagesClear     = 0;
+    zeroPageCount       = 0;
+

     if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
         return (false /* sleep */ );
 
     if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
         return (false /* sleep */ );
 

+    if (kIOHibernateModeSleep & gIOHibernateMode)
+       kdebug_enable = save_kdebug_enable;
+
+    KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START, 0, 0, 0, 0, 0);
+    IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
+

     restore1Sum = sum1 = sum2 = 0;
 
     restore1Sum = sum1 = sum2 = 0;
 

+    hibernate_pal_prepare();
+

 #if CRYPTO
     // encryption data. "iv" is the "initial vector".
     if (kIOHibernateModeEncrypt & gIOHibernateMode)
 #if CRYPTO
     // encryption data. "iv" is the "initial vector".
     if (kIOHibernateModeEncrypt & gIOHibernateMode)


@@ -1753,6 +2465,9 @@ hibernate_write_image(void)
 
         cryptvars = &_cryptvars;
         bzero(cryptvars, sizeof(hibernate_cryptvars_t));
 
         cryptvars = &_cryptvars;
         bzero(cryptvars, sizeof(hibernate_cryptvars_t));

+        for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++)
+            vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
+        bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));

         aes_encrypt_key(vars->wiredCryptKey,
                         kIOHibernateAESKeySize,
                         &cryptvars->ctx.encrypt);
         aes_encrypt_key(vars->wiredCryptKey,
                         kIOHibernateAESKeySize,
                         &cryptvars->ctx.encrypt);


@@ -1760,7 +2475,8 @@ hibernate_write_image(void)
         bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
         bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
         bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
         bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
         bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
         bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));

-        bzero(gIOHibernateCryptWakeVars, sizeof(hibernate_cryptwakevars_t));
+
+        local_cryptvars = cryptvars;

     }
 #endif /* CRYPTO */
 
     }
 #endif /* CRYPTO */
 


@@ -1768,6 +2484,11 @@ hibernate_write_image(void)
 
     hibernate_page_list_setall(vars->page_list,
                                vars->page_list_wired,
 
     hibernate_page_list_setall(vars->page_list,
                                vars->page_list_wired,

+                               vars->page_list_pal,
+                              false /* !preflight */,
+                              /* discard_all */
+                              ((0 == (kIOHibernateModeSleep & gIOHibernateMode)) 
+                              && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),

                                &pageCount);
 
     HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
                                &pageCount);
 
     HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);


@@ -1784,23 +2505,20 @@ hibernate_write_image(void)
     }
 #endif
 
     }
 #endif
 

-    needEncryptStart = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
-
+    needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));

     AbsoluteTime_to_scalar(&compTime) = 0;
     AbsoluteTime_to_scalar(&compTime) = 0;

-    AbsoluteTime_to_scalar(&decoTime) = 0;
+    compBytes = 0;

 
     clock_get_uptime(&allTime);
     IOService::getPMRootDomain()->pmStatsRecordEvent( 
                         kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
 
     clock_get_uptime(&allTime);
     IOService::getPMRootDomain()->pmStatsRecordEvent( 
                         kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);

-

     do 
     {
         compressedSize   = 0;
         uncompressedSize = 0;
     do 
     {
         compressedSize   = 0;
         uncompressedSize = 0;

-        iterDone         = false;
-        pageType         = 0;          // wired pages first
+        zeroPageCount    = 0;

 
 

-        IOPolledFileSeek(vars->fileVars, sizeof(IOHibernateImageHeader));
+        IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);

     
         HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n", 
                 ml_get_interrupts_enabled());
     
         HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n", 
                 ml_get_interrupts_enabled());


@@ -1825,35 +2543,43 @@ hibernate_write_image(void)
         uintptr_t hibernateBase;
         uintptr_t hibernateEnd;
 
         uintptr_t hibernateBase;
         uintptr_t hibernateEnd;
 

-#if defined(__i386__) || defined(__x86_64__)
-        hibernateBase = sectINITPTB;
-#else
-        hibernateBase = sectHIBB;
-#endif
+        hibernateBase = HIB_BASE; /* Defined in PAL headers */
+
+        hibernateEnd = (segHIBB + segSizeHIB);

 
 

-        hibernateEnd = (sectHIBB + sectSizeHIB);

         // copy out restore1 code
         // copy out restore1 code

-    
-        page = atop_32(hibernateBase);
-        count = atop_32(round_page(hibernateEnd)) - page;
-        header->restore1CodePage = page;
+
+        for (count = 0;
+            (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+            count += segLen)
+        {
+           for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++)
+           {
+               gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
+           }
+        }
+
+        page = atop_32(kvtophys(hibernateBase));
+        count = atop_32(round_page(hibernateEnd) - hibernateBase);
+        header->restore1CodePhysPage = page;
+        header->restore1CodeVirt = hibernateBase;

         header->restore1PageCount = count;
         header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint)      - hibernateBase;
         header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
 
         header->restore1PageCount = count;
         header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint)      - hibernateBase;
         header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
 

-        // sum __HIB sect, with zeros for the stack
+        // sum __HIB seg, with zeros for the stack

         src = (uint8_t *) trunc_page(hibernateBase);
         for (page = 0; page < count; page++)
         {
             if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
         src = (uint8_t *) trunc_page(hibernateBase);
         for (page = 0; page < count; page++)
         {
             if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))

-                restore1Sum += hibernate_sum(src, page_size);
+                restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);

             else
             else

-                restore1Sum += 0x10000001;
+                restore1Sum += 0x00000000;

             src += page_size;
         }
         sum1 = restore1Sum;
     
             src += page_size;
         }
         sum1 = restore1Sum;
     

-        // write the __HIB sect, with zeros for the stack
+        // write the __HIB seg, with zeros for the stack

 
         src = (uint8_t *) trunc_page(hibernateBase);
         count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
 
         src = (uint8_t *) trunc_page(hibernateBase);
         count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);


@@ -1878,10 +2604,14 @@ hibernate_write_image(void)
                 break;
         }
 
                 break;
         }
 

-        // write the preview buffer
+       if (kIOHibernateModeEncrypt & gIOHibernateMode)
+       {
+           vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
+           vars->fileVars->encryptEnd   = UINT64_MAX;
+           HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
+       }

 
 

-        addr64_t phys64;
-        IOByteCount segLen;
+        // write the preview buffer

 
         if (vars->previewBuffer)
         {
 
         if (vars->previewBuffer)
         {


@@ -1905,14 +2635,19 @@ hibernate_write_image(void)
                 break;
 
             src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
                 break;
 
             src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);

+
+                       ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
+

             count = vars->previewBuffer->getLength();
 
             header->previewPageListSize = ppnum;
             header->previewSize = count + ppnum;
 
             for (page = 0; page < count; page += page_size)
             count = vars->previewBuffer->getLength();
 
             header->previewPageListSize = ppnum;
             header->previewSize = count + ppnum;
 
             for (page = 0; page < count; page += page_size)

-                sum1 += hibernate_sum(src + page, page_size);
-
+            {
+                phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
+                sum1 += hibernate_sum_page(src + page, atop_64(phys64));
+            }

             err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
             if (kIOReturnSuccess != err)
                 break;
             err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
             if (kIOReturnSuccess != err)
                 break;


@@ -1953,8 +2688,9 @@ hibernate_write_image(void)
 
        hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
     
 
        hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
     

-        page = atop_32(hibernateBase);
-        count = atop_32(round_page(hibernateEnd)) - page;
+
+        page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
+        count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;

         hibernate_set_page_state(vars->page_list, vars->page_list_wired,
                                         page, count,
                                         kIOHibernatePageStateFree);
         hibernate_set_page_state(vars->page_list, vars->page_list_wired,
                                         page, count,
                                         kIOHibernatePageStateFree);


@@ -1970,142 +2706,227 @@ hibernate_write_image(void)
             pageCount -= atop_32(segLen);
         }
 
             pageCount -= atop_32(segLen);
         }
 

-        src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
-    
-        ppnum     = 0;
-        pagesDone = 0;
-        lastBlob = 0;
-    
-        HIBLOG("writing %d pages\n", pageCount);
-
-        do
+        for (count = 0;
+            (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+            count += segLen)

         {
         {

-            count = hibernate_page_list_iterate(pageType ? vars->page_list : vars->page_list_wired,
-                                                    &ppnum);
-//          kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
-    
-            iterDone = !count;
+            hibernate_set_page_state(vars->page_list, vars->page_list_wired, 
+                                        atop_64(phys64), atop_32(segLen),
+                                        kIOHibernatePageStateFree);
+            pageCount -= atop_32(segLen);
+        }

 
 

-            pageAndCount[0] = ppnum;
-            pageAndCount[1] = count;
-            err = IOPolledFileWrite(vars->fileVars, 
-                                    (const uint8_t *) &pageAndCount, sizeof(pageAndCount), 
-                                    cryptvars);
-            if (kIOReturnSuccess != err)
-                break;
+               (void)hibernate_pal_callback;

 
 

-            for (page = 0; page < count; page++)
+        src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
+       compressed = src + page_size;
+        scratch    = compressed + page_size;
+
+       // compress a zero page
+       bzero(src, page_size);
+       zerosCompressed    = vars->handoffBuffer->getBytesNoCopy();
+       zerosCompressedLen = WKdm_compress_new((WK_word*) src,
+                                              (WK_word*) zerosCompressed, 
+                                              (WK_word*) scratch,
+                                              page_size - 4);
+
+        pagesDone  = 0;
+        lastBlob   = 0;
+    
+        HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n", 
+               bitmap_size, header->previewSize,
+               pageCount, vars->fileVars->position);
+
+        enum
+        // pageType
+        { 
+            kWired          = 0x02,
+            kEncrypt        = 0x01,
+            kWiredEncrypt   = kWired | kEncrypt,
+            kWiredClear     = kWired,
+            kUnwiredEncrypt = kEncrypt
+        };
+
+        for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)
+        {
+           if (kUnwiredEncrypt == pageType)
+          {
+               // start unwired image
+               if (kIOHibernateModeEncrypt & gIOHibernateMode)
+               {
+                   vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
+                   vars->fileVars->encryptEnd   = UINT64_MAX;
+                   HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
+               }
+               bcopy(&cryptvars->aes_iv[0], 
+                       &gIOHibernateCryptWakeContext.aes_iv[0], 
+                       sizeof(cryptvars->aes_iv));
+               cryptvars = &gIOHibernateCryptWakeContext;
+            }
+            for (iterDone = false, ppnum = 0; !iterDone; )

             {
             {

-                err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(ppnum), page_size);
-                if (err)
+                count = hibernate_page_list_iterate((kWired & pageType) 
+                                                            ? vars->page_list_wired : vars->page_list,
+                                                        &ppnum);
+//              kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
+                iterDone = !count;
+    
+                if (count && (kWired & pageType) && needEncrypt)

                 {
                 {

-                    HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)ppnum, err);
-                    break;
+                    uint32_t checkIndex;
+                    for (checkIndex = 0;
+                            (checkIndex < count) 
+                                && (((kEncrypt & pageType) == 0) == no_encrypt_page(ppnum + checkIndex)); 
+                            checkIndex++)
+                    {}
+                    if (!checkIndex)
+                    {
+                        ppnum++;
+                        continue;
+                    }
+                    count = checkIndex;

                 }
                 }

-    
-                sum = hibernate_sum(src, page_size);
-   
-                clock_get_uptime(&startTime);

 
 

-                pageCompressedSize = WKdm_compress ((WK_word*) src, (WK_word*) (src + page_size), PAGE_SIZE_IN_WORDS);
-    
-                clock_get_uptime(&endTime);
-                ADD_ABSOLUTETIME(&compTime, &endTime);
-                SUB_ABSOLUTETIME(&compTime, &startTime);
-    
-                if (kIOHibernateModeEncrypt & gIOHibernateMode)
-                    pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
-
-                if (pageCompressedSize > page_size)
+                switch (pageType)

                 {
                 {

-//                  HIBLOG("------------lose: %d\n", pageCompressedSize);
-                    pageCompressedSize = page_size;
+                    case kWiredEncrypt:   wiredPagesEncrypted += count; break;
+                    case kWiredClear:     wiredPagesClear     += count; break;
+                    case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;

                 }
                 }

-
-                if (pageCompressedSize != page_size)
-                    data = (src + page_size);
-                else
-                    data = src;
-
-                tag = pageCompressedSize | kIOHibernateTagSignature;
-
-                if (pageType)
-                    sum2 += sum;
+    
+                if (iterDone && (kWiredEncrypt == pageType))   {/* not yet end of wired list */}

                 else
                 else

-                    sum1 += sum;
-
-                if (needEncryptStart && (ppnum >= atop_32(sectDATAB)))

                 {
                 {

-                    // start encrypting partway into the data about to be written
-                    vars->fileVars->encryptStart = (vars->fileVars->position + AES_BLOCK_SIZE - 1) 
-                                                    & ~(AES_BLOCK_SIZE - 1);
-                    needEncryptStart = false;
+                    pageAndCount[0] = ppnum;
+                    pageAndCount[1] = count;
+                    err = IOPolledFileWrite(vars->fileVars, 
+                                            (const uint8_t *) &pageAndCount, sizeof(pageAndCount), 
+                                            cryptvars);
+                    if (kIOReturnSuccess != err)
+                        break;

                 }
                 }

+    
+                for (page = ppnum; page < (ppnum + count); page++)
+                {
+                    err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
+                    if (err)
+                    {
+                        HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
+                        break;
+                    }
+        
+                    sum = hibernate_sum_page(src, page);
+                    if (kWired & pageType)
+                        sum1 += sum;
+                    else
+                        sum2 += sum;
+       
+                    clock_get_uptime(&startTime);
+                    wkresult = WKdm_compress_new((WK_word*) src,
+                                                (WK_word*) compressed, 
+                                                (WK_word*) scratch,
+                                                page_size - 4);

 
 

-                err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
-                if (kIOReturnSuccess != err)
-                    break;
-
-                err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
-                if (kIOReturnSuccess != err)
-                    break;
+                    clock_get_uptime(&endTime);
+                    ADD_ABSOLUTETIME(&compTime, &endTime);
+                    SUB_ABSOLUTETIME(&compTime, &startTime);

 
 

-                compressedSize += pageCompressedSize;
-                if (pageCompressedSize)
-                    uncompressedSize += page_size;
-                ppnum++;
-                pagesDone++;
+                    compBytes += page_size;
+                    pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;

 
 

-               if (vars->consoleMapping && (0 == (1023 & pagesDone)))
-               {
-                   blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
-                   if (blob != lastBlob)
+                   if ((pageCompressedSize == zerosCompressedLen) 
+                    && !bcmp(compressed, zerosCompressed, zerosCompressedLen))

                    {
                    {

-                       ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
-                       lastBlob = blob;
+                       pageCompressedSize = 0;
+                       zeroPageCount++;

                    }
                    }

-               }
-                if (0 == (8191 & pagesDone))
-                {
-                    clock_get_uptime(&endTime);
-                    SUB_ABSOLUTETIME(&endTime, &allTime);
-                    absolutetime_to_nanoseconds(endTime, &nsec);
-                    progressStamp = nsec / 750000000ULL;
-                    if (progressStamp != lastProgressStamp)
+
+                    if (kIOHibernateModeEncrypt & gIOHibernateMode)
+                        pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
+
+                    if (pageCompressedSize != page_size)
+                        data = compressed;
+                    else
+                        data = src;
+    
+                    tag = pageCompressedSize | kIOHibernateTagSignature;
+                    err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
+                    if (kIOReturnSuccess != err)
+                        break;
+    
+                    err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
+                    if (kIOReturnSuccess != err)
+                        break;
+    
+                    compressedSize += pageCompressedSize;
+                    uncompressedSize += page_size;
+                    pagesDone++;
+    
+                    if (vars->consoleMapping && (0 == (1023 & pagesDone)))

                     {
                     {

-                        lastProgressStamp = progressStamp;
-                        HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
+                        blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
+                        if (blob != lastBlob)
+                        {
+                            ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
+                            lastBlob = blob;
+                        }
+                    }
+                    if (0 == (8191 & pagesDone))
+                    {
+                        clock_get_uptime(&endTime);
+                        SUB_ABSOLUTETIME(&endTime, &allTime);
+                        absolutetime_to_nanoseconds(endTime, &nsec);
+                        progressStamp = nsec / 750000000ULL;
+                        if (progressStamp != lastProgressStamp)
+                        {
+                            lastProgressStamp = progressStamp;
+                            HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
+                        }

                     }
                 }
                     }
                 }

+                if (kIOReturnSuccess != err)
+                    break;
+                ppnum = page;

             }
             }

+

             if (kIOReturnSuccess != err)
                 break;
             if (kIOReturnSuccess != err)
                 break;

-            if (iterDone && !pageType)
+
+            if ((kEncrypt & pageType) && vars->fileVars->encryptStart)
+            {
+                vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
+                HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
+            }
+
+            if (kWiredEncrypt != pageType)

             {
             {

+                // end of image1/2 - fill to next block

                 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
                 if (kIOReturnSuccess != err)
                     break;
                 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
                 if (kIOReturnSuccess != err)
                     break;

+            }
+            if (kWiredClear == pageType)
+            {
+               // enlarge wired image for test
+//              err = IOPolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);

 
 

-                iterDone = false;
-                pageType = 1;
-                ppnum = 0;
+                // end wired image
+                header->encryptStart = vars->fileVars->encryptStart;
+                header->encryptEnd   = vars->fileVars->encryptEnd;

                 image1Size = vars->fileVars->position;
                 image1Size = vars->fileVars->position;

-                if (cryptvars)
-                {
-                    bcopy(&cryptvars->aes_iv[0], 
-                            &gIOHibernateCryptWakeContext.aes_iv[0], 
-                            sizeof(cryptvars->aes_iv));
-                    cryptvars = &gIOHibernateCryptWakeContext;
-                }
-                HIBLOG("image1Size %qd\n", image1Size);
+                HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
+                        image1Size, header->encryptStart, header->encryptEnd);

             }
         }
             }
         }

-        while (!iterDone);
-        if (kIOReturnSuccess != err)
-            break;
-        err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);

         if (kIOReturnSuccess != err)
         if (kIOReturnSuccess != err)

+        {
+            if (kIOReturnOverrun == err)
+            {
+               // update actual compression ratio on not enough space
+                gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
+            }

             break;
             break;

+        }

 
         // Header:
     
 
         // Header:
     


@@ -2113,11 +2934,14 @@ hibernate_write_image(void)
         header->image1Size   = image1Size;
         header->bitmapSize   = bitmap_size;
         header->pageCount    = pageCount;
         header->image1Size   = image1Size;
         header->bitmapSize   = bitmap_size;
         header->pageCount    = pageCount;

-        header->encryptStart = vars->fileVars->encryptStart;

     
         header->restore1Sum  = restore1Sum;
         header->image1Sum    = sum1;
         header->image2Sum    = sum2;
     
         header->restore1Sum  = restore1Sum;
         header->image1Sum    = sum1;
         header->image2Sum    = sum2;

+        header->sleepTime    = gIOLastSleepTime.tv_sec;
+
+       header->compression     = (compressedSize << 8) / uncompressedSize;
+       gIOHibernateCompression = header->compression;

     
         count = vars->fileExtents->getLength();
         if (count > sizeof(header->fileExtentMap))
     
         count = vars->fileExtents->getLength();
         if (count > sizeof(header->fileExtentMap))


@@ -2128,6 +2952,9 @@ hibernate_write_image(void)
         else
             header->fileExtentMapSize = sizeof(header->fileExtentMap);
         bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
         else
             header->fileExtentMapSize = sizeof(header->fileExtentMap);
         bcopy(&fileExtents[0], &header->fileExtentMap[0], count);

+
+        header->deviceBase      = vars->fileVars->block0;
+        header->deviceBlockSize = vars->fileVars->blockSize;

     
         IOPolledFileSeek(vars->fileVars, 0);
         err = IOPolledFileWrite(vars->fileVars,
     
         IOPolledFileSeek(vars->fileVars, 0);
         err = IOPolledFileWrite(vars->fileVars,


@@ -2151,23 +2978,29 @@ hibernate_write_image(void)
 
     SUB_ABSOLUTETIME(&endTime, &allTime);
     absolutetime_to_nanoseconds(endTime, &nsec);
 
     SUB_ABSOLUTETIME(&endTime, &allTime);
     absolutetime_to_nanoseconds(endTime, &nsec);

-    HIBLOG("all time: %qd ms, ", 
-               nsec / 1000000ULL);
+    HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);

 
     absolutetime_to_nanoseconds(compTime, &nsec);
 
     absolutetime_to_nanoseconds(compTime, &nsec);

-    HIBLOG("comp time: %qd ms, ", 
-               nsec / 1000000ULL);
-
-    absolutetime_to_nanoseconds(decoTime, &nsec);
-    HIBLOG("deco time: %qd ms, ", 
-               nsec / 1000000ULL);
-
-    HIBLOG("\nimage %qd, uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n", 
-               header->imageSize,
+    HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ", 
+               compBytes, 
+               nsec / 1000000ULL,
+               nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+    absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
+    HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ", 
+               vars->fileVars->cryptBytes, 
+               nsec / 1000000ULL, 
+               nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+    HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n", 
+               header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,

                uncompressedSize, atop_32(uncompressedSize), compressedSize,
                uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
                sum1, sum2);
 
                uncompressedSize, atop_32(uncompressedSize), compressedSize,
                uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
                sum1, sum2);
 

+    HIBLOG("zeroPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n", 
+             zeroPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
+

     if (vars->fileVars->io)
         (void) IOHibernatePollerIODone(vars->fileVars, false);
 
     if (vars->fileVars->io)
         (void) IOHibernatePollerIODone(vars->fileVars, false);
 


@@ -2183,6 +3016,9 @@ hibernate_write_image(void)
     // should we come back via regular wake, set the state in memory.
     gIOHibernateState = kIOHibernateStateInactive;
 
     // should we come back via regular wake, set the state in memory.
     gIOHibernateState = kIOHibernateStateInactive;
 

+    KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END,
+                         wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted, 0, 0);
+

     if (kIOReturnSuccess == err)
     {
        if (kIOHibernateModeSleep & gIOHibernateMode)
     if (kIOReturnSuccess == err)
     {
        if (kIOHibernateModeSleep & gIOHibernateMode)


@@ -2218,15 +3054,18 @@ hibernate_machine_init(void)
     IOReturn     err;
     uint32_t     sum;
     uint32_t     pagesDone;
     IOReturn     err;
     uint32_t     sum;
     uint32_t     pagesDone;

+    uint32_t     pagesRead = 0;
+    AbsoluteTime startTime, compTime;

     AbsoluteTime allTime, endTime;
     AbsoluteTime allTime, endTime;

-    uint64_t     nsec;
+    AbsoluteTime startIOTime, endIOTime;
+    uint64_t     nsec, nsecIO;
+    uint64_t     compBytes;

     uint32_t     lastProgressStamp = 0;
     uint32_t     progressStamp;
     uint32_t     lastProgressStamp = 0;
     uint32_t     progressStamp;

-    uint64_t    progressZeroPosition = 0;
-    uint32_t    blob, lastBlob = (uint32_t) -1L;

     hibernate_cryptvars_t * cryptvars = 0;
 
     IOHibernateVars * vars  = &gIOHibernateVars;
     hibernate_cryptvars_t * cryptvars = 0;
 
     IOHibernateVars * vars  = &gIOHibernateVars;

+    bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));

 
     if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
        return;
 
     if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
        return;


@@ -2234,10 +3073,6 @@ hibernate_machine_init(void)
     sum = gIOHibernateCurrentHeader->actualImage1Sum;
     pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
 
     sum = gIOHibernateCurrentHeader->actualImage1Sum;
     pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
 

-    HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, image1Size %qx, conflictCount %d, nextFree %x\n",
-           gIOHibernateState, pagesDone, sum, gIOHibernateCurrentHeader->image1Size,
-           gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
-

     if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
     {
        HIBLOG("regular wake\n");
     if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
     {
        HIBLOG("regular wake\n");


@@ -2246,81 +3081,163 @@ hibernate_machine_init(void)
 
     HIBPRINT("diag %x %x %x %x\n",
            gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1], 
 
     HIBPRINT("diag %x %x %x %x\n",
            gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1], 

-           gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]); 
-
-    HIBPRINT("video %x %d %d %d\n",
-           gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth, 
-           gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height); 
+           gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
+
+#define t40ms(x)       (tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)
+#define tStat(x, y)    gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
+    tStat(booterStart, booterStart);
+    gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart,
+    tStat(booterDuration0, booterTime0);
+    tStat(booterDuration1, booterTime1);
+    tStat(booterDuration2, booterTime2);
+    tStat(booterDuration, booterTime);
+    tStat(booterConnectDisplayDuration, connectDisplayTime);
+    tStat(booterSplashDuration, splashTime);
+    tStat(trampolineDuration, trampolineTime);
+
+    gIOHibernateStats->image1Size  = gIOHibernateCurrentHeader->image1Size;
+    gIOHibernateStats->imageSize   = gIOHibernateCurrentHeader->imageSize;
+    gIOHibernateStats->image1Pages = pagesDone;
+
+    HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n", 
+          gIOHibernateStats->booterStart,
+          gIOHibernateStats->smcStart,
+          gIOHibernateStats->booterDuration0,
+          gIOHibernateStats->booterDuration1,
+          gIOHibernateStats->booterDuration2,
+          gIOHibernateStats->booterDuration,
+          gIOHibernateStats->booterConnectDisplayDuration,
+          gIOHibernateStats->booterSplashDuration,
+          gIOHibernateStats->trampolineDuration);
+
+    HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
+           gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
+           gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);

 
 

-    if ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)
+    if ((0 != (kIOHibernateModeSleep & gIOHibernateMode)) 
+     && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)))
+    {

         hibernate_page_list_discard(vars->page_list);
         hibernate_page_list_discard(vars->page_list);

+    }
+
+    cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;

 
 

-    boot_args *args = (boot_args *) PE_state.bootArgs;
+    if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount)
+       panic("handoff overflow");

 
 

-    if (vars->videoMapping 
-       && gIOHibernateGraphicsInfo->physicalAddress
-       && (args->Video.v_baseAddr == gIOHibernateGraphicsInfo->physicalAddress))
+    IOHibernateHandoff * handoff;
+    bool                 done           = false;
+    bool                 foundCryptData = false;
+
+    for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
+        !done;
+        handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])

     {
     {

-        vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height 
-                                        * gIOHibernateGraphicsInfo->rowBytes);
-        IOMapPages(kernel_map, 
-                    vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
-                    vars->videoMapSize, kIOMapInhibitCache );
+//     HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
+       uint8_t * data = &handoff->data[0];
+       switch (handoff->type)
+       {
+           case kIOHibernateHandoffTypeEnd:
+               done = true;
+               break;
+
+           case kIOHibernateHandoffTypeGraphicsInfo:
+               bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
+               break;
+
+           case kIOHibernateHandoffTypeCryptVars:
+               if (cryptvars)
+               {
+                   hibernate_cryptwakevars_t *
+                   wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
+                   bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
+               }
+               foundCryptData = true;
+               bzero(data, handoff->bytecount);
+               break;
+
+           case kIOHibernateHandoffTypeMemoryMap:
+
+               clock_get_uptime(&allTime);
+
+               hibernate_newruntime_map(data, handoff->bytecount, 
+                                        gIOHibernateCurrentHeader->systemTableOffset);
+
+               clock_get_uptime(&endTime);
+           
+               SUB_ABSOLUTETIME(&endTime, &allTime);
+               absolutetime_to_nanoseconds(endTime, &nsec);
+           
+               HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
+
+               break;
+
+           case kIOHibernateHandoffTypeDeviceTree:
+               {
+//                 DTEntry chosen = NULL;
+//                 HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen));
+               }
+               break;
+
+           default:
+               done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
+               break;
+       }    

     }
     }

+    if (cryptvars && !foundCryptData)
+       panic("hibernate handoff");

 
 

-    uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
+    HIBPRINT("video %x %d %d %d status %x\n",
+           gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth, 
+           gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus); 

 
 

-    if (gIOHibernateWakeMapSize)
+    if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)

     {
     {

-       err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(gIOHibernateWakeMap), 
-                                                   gIOHibernateWakeMapSize);
-       if (kIOReturnSuccess == err)
-           hibernate_newruntime_map(src, gIOHibernateWakeMapSize, 
-                                    gIOHibernateCurrentHeader->systemTableOffset);
-       gIOHibernateWakeMap = 0;
-       gIOHibernateWakeMapSize = 0;
+        vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height 
+                                        * gIOHibernateGraphicsInfo->rowBytes);
+       if (vars->videoMapSize > vars->videoAllocSize) vars->videoMapSize = 0;
+       else
+       {
+           IOMapPages(kernel_map, 
+                       vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
+                       vars->videoMapSize, kIOMapInhibitCache );
+       }

     }
 
     }
 

-    uint32_t decoOffset;
+    if (vars->videoMapSize)
+        ProgressUpdate(gIOHibernateGraphicsInfo, 
+                        (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
+
+    uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
+    uint8_t * compressed = src + page_size;
+    uint8_t * scratch    = compressed + page_size;
+    uint32_t  decoOffset;

 
     clock_get_uptime(&allTime);
 
     clock_get_uptime(&allTime);

+    AbsoluteTime_to_scalar(&compTime) = 0;
+    compBytes = 0;

 
     HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
     err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0);
 
     HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
     err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0);

-    HIBLOG("IOHibernatePollerOpen(%x)\n", err);
-
-    if (gIOHibernateCurrentHeader->previewSize)
-        progressZeroPosition = gIOHibernateCurrentHeader->previewSize 
-                             + gIOHibernateCurrentHeader->fileExtentMapSize 
-                             - sizeof(gIOHibernateCurrentHeader->fileExtentMap) 
-                             + ptoa_64(gIOHibernateCurrentHeader->restore1PageCount);
+    clock_get_uptime(&startIOTime);
+    endTime = startIOTime;
+    SUB_ABSOLUTETIME(&endTime, &allTime);
+    absolutetime_to_nanoseconds(endTime, &nsec);
+    HIBLOG("IOHibernatePollerOpen(%x) %qd ms\n", err, nsec / 1000000ULL);

 
     IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
 
 
     IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
 

-    if (vars->videoMapSize)
-    {
-        lastBlob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
-                        / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
-        ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, 0, lastBlob);
-    }
-
-    cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
-    if (kIOHibernateModeEncrypt & gIOHibernateMode)
-    {
-        cryptvars = &gIOHibernateCryptWakeContext;
-        bcopy(&gIOHibernateCryptWakeVars->aes_iv[0], 
-                &cryptvars->aes_iv[0], 
-                sizeof(cryptvars->aes_iv));
-    }
-

     // kick off the read ahead
     vars->fileVars->io          = false;
     vars->fileVars->bufferHalf   = 0;
     vars->fileVars->bufferLimit  = 0;
     vars->fileVars->lastRead     = 0;
     // kick off the read ahead
     vars->fileVars->io          = false;
     vars->fileVars->bufferHalf   = 0;
     vars->fileVars->bufferLimit  = 0;
     vars->fileVars->lastRead     = 0;

+    vars->fileVars->readEnd      = gIOHibernateCurrentHeader->imageSize;

     vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
     vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;

+    vars->fileVars->cryptBytes   = 0;
+    AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;

 
 

-    IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
+    err = IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);

     vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
     // --
 
     vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
     // --
 


@@ -2329,7 +3246,7 @@ hibernate_machine_init(void)
     uint32_t * header = (uint32_t *) src;
     sum = 0;
 
     uint32_t * header = (uint32_t *) src;
     sum = 0;
 

-    do
+    while (kIOReturnSuccess == err)

     {
        unsigned int count;
        unsigned int page;
     {
        unsigned int count;
        unsigned int page;


@@ -2361,47 +3278,35 @@ hibernate_machine_init(void)
                break;
            }
 
                break;
            }
 

-           if (!compressedSize)
-           {
-               ppnum++;
-               pagesDone++;
-               continue;
-           }
-
-           err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
-           if (kIOReturnSuccess != err)
-               break;
-
-           if (compressedSize < page_size)
-           {
-               decoOffset = page_size;
-               WKdm_decompress((WK_word*) src, (WK_word*) (src + decoOffset), PAGE_SIZE_IN_WORDS);
-           }
+           if (!compressedSize) bzero_phys(ptoa_64(ppnum), page_size);

            else
            else

-               decoOffset = 0;
-
-           sum += hibernate_sum((src + decoOffset), page_size);
-
-           err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
-           if (err)

            {
            {

-               HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
-               break;
+               err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
+               if (kIOReturnSuccess != err) break;
+               if (compressedSize < page_size)
+               {
+                   decoOffset = page_size;
+                   clock_get_uptime(&startTime);
+                   WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, page_size);
+                   clock_get_uptime(&endTime);
+                   ADD_ABSOLUTETIME(&compTime, &endTime);
+                   SUB_ABSOLUTETIME(&compTime, &startTime);
+                   compBytes += page_size;
+               }
+               else decoOffset = 0;
+
+               sum += hibernate_sum_page((src + decoOffset), ppnum);
+               err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
+               if (err)
+               {
+                   HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
+                   break;
+               }

            }
 
            ppnum++;
            pagesDone++;
            }
 
            ppnum++;
            pagesDone++;

-
-            if (vars->videoMapSize && (0 == (1023 & pagesDone)))
-            {
-                blob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
-                        / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
-                if (blob != lastBlob)
-                {
-                    ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, lastBlob, blob);
-                    lastBlob = blob;
-                }
-            }
+           pagesRead++;

 
            if (0 == (8191 & pagesDone))
            {
 
            if (0 == (8191 & pagesDone))
            {


@@ -2418,21 +3323,21 @@ hibernate_machine_init(void)
            }
        }
     }
            }
        }
     }

-    while (true);
+    if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages))
+       err = kIOReturnLockedRead;

 
     if (kIOReturnSuccess != err)
        panic("Hibernate restore error %x", err);
 
     gIOHibernateCurrentHeader->actualImage2Sum = sum;
 
     if (kIOReturnSuccess != err)
        panic("Hibernate restore error %x", err);
 
     gIOHibernateCurrentHeader->actualImage2Sum = sum;

+    gIOHibernateCompression = gIOHibernateCurrentHeader->compression;

 
     if (vars->fileVars->io)
         (void) IOHibernatePollerIODone(vars->fileVars, false);
 
 
     if (vars->fileVars->io)
         (void) IOHibernatePollerIODone(vars->fileVars, false);
 

-    err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
+    clock_get_uptime(&endIOTime);

 
 

-    if (vars->videoMapSize)
-        ProgressUpdate(gIOHibernateGraphicsInfo, 
-                        (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
+    err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);

 
     clock_get_uptime(&endTime);
 
 
     clock_get_uptime(&endTime);
 


@@ -2444,8 +3349,92 @@ hibernate_machine_init(void)
     SUB_ABSOLUTETIME(&endTime, &allTime);
     absolutetime_to_nanoseconds(endTime, &nsec);
 
     SUB_ABSOLUTETIME(&endTime, &allTime);
     absolutetime_to_nanoseconds(endTime, &nsec);
 

-    HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %qd ms\n", 
-               pagesDone, sum, nsec / 1000000ULL);
+    SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
+    absolutetime_to_nanoseconds(endIOTime, &nsecIO);
+
+    gIOHibernateStats->kernelImageReadDuration = nsec / 1000000ULL;
+    gIOHibernateStats->imagePages              = pagesDone;
+
+    HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ", 
+               pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
+               nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
+
+    absolutetime_to_nanoseconds(compTime, &nsec);
+    HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ", 
+               compBytes, 
+               nsec / 1000000ULL,
+               nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+    absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
+    HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n", 
+               vars->fileVars->cryptBytes, 
+               nsec / 1000000ULL, 
+               nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+    KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_NONE, pagesRead, pagesDone, 0, 0, 0);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+void IOHibernateSetWakeCapabilities(uint32_t capability)
+{
+    if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
+    {
+       gIOHibernateStats->wakeCapability = capability;
+
+       if (kIOPMSystemCapabilityGraphics & capability)
+       {
+               vm_compressor_do_warmup();
+       }
+    }

 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

+
+void IOHibernateSystemRestart(void)
+{
+    static uint8_t    noteStore[32] __attribute__((aligned(32)));
+    IORegistryEntry * regEntry;
+    const OSSymbol *  sym;
+    OSData *          noteProp;
+    OSData *          data;
+    uintptr_t *       smcVars;
+    uint8_t *         smcBytes;
+    size_t            len;
+    addr64_t          element;
+
+    data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
+    if (!data) return;
+
+    smcVars = (typeof(smcVars)) data->getBytesNoCopy();
+    smcBytes = (typeof(smcBytes)) smcVars[1];
+    len = smcVars[0];
+    if (len > sizeof(noteStore)) len = sizeof(noteStore);
+    noteProp = OSData::withCapacity(3 * sizeof(element));
+    if (!noteProp) return;
+    element = len;
+    noteProp->appendBytes(&element, sizeof(element));
+    element = crc32(0, smcBytes, len);
+    noteProp->appendBytes(&element, sizeof(element));
+
+    bcopy(smcBytes, noteStore, len);
+    element = (addr64_t) &noteStore[0];
+    element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
+    noteProp->appendBytes(&element, sizeof(element));
+
+    if (!gIOOptionsEntry)
+    {
+       regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
+       gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
+       if (regEntry && !gIOOptionsEntry)
+           regEntry->release();
+    }
+
+    sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
+    if (gIOOptionsEntry && sym) gIOOptionsEntry->setProperty(sym, noteProp);
+    if (noteProp)               noteProp->release();
+    if (sym)                    sym->release();
+}
+
+
+