xnu-3247.1.106.tar.gz

[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp

diff --git a/iokit/Kernel/IOHibernateIO.cpp b/iokit/Kernel/IOHibernateIO.cpp
index 4beb2606cf00c9d27a48af4514296a44c7f79ded..867251b27f3fe04de23a8ab15d9c93c48576025a 100644 (file)
--- a/iokit/Kernel/IOHibernateIO.cpp
+++ b/iokit/Kernel/IOHibernateIO.cpp
@@ -1,23 +1,29 @@
 /*
 /*

- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2008 Apple Computer, Inc. All rights reserved.

  *
  *

- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

  * 
  * 

- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.

  * 
  * 

- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER

  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,

- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.

  * 
  * 

- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@

  */
 
 
  */
 
 


@@ -52,12 +58,12 @@ Sleep:
   by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
   The image header and block list are written. The header includes the second file extent so
   only the header block is needed to read the file, regardless of filesystem.
   by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
   The image header and block list are written. The header includes the second file extent so
   only the header block is needed to read the file, regardless of filesystem.

-  The kernel section "__HIB" is written uncompressed to the image. This section of code and data 
+  The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data 

   (only) is used to decompress the image during wake/boot.
   Some additional pages are removed from the bitmaps - the buffers used for hibernation.
   The bitmaps are written to the image.
   More areas are removed from the bitmaps (after they have been written to the image) - the 
   (only) is used to decompress the image during wake/boot.
   Some additional pages are removed from the bitmaps - the buffers used for hibernation.
   The bitmaps are written to the image.
   More areas are removed from the bitmaps (after they have been written to the image) - the 

-  section "__HIB" pages and interrupt stack.
+  segment "__HIB" pages and interrupt stack.

   Each wired page is compressed and written and then each non-wired page. Compression and 
   disk writes are in parallel.
   The image header is written to the start of the file and the polling driver closed.
   Each wired page is compressed and written and then each non-wired page. Compression and 
   disk writes are in parallel.
   The image header is written to the start of the file and the polling driver closed.


@@ -145,81 +151,92 @@ to restrict I/O ops.
 #include <IOKit/pwr_mgt/IOPowerConnection.h>
 #include "IOPMPowerStateQueue.h"
 #include <IOKit/IOBufferMemoryDescriptor.h>
 #include <IOKit/pwr_mgt/IOPowerConnection.h>
 #include "IOPMPowerStateQueue.h"
 #include <IOKit/IOBufferMemoryDescriptor.h>

-#include <crypto/aes.h>
+#include <IOKit/AppleKeyStoreInterface.h>
+#include <libkern/crypto/aes.h>

 
 #include <sys/uio.h>
 #include <sys/conf.h>
 #include <sys/stat.h>
 #include <sys/fcntl.h>                       // (FWRITE, ...)
 #include <sys/sysctl.h>
 
 #include <sys/uio.h>
 #include <sys/conf.h>
 #include <sys/stat.h>
 #include <sys/fcntl.h>                       // (FWRITE, ...)
 #include <sys/sysctl.h>

+#include <sys/kdebug.h>
+#include <stdint.h>

 
 #include <IOKit/IOHibernatePrivate.h>
 #include <IOKit/IOPolledInterface.h>
 #include <IOKit/IONVRAM.h>
 #include "IOHibernateInternal.h"
 
 #include <IOKit/IOHibernatePrivate.h>
 #include <IOKit/IOPolledInterface.h>
 #include <IOKit/IONVRAM.h>
 #include "IOHibernateInternal.h"

-#include "WKdm.h"
+#include <vm/WKdm_new.h>

 #include "IOKitKernelInternal.h"
 #include "IOKitKernelInternal.h"

+#include <pexpert/device_tree.h>

 
 

-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+#include <machine/pal_routines.h>
+#include <machine/pal_hibernate.h>
+#include <i386/tsc.h>

 
 

-OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
-
-OSMetaClassDefineReservedUnused(IOPolledInterface, 0);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
+extern "C" addr64_t            kvtophys(vm_offset_t va);
+extern "C" ppnum_t             pmap_find_phys(pmap_t pmap, addr64_t va);

 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 

+#define        DISABLE_TRIM            0
+#define TRIM_DELAY             5000
+
+extern boolean_t               root_is_CF_drive;
+extern unsigned int            save_kdebug_enable;

 extern uint32_t                gIOHibernateState;
 uint32_t                       gIOHibernateMode;
 static char                    gIOHibernateBootSignature[256+1];
 static char                    gIOHibernateFilename[MAXPATHLEN+1];
 extern uint32_t                gIOHibernateState;
 uint32_t                       gIOHibernateMode;
 static char                    gIOHibernateBootSignature[256+1];
 static char                    gIOHibernateFilename[MAXPATHLEN+1];

-static uint32_t                        gIOHibernateFreeRatio = 0;              // free page target (percent)
-uint32_t                       gIOHibernateFreeTime  = 0*1000; // max time to spend freeing pages (ms)
+static uint32_t                        gIOHibernateFreeRatio = 0;       // free page target (percent)
+uint32_t                       gIOHibernateFreeTime  = 0*1000;  // max time to spend freeing pages (ms)
+static uint64_t                        gIOHibernateCompression = 0x80;  // default compression 50%

 
 static IODTNVRAM *             gIOOptionsEntry;
 static IORegistryEntry *       gIOChosenEntry;
 
 static IODTNVRAM *             gIOOptionsEntry;
 static IORegistryEntry *       gIOChosenEntry;

-#ifdef __i386__
-static const OSSymbol *         gIOCreateEFIDevicePathSymbol;
-#endif

 
 

-static IOPolledFileIOVars                gFileVars;
-static IOHibernateVars                   gIOHibernateVars;
-static struct kern_direct_file_io_ref_t * gIOHibernateFileRef;
-static hibernate_cryptvars_t             gIOHibernateCryptWakeContext;
+static const OSSymbol *        gIOHibernateBootImageKey;

 
 

-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+#if defined(__i386__) || defined(__x86_64__)

 
 

-enum { kXPRamAudioVolume = 8 };
-enum { kDefaultIOSize = 128 * 1024 };
-enum { kVideoMapSize  = 32 * 1024 * 1024 };
+static const OSSymbol *        gIOHibernateRTCVariablesKey;
+static const OSSymbol *         gIOHibernateBoot0082Key;
+static const OSSymbol *         gIOHibernateBootNextKey;
+static OSData *                        gIOHibernateBoot0082Data;
+static OSData *                        gIOHibernateBootNextData;
+static OSObject *              gIOHibernateBootNextSave;

 
 

-#ifndef kIOMediaPreferredBlockSizeKey
-#define kIOMediaPreferredBlockSizeKey  "Preferred Block Size"
-#endif
+static IOPolledFileIOVars *     gDebugImageFileVars;
+static IOLock             *     gDebugImageLock;

 
 

-#ifndef kIOBootPathKey 
-#define kIOBootPathKey                 "bootpath"
-#endif
-#ifndef kIOSelectedBootDeviceKey       
-#define kIOSelectedBootDeviceKey       "boot-device"
-#endif
+#endif /* defined(__i386__) || defined(__x86_64__) */
+
+static IOLock *                           gFSLock;
+static uint32_t                           gFSState;
+static IOPolledFileIOVars                gFileVars;
+static IOHibernateVars                   gIOHibernateVars;
+static IOPolledFileCryptVars             gIOHibernateCryptWakeContext;
+static hibernate_graphics_t              _hibernateGraphics;
+static hibernate_graphics_t *            gIOHibernateGraphicsInfo = &_hibernateGraphics;
+static hibernate_statistics_t            _hibernateStats;
+static hibernate_statistics_t *                  gIOHibernateStats = &_hibernateStats;
+
+enum 
+{
+    kFSIdle     = 0,
+    kFSOpening  = 2,
+    kFSOpened   = 3,
+    kFSTimedOut = 4,
+};
+
+static IOReturn IOHibernateDone(IOHibernateVars * vars);
+static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature);
+static void     IOSetBootImageNVRAM(OSData * data);

 
 

+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

 
 

-enum { kIOHibernateMinPollersNeeded = 2 };
+enum { kDefaultIOSize = 128 * 1024 };
+enum { kVideoMapSize  = 80 * 1024 * 1024 };

 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 


@@ -237,7 +254,7 @@ IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
         addr64_t    dstAddr64;
         IOByteCount dstLen;
 
         addr64_t    dstAddr64;
         IOByteCount dstLen;
 

-        dstAddr64 = md->getPhysicalSegment64(offset, &dstLen);
+        dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);

         if (!dstAddr64)
             break;
 
         if (!dstAddr64)
             break;
 


@@ -275,7 +292,7 @@ IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
         addr64_t    srcAddr64;
         IOByteCount dstLen;
 
         addr64_t    srcAddr64;
         IOByteCount dstLen;
 

-        srcAddr64 = md->getPhysicalSegment64(offset, &dstLen);
+        srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);

         if (!srcAddr64)
             break;
 
         if (!srcAddr64)
             break;
 


@@ -365,927 +382,729 @@ hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 

-static IOReturn
-IOHibernatePollerProbe(IOPolledFileIOVars * vars, IOService * target)
+IOReturn
+IOHibernateSystemSleep(void)

 {
 {

-    IOReturn            err = kIOReturnError;
-    int32_t            idx;
-    IOPolledInterface * poller;
-
-    for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
-    {
-        poller = (IOPolledInterface *) vars->pollers->getObject(idx);
-        err = poller->probe(target);
-        if (err)
-        {
-            HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
-            break;
-        }
-    }
+    IOReturn   err;
+    OSData *   nvramData;
+    OSObject * obj;
+    OSString * str;
+    OSNumber * num;
+    bool       dsSSD, vmflush;
+    IOHibernateVars * vars;
+    uint64_t   setFileSize = 0;

 
 

-    return (err);
-}
+    gIOHibernateState = kIOHibernateStateInactive;

 
 

-static IOReturn
-IOHibernatePollerOpen(IOPolledFileIOVars * vars, uint32_t state, IOMemoryDescriptor * md)
-{
-    IOReturn            err = kIOReturnError;
-    int32_t            idx;
-    IOPolledInterface * poller;
+    gIOHibernateDebugFlags = 0;
+    if (kIOLogHibernate & gIOKitDebug)
+       gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;

 
 

-    for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
+    if (IOService::getPMRootDomain()->getHibernateSettings(
+        &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime))

     {
     {

-        poller = (IOPolledInterface *) vars->pollers->getObject(idx);
-        err = poller->open(state, md);
-        if (err)
-        {
-            HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
-            break;
-        }
+        if (kIOHibernateModeSleep & gIOHibernateMode)
+            // default to discard clean for safe sleep
+            gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive 
+                                | kIOHibernateModeDiscardCleanActive);

     }
 
     }
 

-    return (err);
-}
-
-static IOReturn
-IOHibernatePollerClose(IOPolledFileIOVars * vars, uint32_t state)
-{
-    IOReturn            err = kIOReturnError;
-    int32_t            idx;
-    IOPolledInterface * poller;
-
-    for (idx = 0;
-         (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
-         idx++)
+    if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))

     {
     {

-        err = poller->close(state);
-        if (err)
-            HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
+       if ((str = OSDynamicCast(OSString, obj)))
+           strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
+                           sizeof(gIOHibernateFilename));
+       obj->release();

     }
 
     }
 

-    return (err);
-}
-
-static void
-IOHibernatePollerIOComplete(void *   target,
-                            void *   parameter,
-                            IOReturn status,
-                            UInt64   actualByteCount)
-{
-    IOPolledFileIOVars * vars = (IOPolledFileIOVars *) parameter;
-
-    vars->ioStatus = status;
-}
-
-static IOReturn
-IOHibernatePollerIO(IOPolledFileIOVars * vars, 
-                    uint32_t operation, uint32_t bufferOffset, 
-                   uint64_t deviceOffset, uint64_t length)
-{
-
-    IOReturn            err = kIOReturnError;
-    IOPolledInterface * poller;
-    IOPolledCompletion  completion;
+    if (!gIOHibernateMode || !gIOHibernateFilename[0])
+       return (kIOReturnUnsupported);

 
 

-    completion.target    = 0;
-    completion.action    = &IOHibernatePollerIOComplete;
-    completion.parameter = vars;
+    HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);

 
 

-    vars->ioStatus = -1;
+    vars = IONew(IOHibernateVars, 1);
+    if (!vars) return (kIOReturnNoMemory);
+    bzero(vars, sizeof(*vars));

 
 

-    poller = (IOPolledInterface *) vars->pollers->getObject(0);
-    err = poller->startIO(operation, bufferOffset, deviceOffset + vars->block0, length, completion);
-    if (err)
-        HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
+    IOLockLock(gFSLock);
+    if (kFSIdle != gFSState)
+    {
+       HIBLOG("hibernate file busy\n");
+       IOLockUnlock(gFSLock);
+       IODelete(vars, IOHibernateVars, 1);
+        return (kIOReturnBusy);
+    }
+    gFSState = kFSOpening;
+    IOLockUnlock(gFSLock);

 
 

-    return (err);
-}
+    do
+    {
+        vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
+                                   2 * page_size + WKdm_SCRATCH_BUF_SIZE, page_size);

 
 

-static IOReturn
-IOHibernatePollerIODone(IOPolledFileIOVars * vars)
-{
-    IOReturn            err = kIOReturnError;
-    int32_t            idx;
-    IOPolledInterface * poller;
+       vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, 
+                                   ptoa_64(gIOHibernateHandoffPageCount), page_size);

 
 

-    while (-1 == vars->ioStatus)
-    {
-        for (idx = 0;
-             (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
-             idx++)
+        if (!vars->srcBuffer || !vars->handoffBuffer)

         {
         {

-            err = poller->checkForWork();
-            if (err)
-                HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
+            err = kIOReturnNoMemory;
+            break;

         }
         }

-    }

 
 

-    if (kIOReturnSuccess != vars->ioStatus)
-        HIBLOG("IOPolledInterface::ioStatus 0x%x\n", vars->ioStatus);
-
-    return (vars->ioStatus);
-}
+       if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey)))
+       {
+           if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMinSize = num->unsigned64BitValue();
+           obj->release();
+       }
+       if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey)))
+       {
+           if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMaxSize = num->unsigned64BitValue();
+           obj->release();
+       }

 
 

-IOReturn
-IOPolledInterface::checkAllForWork(void)
-{
-    IOReturn            err = kIOReturnNotReady;
-    int32_t            idx;
-    IOPolledInterface * poller;
+        boolean_t encryptedswap = true;
+        uint32_t pageCount;
+        AbsoluteTime startTime, endTime;
+        uint64_t nsec;

 
 

-    IOHibernateVars * vars  = &gIOHibernateVars;
+       bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
+       gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
+       gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;

 
 

-    if (!vars->fileVars || !vars->fileVars->pollers)
-       return (err);
+       vmflush = ((kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)) && root_is_CF_drive == FALSE);
+        err = hibernate_alloc_page_lists(&vars->page_list, 
+                                        &vars->page_list_wired,
+                                        &vars->page_list_pal);
+        if (KERN_SUCCESS != err)
+            break;

 
 

-    for (idx = 0;
-            (poller = (IOPolledInterface *) vars->fileVars->pollers->getObject(idx));
-            idx++)
-    {
-        err = poller->checkForWork();
-        if (err)
-            HIBLOG("IOPolledInterface::checkAllForWork[%d] 0x%x\n", idx, err);
+       if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode))
+       {
+           hibernate_page_list_setall(vars->page_list,
+                                      vars->page_list_wired,
+                                      vars->page_list_pal,
+                                      true /* preflight */,
+                                      vmflush /* discard */,
+                                      &pageCount);
+           PE_Video consoleInfo;
+           bzero(&consoleInfo, sizeof(consoleInfo));
+           IOService::getPlatform()->getConsoleInfo(&consoleInfo);
+
+           // estimate: 6% increase in pages compressed
+           // screen preview 2 images compressed 0%
+           setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
+                               + vars->page_list->list_size
+                               + (consoleInfo.v_width * consoleInfo.v_height * 8);
+           enum { setFileRound = 1024*1024ULL };
+           setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
+       
+           HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n", 
+                   pageCount, (100ULL * gIOHibernateCompression) >> 8,
+                   setFileSize, vars->fileMinSize);
+
+           if (!(kIOHibernateModeFileResize & gIOHibernateMode)
+            && (setFileSize < vars->fileMinSize))
+           { 
+               setFileSize = vars->fileMinSize;
+           }
+       }
+    
+       // Invalidate the image file
+    if (gDebugImageLock) {
+        IOLockLock(gDebugImageLock);
+        if (gDebugImageFileVars != 0) {
+            kprintf("IOHIBSystemSleep: Closing debugdata file\n");
+            IOSetBootImageNVRAM(0);
+            IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
+        }
+        IOLockUnlock(gDebugImageLock);

     }
 
     }
 

-    return (err);
-}
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-struct _OpenFileContext
-{
-    OSData * extents;
-    uint64_t size;
-};
+        err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, 0,
+                               gIOHibernateCurrentHeader, sizeof(gIOHibernateCurrentHeader),
+                                &vars->fileVars, &nvramData, 
+                                &vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));

 
 

-static void
-file_extent_callback(void * ref, uint64_t start, uint64_t length)
-{
-    _OpenFileContext * ctx = (_OpenFileContext *) ref;
-    IOPolledFileExtent extent;
+        if (KERN_SUCCESS != err)
+        {
+           IOLockLock(gFSLock);
+           if (kFSOpening != gFSState) err = kIOReturnTimeout;
+           IOLockUnlock(gFSLock);
+       }

 
 

-    extent.start  = start;
-    extent.length = length;
+        if (KERN_SUCCESS != err)
+        {
+           HIBLOG("IOPolledFileOpen(%x)\n", err);
+            break;
+        }

 
 

-    ctx->extents->appendBytes(&extent, sizeof(extent));
-    ctx->size += length;
-}
+       // write extents for debug data usage in EFI
+        IOWriteExtentsToFile(vars->fileVars, kIOHibernateHeaderOpenSignature);

 
 

-IOReturn
-IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
-                           IOPolledFileIOVars ** fileVars, OSData ** fileExtents,
-                           OSData ** imagePath)
-{
-    IOReturn                   err = kIOReturnError;
-    IOPolledFileIOVars *       vars;
-    _OpenFileContext           ctx;
-    OSData *                   extentsData;
-    OSNumber *                 num;
-    IORegistryEntry *          part = 0;
-    OSDictionary *             matching;
-    OSIterator *               iter;
-    dev_t                      hibernate_image_dev;
-    uint64_t                   maxiobytes;
-
-    vars = &gFileVars;
-    do
-    {
-       HIBLOG("sizeof(IOHibernateImageHeader) == %ld\n", sizeof(IOHibernateImageHeader));
-       if (sizeof(IOHibernateImageHeader) != 512)
-           continue;
-    
-       vars->io           = false;
-       vars->buffer       = (uint8_t *) ioBuffer->getBytesNoCopy();
-       vars->bufferHalf   = 0;
-       vars->bufferOffset = 0;
-       vars->bufferSize   = ioBuffer->getLength() >> 1;
-    
-       extentsData = OSData::withCapacity(32);
-    
-       ctx.extents = extentsData;
-       ctx.size    = 0;
-       vars->fileRef = kern_open_file_for_direct_io(filename, 
-                                                   &file_extent_callback, &ctx, 
-                                                   &hibernate_image_dev,
-                                                    &vars->block0,
-                                                    &maxiobytes);
-       if (!vars->fileRef)
-       {
-           err = kIOReturnNoSpace;
-           break;
-       }
-       HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx\n", filename, ctx.size, 
-                    vars->block0, maxiobytes);
-       if (ctx.size < 1*1024*1024)             // check against image size estimate!
-       {
-           err = kIOReturnNoSpace;
-           break;
-       }
+        err = IOPolledFilePollersSetup(vars->fileVars, kIOPolledPreflightState);
+        if (KERN_SUCCESS != err) break;

 
 

-        if (maxiobytes < vars->bufferSize)
-            vars->bufferSize = maxiobytes;
-    
-       vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
-    
-       matching = IOService::serviceMatching("IOMedia");
-       num = OSNumber::withNumber(major(hibernate_image_dev), 32);
-       matching->setObject(kIOBSDMajorKey, num);
-       num->release();
-       num = OSNumber::withNumber(minor(hibernate_image_dev), 32);
-       matching->setObject(kIOBSDMinorKey, num);
-       num->release();
-       iter = IOService::getMatchingServices(matching);
-       matching->release();
-       if (iter)
-       {
-           part = (IORegistryEntry *) iter->getNextObject();
-           part->retain();
-           iter->release();
-       }
-    
-       int minor, major;
-       IORegistryEntry * next;
-       IORegistryEntry * child;
-       OSData * data;
+        clock_get_uptime(&startTime);
+        err = hibernate_setup(gIOHibernateCurrentHeader, 
+                                vmflush,
+                                vars->page_list, vars->page_list_wired, vars->page_list_pal);
+        clock_get_uptime(&endTime);
+        SUB_ABSOLUTETIME(&endTime, &startTime);
+        absolutetime_to_nanoseconds(endTime, &nsec);
+        HIBLOG("hibernate_setup(%d) took %qd ms\n", err, nsec / 1000000ULL);
+        if (KERN_SUCCESS != err) break;

 
 

-       num = (OSNumber *) part->getProperty(kIOBSDMajorKey);
-       if (!num)
-           break;
-       major = num->unsigned32BitValue();
-       num = (OSNumber *) part->getProperty(kIOBSDMinorKey);
-       if (!num)
-           break;
-       minor = num->unsigned32BitValue();
+        dsSSD = ((0 != (kIOPolledFileSSD & vars->fileVars->flags))
+                && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));

 
 

-       hibernate_image_dev = makedev(major, minor);
+        if (dsSSD) gIOHibernateCurrentHeader->options |= kIOHibernateOptionSSD | kIOHibernateOptionColor;
+        else       gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;

 
 

-        vars->pollers = OSArray::withCapacity(4);
-       if (!vars->pollers)
-           break;

 
 

-       vars->blockSize = 512;
-       next = part;
-       do
+#if defined(__i386__) || defined(__x86_64__)
+       if (!uuid_is_null(vars->volumeCryptKey) &&
+             (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey)))

        {
        {

-            IOPolledInterface * poller;
-           OSObject *          obj;
+           uintptr_t smcVars[2];
+           smcVars[0] = sizeof(vars->volumeCryptKey);
+           smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];

 
 

-           obj = next->getProperty(kIOPolledInterfaceSupportKey);
-           if (kOSBooleanFalse == obj)
-           {
-               vars->pollers->flushCollection();
-               break;
-           }
-            else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
-                vars->pollers->setObject(poller);
-           if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey))))
-               vars->blockSize = num->unsigned32BitValue();
-            child = next;
+           IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
+           bzero(smcVars, sizeof(smcVars));

        }
        }

-       while ((next = child->getParentEntry(gIOServicePlane)) 
-                && child->isParent(next, gIOServicePlane, true));
-
-       HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
-                   major, minor, vars->blockSize, vars->pollers->getCount());
-       if (vars->pollers->getCount() < kIOHibernateMinPollersNeeded)
-           continue;
+#endif

 
 

-       err = IOHibernatePollerProbe(vars, (IOService *) part);
-       if (kIOReturnSuccess != err)
-           break;

 
 

-       err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer);
-       if (kIOReturnSuccess != err)
-           break;
+        if (encryptedswap || !uuid_is_null(vars->volumeCryptKey))
+            gIOHibernateMode ^= kIOHibernateModeEncrypt; 

 
 

-       *fileVars    = vars;
-       *fileExtents = extentsData;
-    
-       // make imagePath
+        if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
+        {
+            vars->videoAllocSize = kVideoMapSize;
+            if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize, VM_KERN_MEMORY_IOKIT))
+                vars->videoMapping = 0;
+        }

 
 

-       if ((extentsData->getLength() >= sizeof(IOPolledFileExtent)))
-       {
-           char str2[24];
+       // generate crypt keys
+        for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
+            vars->wiredCryptKey[i] = random();
+        for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
+            vars->cryptKey[i] = random();

 
 

-#if __i386__
-           if (!gIOCreateEFIDevicePathSymbol)
-               gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
+       // set nvram

 
 

-           sprintf(str2, "%qx", vars->extentMap[0]);
+       IOSetBootImageNVRAM(nvramData);
+        nvramData->release();

 
 

-           err = IOService::getPlatform()->callPlatformFunction(
-                                               gIOCreateEFIDevicePathSymbol, false,
-                                               (void *) part, (void *) str2, (void *) true,
-                                               (void *) &data);
-#else
-           char str1[256];
-           int len = sizeof(str1);
+#if defined(__i386__) || defined(__x86_64__)
+       {
+           struct AppleRTCHibernateVars
+           {
+               uint8_t     signature[4];
+               uint32_t    revision;
+               uint8_t     booterSignature[20];
+               uint8_t     wiredCryptKey[16];
+           };
+           AppleRTCHibernateVars rtcVars;
+           OSData * data;

 
 

-           if (!part->getPath(str1, &len, gIODTPlane))
-               err = kIOReturnNotFound;
-           else
+           rtcVars.signature[0] = 'A';
+           rtcVars.signature[1] = 'A';
+           rtcVars.signature[2] = 'P';
+           rtcVars.signature[3] = 'L';
+           rtcVars.revision     = 1;
+           bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
+           if (gIOHibernateBootSignature[0])

            {
            {

-               sprintf(str2, ",%qx", vars->extentMap[0]);
-               // (strip the plane name)
-               char * tail = strchr(str1, ':');
-               if (!tail)
-                   tail = str1 - 1;
-               data = OSData::withBytes(tail + 1, strlen(tail + 1));
-               data->appendBytes(str2, strlen(str2));
+               char c;
+               uint8_t value = 0;
+               for (uint32_t i = 0;
+                   (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
+                   i++)
+               {
+                   if (c >= 'a')      c -= 'a' - 10;
+                   else if (c >= 'A') c -= 'A' - 10;
+                   else if (c >= '0') c -= '0';
+                   else               continue;
+                   value = (value << 4) | c;
+                   if (i & 1) rtcVars.booterSignature[i >> 1] = value;
+               }

            }
            }

-#endif
-       if (kIOReturnSuccess == err)
-           *imagePath = data;
-       else
-           HIBLOG("error 0x%x getting path\n", err);
+           data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
+           if (data)
+           { 
+               if (gIOHibernateRTCVariablesKey)
+                   IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
+               data->release();
+           }
+            if (gIOChosenEntry)
+            {
+                data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
+                if (data) gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
+               // set BootNext
+               if (!gIOHibernateBoot0082Data)
+               {
+                   data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
+                   if (data)
+                   {
+                       // AppleNVRAM_EFI_LOAD_OPTION
+                       struct {
+                           uint32_t Attributes;
+                           uint16_t FilePathLength;
+                           uint16_t Desc;
+                       } loadOptionHeader;
+                       loadOptionHeader.Attributes     = 1;
+                       loadOptionHeader.FilePathLength = data->getLength();
+                       loadOptionHeader.Desc           = 0;
+                       gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
+                       if (gIOHibernateBoot0082Data)
+                       {
+                           gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
+                           gIOHibernateBoot0082Data->appendBytes(data);
+                       }
+                   }
+               }
+               if (!gIOHibernateBootNextData)
+               {
+                   uint16_t bits = 0x0082;
+                   gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
+               }
+               if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData)
+               {
+                   gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
+                   gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
+                   gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
+               }
+               // BootNext
+            }

        }
        }

+#endif /* !i386 && !x86_64 */

     }
     while (false);
 
     }
     while (false);
 

-    if (kIOReturnSuccess != err)
+    IOLockLock(gFSLock);
+    if ((kIOReturnSuccess == err) && (kFSOpening != gFSState))
+    {
+       HIBLOG("hibernate file close due timeout\n");
+       err = kIOReturnTimeout;
+    }
+    if (kIOReturnSuccess == err)
+    {
+       gFSState = kFSOpened;
+       gIOHibernateVars = *vars;
+       gFileVars = *vars->fileVars;
+       gFileVars.allocated = false;
+       gIOHibernateVars.fileVars = &gFileVars;
+       gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
+       gIOHibernateState = kIOHibernateStateHibernating;
+    }
+    else

     {
     {

-        HIBLOG("error 0x%x opening hibernation file\n", err);
-       if (vars->fileRef)
-           kern_close_file_for_direct_io(vars->fileRef);
+       IOPolledFileIOVars * fileVars = vars->fileVars;
+       IOHibernateDone(vars);
+    IOPolledFileClose(&fileVars,
+#if DISABLE_TRIM
+                      0, NULL, 0, 0, 0);
+#else
+                      0, NULL, 0, sizeof(IOHibernateImageHeader), setFileSize);
+#endif
+       gFSState = kFSIdle;

     }
     }

+    IOLockUnlock(gFSLock);

 
 

-    if (part)
-       part->release();
+    if (vars->fileVars) IODelete(vars->fileVars, IOPolledFileIOVars, 1);
+    IODelete(vars, IOHibernateVars, 1);

 
     return (err);
 }
 
 
     return (err);
 }
 

-IOReturn
-IOPolledFileClose( IOPolledFileIOVars * vars )
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+static void
+IOSetBootImageNVRAM(OSData * data)

 {
 {

-    if (vars->pollers)
+    IORegistryEntry * regEntry;
+
+    if (!gIOOptionsEntry)

     {
     {

-       IOHibernatePollerClose(vars, kIOPolledPostflightState);
-        vars->pollers->release();
+        regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
+        gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
+        if (regEntry && !gIOOptionsEntry)
+            regEntry->release();
+    }
+    if (gIOOptionsEntry && gIOHibernateBootImageKey)
+    {
+       if (data) gIOOptionsEntry->setProperty(gIOHibernateBootImageKey, data);
+       else
+       {
+           gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
+           gIOOptionsEntry->sync();
+       }

     }
     }

-
-    gIOHibernateFileRef = vars->fileRef;
-
-    bzero(vars, sizeof(IOPolledFileIOVars));
-
-    return (kIOReturnSuccess);

 }
 
 }
 

+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+/* 
+ * Writes header to disk with signature, block size and file extents data.
+ * If there are more than 2 extents, then they are written on second block.
+ */

 static IOReturn
 static IOReturn

-IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
+IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature)

 {
 {

-    IOPolledFileExtent * extentMap;
+    IOHibernateImageHeader hdr;
+    IOItemCount            count;
+    IOReturn               err = kIOReturnSuccess;
+    int                    rc;
+    IOPolledFileExtent *   fileExtents;

 
 

-    extentMap = vars->extentMap;
+    fileExtents = (typeof(fileExtents)) vars->fileExtents->getBytesNoCopy(),

 
 

-    vars->position = position;
-
-    while (position >= extentMap->length)
+    memset(&hdr, 0, sizeof(IOHibernateImageHeader));
+    count = vars->fileExtents->getLength();
+    if (count > sizeof(hdr.fileExtentMap))

     {
     {

-       position -= extentMap->length;
-       extentMap++;
+        hdr.fileExtentMapSize = count;
+        count = sizeof(hdr.fileExtentMap);

     }
     }

+    else
+        hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap);

 
 

-    vars->currentExtent   = extentMap;
-    vars->extentRemaining = extentMap->length - position;
-    vars->extentPosition  = vars->position - position;
+    bcopy(fileExtents, &hdr.fileExtentMap[0], count);

 
 

-    if (vars->bufferSize <= vars->extentRemaining)
-       vars->bufferLimit = vars->bufferSize;
-    else
-       vars->bufferLimit = vars->extentRemaining;
+    // copy file block extent list if larger than header
+    if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap))
+    {
+            count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap);
+            rc = kern_write_file(vars->fileRef, vars->blockSize, 
+                                 (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)), 
+                                 count, IO_SKIP_ENCRYPTION);
+            if (rc != 0) {
+                HIBLOG("kern_write_file returned %d\n", rc);
+                err = kIOReturnIOError;
+                goto exit;
+            }    
+    }
+    hdr.signature = signature;
+    hdr.deviceBlockSize = vars->blockSize;
+
+    rc = kern_write_file(vars->fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION);
+    if (rc != 0) {
+        HIBLOG("kern_write_file returned %d\n", rc);
+        err = kIOReturnIOError;
+        goto exit;
+    }

 
 

-    return (kIOReturnSuccess);
+exit:
+    return err;

 }
 
 }
 

-static IOReturn
-IOPolledFileWrite(IOPolledFileIOVars * vars,
-                    const uint8_t * bytes, IOByteCount size,
-                    hibernate_cryptvars_t * cryptvars)
+void
+IOOpenDebugDataFile(const char *fname, uint64_t size)

 {
 {

-    IOReturn    err = kIOReturnSuccess;
-    IOByteCount copy;
-    bool       flush = false;
+    IOReturn   err;
+    OSData *   imagePath = NULL;
+    uint64_t   padding;

 
 

-    do
-    {
-       if (!bytes && !size)
-       {
-           // seek to end of block & flush
-           size = vars->position & (vars->blockSize - 1);
-           if (size)
-               size = vars->blockSize - size;
-           flush = true;
-            // use some garbage for the fill
-            bytes = vars->buffer + vars->bufferOffset;
-       }
+    if (!gDebugImageLock) {
+        gDebugImageLock = IOLockAlloc();
+    }

 
 

-       copy = vars->bufferLimit - vars->bufferOffset;
-       if (copy > size)
-           copy = size;
-       else
-           flush = true;
+    // Try to get a lock, but don't block for getting lock
+    if (!IOLockTryLock(gDebugImageLock)) {
+        HIBLOG("IOOpenDebugDataFile: Failed to get lock\n");
+        return;
+    }

 
 

-       if (bytes)
-       {
-           bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
-           bytes += copy;
-       }
-        else
-           bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
-        
-       size -= copy;
-       vars->bufferOffset += copy;
-       vars->position += copy;
+    if (gDebugImageFileVars ||  !fname || !size) {
+        HIBLOG("IOOpenDebugDataFile: conditions failed\n");
+        goto exit;
+    }

 
 

-       if (flush && vars->bufferOffset)
-       {
-           uint64_t offset = (vars->position - vars->bufferOffset 
-                               - vars->extentPosition + vars->currentExtent->start);
-           uint32_t length = (vars->bufferOffset);
+    padding = (PAGE_SIZE*2);  // allocate couple more pages for header and fileextents
+    err = IOPolledFileOpen(fname, size+padding, 32ULL*1024*1024*1024,
+                           NULL, 0,
+                           &gDebugImageFileVars, &imagePath, NULL, 0);

 
 

-            if (cryptvars && vars->encryptStart && (vars->position > vars->encryptStart))
-            {
-                uint32_t encryptLen, encryptStart;
-                encryptLen = vars->position - vars->encryptStart;
-                if (encryptLen > length)
-                    encryptLen = length;
-                encryptStart = length - encryptLen;
-                
-                // encrypt the buffer
-                aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
-                                &cryptvars->aes_iv[0],
-                                encryptLen / AES_BLOCK_SIZE,
-                                vars->buffer + vars->bufferHalf + encryptStart,
-                                &cryptvars->ctx.encrypt);
-                // save initial vector for following encrypts
-                bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
-                        &cryptvars->aes_iv[0],
-                        AES_BLOCK_SIZE);
-            }
+    if ((kIOReturnSuccess == err) && imagePath)
+    {
+        if ((gDebugImageFileVars->fileSize < (size+padding)) ||
+            (gDebugImageFileVars->fileExtents->getLength() > PAGE_SIZE)) {
+            // Can't use the file
+            IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
+            HIBLOG("IOOpenDebugDataFile: too many file extents\n");
+            goto exit;
+        }

 
 

-           if (vars->io)
-            {
-               err = IOHibernatePollerIODone(vars);
-                if (kIOReturnSuccess != err)
-                    break;
-            }
+        // write extents for debug data usage in EFI
+        IOWriteExtentsToFile(gDebugImageFileVars, kIOHibernateHeaderOpenSignature);
+        IOSetBootImageNVRAM(imagePath);
+        kprintf("IOOpenDebugDataFile: opened debugdata file\n");
+    }

 
 

-if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
-//if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
+exit:
+    IOLockUnlock(gDebugImageLock);

 
 

-           err = IOHibernatePollerIO(vars, kIOPolledWrite, vars->bufferHalf, offset, length);
-            if (kIOReturnSuccess != err)
-                break;
-           vars->io = true;
-
-           vars->extentRemaining -= vars->bufferOffset;
-           if (!vars->extentRemaining)
-           {
-               vars->currentExtent++;
-               vars->extentRemaining = vars->currentExtent->length;
-               vars->extentPosition  = vars->position;
-                if (!vars->extentRemaining)
-                {
-                    err = kIOReturnOverrun;
-                    break;
-                }
-           }
-
-           vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
-           vars->bufferOffset = 0;
-           if (vars->bufferSize <= vars->extentRemaining)
-               vars->bufferLimit = vars->bufferSize;
-           else
-               vars->bufferLimit = vars->extentRemaining;
+    if (imagePath) imagePath->release();
+    return;
+}

 
 

-           flush = false;
-       }
-    }
-    while (size);
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

 
 

-    return (err);
-}
+DECLARE_IOHIBERNATEPROGRESSALPHA

 
 

-static IOReturn
-IOPolledFileRead(IOPolledFileIOVars * vars,
-                    uint8_t * bytes, IOByteCount size,
-                    hibernate_cryptvars_t * cryptvars)
+static void
+ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)

 {
 {

-    IOReturn    err = kIOReturnSuccess;
-    IOByteCount copy;
-
-//    bytesWritten += size;
+    uint32_t   rowBytes, pixelShift;
+    uint32_t   x, y;
+    int32_t    blob;
+    uint32_t   alpha, in, color, result;
+    uint8_t *  out;
+    uint32_t   saveindex[kIOHibernateProgressCount] = { 0 };

 
 

-    do
+    rowBytes = display->rowBytes;
+    pixelShift = display->depth >> 4;
+    if (pixelShift < 1) return;
+    
+    screen += ((display->width
+                - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
+        + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
+    
+    for (y = 0; y < kIOHibernateProgressHeight; y++)

     {
     {

-       copy = vars->bufferLimit - vars->bufferOffset;
-       if (copy > size)
-           copy = size;
-
-       if (bytes)
-       {
-           bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
-           bytes += copy;
-       }
-       size -= copy;
-       vars->bufferOffset += copy;
-//     vars->position += copy;
-
-       if (vars->bufferOffset == vars->bufferLimit)
-       {
-           if (vars->io)
+        out = screen + y * rowBytes;
+        for (blob = 0; blob < kIOHibernateProgressCount; blob++)
+        {
+            color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
+            for (x = 0; x < kIOHibernateProgressWidth; x++)

             {
             {

-               err = IOHibernatePollerIODone(vars);
-                if (kIOReturnSuccess != err)
-                    break;
+                alpha  = gIOHibernateProgressAlpha[y][x];
+                result = color;
+                if (alpha)
+                {
+                    if (0xff != alpha)
+                    {
+                        if (1 == pixelShift)
+                        {
+                            in = *((uint16_t *)out) & 0x1f;    // 16
+                            in = (in << 3) | (in >> 2);
+                        }
+                        else
+                            in = *((uint32_t *)out) & 0xff;    // 32
+                        saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
+                        result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
+                    }
+                    if (1 == pixelShift)
+                    {
+                        result >>= 3;
+                        *((uint16_t *)out) = (result << 10) | (result << 5) | result;  // 16
+                    }
+                    else
+                        *((uint32_t *)out) = (result << 16) | (result << 8) | result;  // 32
+                }
+                out += (1 << pixelShift);

             }
             }

-            else
-                cryptvars = 0;
-
-if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
+            out += (kIOHibernateProgressSpacing << pixelShift);
+        }
+    }
+}

 
 

-           vars->position += vars->lastRead;
-           vars->extentRemaining -= vars->lastRead;
-           vars->bufferLimit = vars->lastRead;

 
 

-           if (!vars->extentRemaining)
-           {
-               vars->currentExtent++;
-               vars->extentRemaining = vars->currentExtent->length;
-               vars->extentPosition  = vars->position;
-                if (!vars->extentRemaining)
-                {
-                    err = kIOReturnOverrun;
-                    break;
-                }
-           }
+static void
+ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
+{
+    uint32_t  rowBytes, pixelShift;
+    uint32_t  x, y;
+    int32_t   blob, lastBlob;
+    uint32_t  alpha, in, color, result;
+    uint8_t * out;
+    uint32_t  saveindex[kIOHibernateProgressCount] = { 0 };

 
 

-           if (vars->extentRemaining <= vars->bufferSize)
-               vars->lastRead = vars->extentRemaining;
-           else
-               vars->lastRead = vars->bufferSize;
+    pixelShift = display->depth >> 4;
+    if (pixelShift < 1)
+        return;

 
 

-           uint64_t offset = (vars->position 
-                               - vars->extentPosition + vars->currentExtent->start);
-           uint64_t length = (vars->lastRead);
+    rowBytes = display->rowBytes;

 
 

-//if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
+    screen += ((display->width 
+            - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
+                + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;

 
 

-           err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
-            if (kIOReturnSuccess != err)
-                break;
-           vars->io = true;
+    lastBlob  = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);

 
 

-           vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
-           vars->bufferOffset = 0;
+    screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;

 
 

-            if (cryptvars)
+    for (y = 0; y < kIOHibernateProgressHeight; y++)
+    {
+        out = screen + y * rowBytes;
+        for (blob = firstBlob; blob <= lastBlob; blob++)
+        {
+            color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
+            for (x = 0; x < kIOHibernateProgressWidth; x++)

             {
             {

-                uint8_t thisVector[AES_BLOCK_SIZE];
-                // save initial vector for following decrypts
-                bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
-                bcopy(vars->buffer + vars->bufferHalf + vars->lastRead - AES_BLOCK_SIZE, 
-                        &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
-                // decrypt the buffer
-                aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
-                                &thisVector[0],
-                                vars->lastRead / AES_BLOCK_SIZE,
-                                vars->buffer + vars->bufferHalf,
-                                &cryptvars->ctx.decrypt);
+                alpha  = gIOHibernateProgressAlpha[y][x];
+                result = color;
+                if (alpha)
+                {
+                    if (0xff != alpha)
+                    {
+                        in = display->progressSaveUnder[blob][saveindex[blob]++];
+                        result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
+                    }
+                    if (1 == pixelShift)
+                    {
+                        result >>= 3;
+                        *((uint16_t *)out) = (result << 10) | (result << 5) | result;  // 16
+                    }
+                    else
+                        *((uint32_t *)out) = (result << 16) | (result << 8) | result;  // 32
+                }
+                out += (1 << pixelShift);

             }
             }

-       }
+            out += (kIOHibernateProgressSpacing << pixelShift);
+        }

     }
     }

-    while (size);
-
-    return (err);

 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

-               
+

 IOReturn
 IOReturn

-IOHibernateSystemSleep(void)
+IOHibernateIOKitSleep(void)

 {
 {

-    IOReturn   err;
-    OSData *   data;
-    OSObject * obj;
-    OSString * str;
-    OSNumber * num;
-
-    IOHibernateVars * vars  = &gIOHibernateVars;
+    IOReturn ret = kIOReturnSuccess;
+    IOLockLock(gFSLock);
+    if (kFSOpening == gFSState)
+    {
+       gFSState = kFSTimedOut;
+       HIBLOG("hibernate file open timed out\n");
+       ret = kIOReturnTimeout;
+    }
+    IOLockUnlock(gFSLock);
+    return (ret);
+}

 
 

-    if (vars->fileVars && vars->fileVars->fileRef)
-       // already on the way down
-       return (kIOReturnSuccess);
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

 
 

-    gIOHibernateState = kIOHibernateStateInactive;
+IOReturn
+IOHibernateSystemHasSlept(void)
+{
+    IOReturn          ret = kIOReturnSuccess;
+    IOHibernateVars * vars  = &gIOHibernateVars;
+    OSObject        * obj = 0;
+    OSData          * data;

 
 

-    if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateModeKey)))
+    IOLockLock(gFSLock);
+    if ((kFSOpened != gFSState) && gIOHibernateMode)

     {
     {

-       if ((num = OSDynamicCast(OSNumber, obj)))
-           gIOHibernateMode = num->unsigned32BitValue();
-        if (kIOHibernateModeSleep & gIOHibernateMode)
-            // default to discard clean for safe sleep
-            gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive 
-                                | kIOHibernateModeDiscardCleanActive);
-
-       obj->release();
+       ret = kIOReturnTimeout;

     }
     }

-    if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeRatioKey)))
-    {
-       if ((num = OSDynamicCast(OSNumber, obj)))
-           gIOHibernateFreeRatio = num->unsigned32BitValue();
+    IOLockUnlock(gFSLock);
+    if (kIOReturnSuccess != ret) return (ret);
+
+    if (gIOHibernateMode) obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
+    vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
+    if (obj && !vars->previewBuffer)

        obj->release();
        obj->release();

-    }
-    if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeTimeKey)))
+
+    vars->consoleMapping = NULL;
+    if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))

     {
     {

-       if ((num = OSDynamicCast(OSNumber, obj)))
-           gIOHibernateFreeTime = num->unsigned32BitValue();
-       obj->release();
+       vars->previewBuffer->release();
+       vars->previewBuffer = 0;

     }
     }

-    if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
-    {
-       if ((str = OSDynamicCast(OSString, obj)))
-           strcpy(gIOHibernateFilename, str->getCStringNoCopy());
-       obj->release();
-    }
-
-    if (!gIOHibernateMode || !gIOHibernateFilename[0])
-       return (kIOReturnUnsupported);
-
-    HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);

 
 

-    do
+    if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
+        && vars->previewBuffer 
+        && (data = OSDynamicCast(OSData, 
+       IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))

     {
     {

-        vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
-                                   4 * page_size, page_size);
-        vars->ioBuffer  = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, 
-                                   2 * kDefaultIOSize, page_size);
+       UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
+       HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);

 
 

-        if (!vars->srcBuffer || !vars->ioBuffer)
-        {
-            err = kIOReturnNoMemory;
-            break;
-        }
+       IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);

 
 

-        err = IOPolledFileOpen(gIOHibernateFilename, vars->ioBuffer,
-                                &vars->fileVars, &vars->fileExtents, &data);
-        if (KERN_SUCCESS != err)
-        {
-           HIBLOG("IOPolledFileOpen(%x)\n", err);
-            break;
-        }
-       if (vars->fileVars->fileRef)
+       if (kIOHibernatePreviewUpdates & flags)

        {
        {

-           // invalidate the image file
-           gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
-           int err = kern_write_file(vars->fileVars->fileRef, 0,
-                                       (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
-            if (KERN_SUCCESS != err)
-                HIBLOG("kern_write_file(%d)\n", err);
+           PE_Video           consoleInfo;
+           hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
+
+           IOService::getPlatform()->getConsoleInfo(&consoleInfo);
+
+           graphicsInfo->width    = consoleInfo.v_width;
+           graphicsInfo->height   = consoleInfo.v_height;
+           graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
+           graphicsInfo->depth    = consoleInfo.v_depth;
+           vars->consoleMapping   = (uint8_t *) consoleInfo.v_baseAddr;
+
+           HIBPRINT("video %p %d %d %d\n",
+                       vars->consoleMapping, graphicsInfo->depth, 
+                       graphicsInfo->width, graphicsInfo->height);
+           if (vars->consoleMapping)
+                       ProgressInit(graphicsInfo, vars->consoleMapping,
+                                       &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));

        }
        }

+    }

 
 

-       bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
-
-        boolean_t encryptedswap;
-        err = hibernate_setup(gIOHibernateCurrentHeader, 
-                                gIOHibernateFreeRatio, gIOHibernateFreeTime,
-                                &vars->page_list, &vars->page_list_wired, &encryptedswap);
-        if (KERN_SUCCESS != err)
-        {
-           HIBLOG("hibernate_setup(%d)\n", err);
-            break;
-        }
-
-        if (encryptedswap)
-            gIOHibernateMode ^= kIOHibernateModeEncrypt; 
-
-        vars->videoAllocSize = kVideoMapSize;
-        if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
-            vars->videoMapping = 0;
-
-       // generate crypt keys
-        for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
-            vars->wiredCryptKey[i] = random();
-        for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
-            vars->cryptKey[i] = random();
+    if (gIOOptionsEntry)
+        gIOOptionsEntry->sync();

 
 

-       // set nvram
+    return (ret);
+}

 
 

-        IORegistryEntry * regEntry;
-        if (!gIOOptionsEntry)
-        {
-            regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
-            gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
-            if (regEntry && !gIOOptionsEntry)
-                regEntry->release();
-        }
-        if (!gIOChosenEntry)
-            gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

 
 

-       if (gIOOptionsEntry)
+static DeviceTreeNode *
+MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry)
+{
+    DeviceTreeNodeProperty * prop;
+    DeviceTreeNode *         child;
+    IORegistryEntry *        childRegEntry;
+    const char *             nameProp;
+    unsigned int             propLen, idx;
+
+    prop = (DeviceTreeNodeProperty *) (entry + 1);
+    for (idx = 0; idx < entry->nProperties; idx++)
+    {
+       if (regEntry && (0 != strcmp("name", prop->name)))

        {
        {

-            const OSSymbol *  sym;
-
-            sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
-            if (sym)
-            {
-                gIOOptionsEntry->setProperty(sym, data);
-                sym->release();
-            }
-            data->release();
-
-#ifdef __ppc__
-            size_t           len;
-            char              valueString[16];
-
-           vars->saveBootDevice = gIOOptionsEntry->copyProperty(kIOSelectedBootDeviceKey);
-            if (gIOChosenEntry)
-            {
-               OSData * bootDevice = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOBootPathKey));
-               if (bootDevice)
-               {
-                   sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
-                   OSString * str2 = OSString::withCStringNoCopy((const char *) bootDevice->getBytesNoCopy());
-                   if (sym && str2)
-                       gIOOptionsEntry->setProperty(sym, str2);
-                   if (sym)
-                       sym->release();
-                   if (str2)
-                       str2->release();
-               }
-                data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMemorySignatureKey));
-                if (data)
-                {
-                    vars->haveFastBoot = true;
-
-                    len = sprintf(valueString, "0x%lx", *((UInt32 *)data->getBytesNoCopy()));
-                    data = OSData::withBytes(valueString, len + 1);
-                    sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
-                    if (sym && data)
-                        gIOOptionsEntry->setProperty(sym, data);
-                    if (sym)
-                        sym->release();
-                    if (data)
-                        data->release();
-                }
-                data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
-                if (data)
-                    gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
-            }
-#endif /* __ppc__ */
-#ifdef __i386__
-           struct AppleRTCHibernateVars
-           {
-               uint8_t     signature[4];
-               uint32_t    revision;
-               uint8_t     booterSignature[20];
-               uint8_t     wiredCryptKey[16];
-           };
-           AppleRTCHibernateVars rtcVars;
-
-           rtcVars.signature[0] = 'A';
-           rtcVars.signature[1] = 'A';
-           rtcVars.signature[2] = 'P';
-           rtcVars.signature[3] = 'L';
-           rtcVars.revision     = 1;
-           bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
-           if (gIOHibernateBootSignature[0])
-           {
-               char c;
-               uint8_t value = 0;
-               for (uint32_t i = 0;
-                   (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
-                   i++)
-               {
-                   if (c >= 'a')
-                       c -= 'a' - 10;
-                   else if (c >= 'A')
-                       c -= 'A' - 10;
-                   else if (c >= '0')
-                       c -= '0';
-                   else
-                       continue;
-                   value = (value << 4) | c;
-                   if (i & 1)
-                       rtcVars.booterSignature[i >> 1] = value;
-               }
-           }
-           data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
-           if (data)
-           {
-               IOService::getPMRootDomain()->setProperty(kIOHibernateRTCVariablesKey, data);
-               data->release();
-           }
-            if (gIOChosenEntry)
-            {
-                data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
-                if (data)
-                    gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
-            }
-#else /* !__i386__ */
-            if (kIOHibernateModeEncrypt & gIOHibernateMode)
-            {
-                data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
-                sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
-                if (sym && data)
-                    gIOOptionsEntry->setProperty(sym, data);
-                if (sym)
-                    sym->release();
-                if (data)
-                    data->release();
-                if (gIOHibernateBootSignature[0])
-                {
-                    data = OSData::withCapacity(16);
-                    sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
-                    if (sym && data)
-                    {
-                        char c;
-                        uint8_t value = 0;
-                        for (uint32_t i = 0; (c = gIOHibernateBootSignature[i]); i++)
-                        {
-                            if (c >= 'a')
-                                c -= 'a' - 10;
-                            else if (c >= 'A')
-                                c -= 'A' - 10;
-                            else if (c >= '0')
-                                c -= '0';
-                            else
-                                continue;
-                            value = (value << 4) | c;
-                            if (i & 1)
-                                data->appendBytes(&value, sizeof(value));
-                        }
-                        gIOOptionsEntry->setProperty(sym, data);
-                    }
-                    if (sym)
-                        sym->release();
-                    if (data)
-                        data->release();
-                }
-            }
-            if (!vars->haveFastBoot)
-            {
-                // set boot volume to zero
-                IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
-                if (platform && (kIOReturnSuccess == platform->readXPRAM(kXPRamAudioVolume, 
-                                            &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume))))
-                {
-                    uint8_t newVolume;
-                    newVolume = vars->saveBootAudioVolume & 0xf8;
-                    platform->writeXPRAM(kXPRamAudioVolume, 
-                                            &newVolume, sizeof(newVolume));
-                }
-            }
-#endif /* !__i386__ */
+           regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
+//         HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);

        }
        }

-       // --
-
-       gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
-       gIOHibernateState = kIOHibernateStateHibernating;
+       prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));

     }
     }

-    while (false);

 
 

-    return (err);
+    child = (DeviceTreeNode *) prop;
+    for (idx = 0; idx < entry->nChildren; idx++)
+    {
+       if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen))
+           panic("no name");
+       childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
+//     HIBPRINT("%s == %p\n", nameProp, childRegEntry);
+       child = MergeDeviceTree(child, childRegEntry);
+    }
+    return (child);

 }
 
 }
 

-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-

 IOReturn
 IOReturn

-IOHibernateSystemHasSlept(void)
+IOHibernateSystemWake(void)

 {
 {

-    IOHibernateVars * vars  = &gIOHibernateVars;
-
-    if ((vars->previewData = OSDynamicCast(OSData, 
-            IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewBufferKey))))
+    if (kFSOpened == gFSState)

     {
     {

-        vars->previewBuffer = IOMemoryDescriptor::withAddress(
-                                    (void *) vars->previewData->getBytesNoCopy(), 
-                                    vars->previewData->getLength(), 
-                                    kIODirectionInOut);
-
-        if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
-        {
-            vars->previewBuffer->release();
-            vars->previewBuffer = 0;
-        }
-        if (!vars->previewBuffer)
-            vars->previewData = 0;
+       IOPolledFilePollersClose(gIOHibernateVars.fileVars, kIOPolledPostflightState);
+       IOHibernateDone(&gIOHibernateVars);
+    }
+    else
+    {
+        IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
+        IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);

     }
     }

-    if (gIOOptionsEntry)
-        gIOOptionsEntry->sync();
-

     return (kIOReturnSuccess);
 }
 
     return (kIOReturnSuccess);
 }
 

-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-IOReturn
-IOHibernateSystemWake(void)
+static IOReturn
+IOHibernateDone(IOHibernateVars * vars)

 {
 {

-    IOHibernateVars * vars  = &gIOHibernateVars;
-
-    hibernate_teardown(vars->page_list, vars->page_list_wired);
+    hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);

 
     if (vars->videoMapping)
     {
 
     if (vars->videoMapping)
     {


@@ -1294,7 +1113,7 @@ IOHibernateSystemWake(void)
             IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
         if (vars->videoAllocSize)
             // dealloc range
             IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
         if (vars->videoAllocSize)
             // dealloc range

-            kmem_free(kernel_map, trunc_page_32(vars->videoMapping), vars->videoAllocSize);
+            kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);

     }
 
     if (vars->previewBuffer)
     }
 
     if (vars->previewBuffer)


@@ -1303,79 +1122,106 @@ IOHibernateSystemWake(void)
         vars->previewBuffer = 0;
     }
 
         vars->previewBuffer = 0;
     }
 

-    if (vars->fileVars)
+    if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
+    {
+        IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey, 
+                                            gIOHibernateCurrentHeader->options, 32);
+    }
+    else
+    {
+        IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
+    }
+
+    if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
+      && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus))
+    {
+        IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey, 
+                                        &gIOHibernateGraphicsInfo->gfxStatus,
+                                        sizeof(gIOHibernateGraphicsInfo->gfxStatus));
+    }
+    else

     {
     {

-       IOPolledFileClose(vars->fileVars);
+        IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);

     }
 
     // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
 
     }
 
     // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
 

-#ifdef __ppc__
-    OSData * data = OSData::withCapacity(4);
-    if (gIOOptionsEntry && data)
-    {
-        const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
-        if (sym)
-        {
-            gIOOptionsEntry->setProperty(sym, data);
-            sym->release();
-        }
-        sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
-        if (sym)
-        {
-           if (vars->saveBootDevice)
+#if defined(__i386__) || defined(__x86_64__)
+       IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
+       IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
+
+       /*
+        * Hibernate variable is written to NVRAM on platforms in which RtcRam
+        * is not backed by coin cell.  Remove Hibernate data from NVRAM.
+        */
+       if (gIOOptionsEntry) {
+
+           if (gIOHibernateRTCVariablesKey) {
+               if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
+                   gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
+               }
+           }
+
+           if (gIOHibernateBootNextKey)

            {
            {

-               gIOOptionsEntry->setProperty(sym, vars->saveBootDevice);
-               vars->saveBootDevice->release();
+               if (gIOHibernateBootNextSave)
+               {
+                   gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
+                   gIOHibernateBootNextSave->release();
+                   gIOHibernateBootNextSave = NULL;
+               }
+               else
+                   gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);

            }
            }

-            sym->release();
-        }
-        sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
-        if (sym)
-        {
-            gIOOptionsEntry->setProperty(sym, data);
-            sym->release();
-        }
-        sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
-        if (sym)
-        {
-            gIOOptionsEntry->removeProperty(sym);
-            sym->release();
-        }
-    }
-    if (data)
-        data->release();
+           if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) gIOOptionsEntry->sync();
+       }
+#endif

 
 

-    if (gIOOptionsEntry)
+    if (vars->srcBuffer) vars->srcBuffer->release();
+    bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
+    if (vars->handoffBuffer)

     {
     {

-       if (!vars->haveFastBoot)
+       if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)

        {
        {

-           // reset boot audio volume
-           IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
-           if (platform)
-               platform->writeXPRAM(kXPRamAudioVolume, 
-                                       &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume));
+           IOHibernateHandoff * handoff;
+           bool done = false;
+           for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
+                !done;
+                handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
+           {
+               HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
+               uint8_t * data = &handoff->data[0];
+               switch (handoff->type)
+               {
+                   case kIOHibernateHandoffTypeEnd:
+                       done = true;
+                       break;
+
+                   case kIOHibernateHandoffTypeDeviceTree:
+                       MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
+                       break;
+       
+                   case kIOHibernateHandoffTypeKeyStore:
+#if defined(__i386__) || defined(__x86_64__)
+                       {
+                           IOBufferMemoryDescriptor *
+                           md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
+                           if (md)
+                           {
+                               IOSetKeyStoreData(md);
+                           }
+                       }
+#endif
+                       break;
+       
+                   default:
+                       done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
+                       break;
+               }    
+           }

        }
        }

-
-       // sync now to hardware if the booter has not
-       if (kIOHibernateStateInactive == gIOHibernateState)
-           gIOOptionsEntry->sync();
-       else
-           // just sync the variables in case a later panic syncs nvram (it won't sync variables)
-           gIOOptionsEntry->syncOFVariables();
+       vars->handoffBuffer->release();

     }
     }

-#endif
-
-#ifdef __i386__
-    IOService::getPMRootDomain()->removeProperty(kIOHibernateRTCVariablesKey);
-#endif
-
-    if (vars->srcBuffer)
-       vars->srcBuffer->release();
-    if (vars->ioBuffer)
-       vars->ioBuffer->release();
-    if (vars->fileExtents)
-       vars->fileExtents->release();

 
     bzero(vars, sizeof(*vars));
 
 
     bzero(vars, sizeof(*vars));
 


@@ -1387,26 +1233,100 @@ IOHibernateSystemWake(void)
 IOReturn
 IOHibernateSystemPostWake(void)
 {
 IOReturn
 IOHibernateSystemPostWake(void)
 {

-    if (gIOHibernateFileRef)
+    gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
+    if (kFSOpened == gFSState)

     {
     {

-       // invalidate the image file
-       gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
-       int err = kern_write_file(gIOHibernateFileRef, 0,
-                                   (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
-       if (KERN_SUCCESS != err)
-           HIBLOG("kern_write_file(%d)\n", err);
+       // invalidate & close the image file
+       IOSleep(TRIM_DELAY);
+       IOPolledFileIOVars * vars = &gFileVars;
+       IOPolledFileClose(&vars,
+#if DISABLE_TRIM
+                                      0, NULL, 0, 0, 0);
+#else
+                                      0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader),
+                                      sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize);
+#endif
+    }
+    gFSState = kFSIdle;
+
+    IOSetBootImageNVRAM(0);

 
 

-       kern_close_file_for_direct_io(gIOHibernateFileRef);
-        gIOHibernateFileRef = 0;
+    if (gDebugImageLock) {
+        IOLockLock(gDebugImageLock);
+        if (gDebugImageFileVars != 0) {
+            kprintf("IOHibernateSystemPostWake: Closing debugdata file\n");
+            IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
+        }
+        IOLockUnlock(gDebugImageLock);

     }
     }

+

     return (kIOReturnSuccess);
 }
 
     return (kIOReturnSuccess);
 }
 

+bool IOHibernateWasScreenLocked(void)
+{
+    bool ret = false;
+    if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry)
+    {
+       OSData *
+       data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
+       if (data) switch (*((uint32_t *)data->getBytesNoCopy()))
+       {
+           case kIOScreenLockLocked:
+           case kIOScreenLockFileVaultDialog:
+               ret = true;
+               break;
+           case kIOScreenLockNoLock:
+           case kIOScreenLockUnlocked:
+           default:
+               ret = false;
+               break;
+       }
+    }
+    return (ret);
+}
+

 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 

+SYSCTL_STRING(_kern, OID_AUTO, hibernatefile, 
+               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
+               gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
+SYSCTL_STRING(_kern, OID_AUTO, bootsignature, 
+               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
+               gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatemode, 
+               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
+               &gIOHibernateMode, 0, "");
+SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
+               CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
+               gIOHibernateStats, hibernate_statistics_t, "");
+
+SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
+               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+               &gIOHibernateStats->graphicsReadyTime, 0, "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
+               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+               &gIOHibernateStats->wakeNotificationTime, 0, "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
+               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+               &gIOHibernateStats->lockScreenReadyTime, 0, "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
+               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+               &gIOHibernateStats->hidReadyTime, 0, "");
+
+

 void
 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
 {
 void
 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
 {

+    gIOHibernateBootImageKey = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
+
+#if defined(__i386__) || defined(__x86_64__)
+    gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
+    gIOHibernateBoot0082Key     = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
+    gIOHibernateBootNextKey     = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
+    gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
+#endif /* defined(__i386__) || defined(__x86_64__) */
+

     OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
     if (data)
     {
     OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
     if (data)
     {


@@ -1414,81 +1334,117 @@ IOHibernateSystemInit(IOPMrootDomain * rootDomain)
        data->release();
     }
 
        data->release();
     }
 

-    if (PE_parse_boot_arg("hfile", gIOHibernateFilename))
+    if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))

        gIOHibernateMode = kIOHibernateModeOn;
     else
        gIOHibernateFilename[0] = 0;
 
        gIOHibernateMode = kIOHibernateModeOn;
     else
        gIOHibernateFilename[0] = 0;
 

-    static SYSCTL_STRING(_kern, OID_AUTO, hibernatefile, 
-                               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, 
-                               gIOHibernateFilename, sizeof(gIOHibernateFilename), "");

     sysctl_register_oid(&sysctl__kern_hibernatefile);
     sysctl_register_oid(&sysctl__kern_hibernatefile);

-
-    static SYSCTL_STRING(_kern, OID_AUTO, bootsignature, 
-                               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, 
-                               gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");

     sysctl_register_oid(&sysctl__kern_bootsignature);
     sysctl_register_oid(&sysctl__kern_bootsignature);

-
-    static SYSCTL_UINT(_kern, OID_AUTO, hibernatemode, 
-                               CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, 
-                               &gIOHibernateMode, 0, "");

     sysctl_register_oid(&sysctl__kern_hibernatemode);
     sysctl_register_oid(&sysctl__kern_hibernatemode);

+    sysctl_register_oid(&sysctl__kern_hibernatestatistics);
+    sysctl_register_oid(&sysctl__kern_hibernategraphicsready);
+    sysctl_register_oid(&sysctl__kern_hibernatewakenotification);
+    sysctl_register_oid(&sysctl__kern_hibernatelockscreenready);
+    sysctl_register_oid(&sysctl__kern_hibernatehidready);
+
+    gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
+
+    gFSLock = IOLockAlloc();

 }
 
 }
 

+

 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 static void
 hibernate_setup_for_wake(void)
 {
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 static void
 hibernate_setup_for_wake(void)
 {

-#if __ppc__
-    // go slow (state needed for wake)
-    ml_set_processor_speed(1);
-#endif

 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 

-#define C_ASSERT(e) typedef char    __C_ASSERT__[(e) ? 1 : -1]
+static IOReturn 
+IOHibernatePolledFileWrite(IOPolledFileIOVars * vars,
+                          const uint8_t * bytes, IOByteCount size,
+                          IOPolledFileCryptVars * cryptvars)
+{
+    IOReturn err;
+
+    err = IOPolledFileWrite(vars, bytes, size, cryptvars);
+    if ((kIOReturnSuccess == err) && hibernate_should_abort()) err = kIOReturnAborted;
+
+    return (err);
+}

 
 

-extern "C" boolean_t
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+extern "C" uint32_t

 hibernate_write_image(void)
 {
     IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
     IOHibernateVars *        vars  = &gIOHibernateVars;
     IOPolledFileExtent *     fileExtents;
 
 hibernate_write_image(void)
 {
     IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
     IOHibernateVars *        vars  = &gIOHibernateVars;
     IOPolledFileExtent *     fileExtents;
 

-    C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
+    assert_static(sizeof(IOHibernateImageHeader) == 512);

 
     uint32_t    pageCount, pagesDone;
     IOReturn     err;
 
     uint32_t    pageCount, pagesDone;
     IOReturn     err;

-    vm_offset_t  ppnum;
-    IOItemCount  page, count;
+    vm_offset_t  ppnum, page;
+    IOItemCount  count;

     uint8_t *   src;
     uint8_t *   data;
     uint8_t *   src;
     uint8_t *   data;

+    uint8_t *   compressed;
+    uint8_t *   scratch;

     IOByteCount  pageCompressedSize;
     uint64_t    compressedSize, uncompressedSize;
     uint64_t    image1Size = 0;
     uint32_t    bitmap_size;
     IOByteCount  pageCompressedSize;
     uint64_t    compressedSize, uncompressedSize;
     uint64_t    image1Size = 0;
     uint32_t    bitmap_size;

-    bool        iterDone, pollerOpen, needEncryptStart;
+    bool        iterDone, pollerOpen, needEncrypt;

     uint32_t    restore1Sum, sum, sum1, sum2;
     uint32_t    restore1Sum, sum, sum1, sum2;

+    int          wkresult;

     uint32_t    tag;
     uint32_t    pageType;
     uint32_t    pageAndCount[2];
     uint32_t    tag;
     uint32_t    pageType;
     uint32_t    pageAndCount[2];

+    addr64_t     phys64;
+    IOByteCount  segLen;
+    uintptr_t    hibernateBase;
+    uintptr_t    hibernateEnd;

 
     AbsoluteTime startTime, endTime;
 
     AbsoluteTime startTime, endTime;

-    AbsoluteTime allTime, compTime, decoTime;
+    AbsoluteTime allTime, compTime;
+    uint64_t     compBytes;

     uint64_t     nsec;
     uint32_t     lastProgressStamp = 0;
     uint32_t     progressStamp;
     uint64_t     nsec;
     uint32_t     lastProgressStamp = 0;
     uint32_t     progressStamp;

+    uint32_t    blob, lastBlob = (uint32_t) -1L;

 
 

-    hibernate_cryptvars_t _cryptvars;
-    hibernate_cryptvars_t * cryptvars = 0;
+    uint32_t    wiredPagesEncrypted;
+    uint32_t    dirtyPagesEncrypted;
+    uint32_t    wiredPagesClear;
+    uint32_t    svPageCount;
+    uint32_t    zvPageCount;

 
 

-    if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
+    IOPolledFileCryptVars _cryptvars;
+    IOPolledFileCryptVars * cryptvars = 0;
+
+    wiredPagesEncrypted = 0;
+    dirtyPagesEncrypted = 0;
+    wiredPagesClear     = 0;
+    svPageCount         = 0;
+    zvPageCount         = 0;
+
+    if (!vars->fileVars || !vars->fileVars->pollers)

         return (false /* sleep */ );
 
         return (false /* sleep */ );
 

+    if (kIOHibernateModeSleep & gIOHibernateMode)
+       kdebug_enable = save_kdebug_enable;
+
+    KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START, 0, 0, 0, 0, 0);
+    IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
+

     restore1Sum = sum1 = sum2 = 0;
 
     restore1Sum = sum1 = sum2 = 0;
 

+#if CRYPTO

     // encryption data. "iv" is the "initial vector".
     if (kIOHibernateModeEncrypt & gIOHibernateMode)
     {
     // encryption data. "iv" is the "initial vector".
     if (kIOHibernateModeEncrypt & gIOHibernateMode)
     {


@@ -1497,7 +1453,7 @@ hibernate_write_image(void)
              0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
     
         cryptvars = &gIOHibernateCryptWakeContext;
              0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
     
         cryptvars = &gIOHibernateCryptWakeContext;

-        bzero(cryptvars, sizeof(hibernate_cryptvars_t));
+        bzero(cryptvars, sizeof(IOPolledFileCryptVars));

         aes_encrypt_key(vars->cryptKey,
                         kIOHibernateAESKeySize,
                         &cryptvars->ctx.encrypt);
         aes_encrypt_key(vars->cryptKey,
                         kIOHibernateAESKeySize,
                         &cryptvars->ctx.encrypt);


@@ -1506,7 +1462,10 @@ hibernate_write_image(void)
                         &cryptvars->ctx.decrypt);
 
         cryptvars = &_cryptvars;
                         &cryptvars->ctx.decrypt);
 
         cryptvars = &_cryptvars;

-        bzero(cryptvars, sizeof(hibernate_cryptvars_t));
+        bzero(cryptvars, sizeof(IOPolledFileCryptVars));
+        for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++)
+            vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
+        bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));

         aes_encrypt_key(vars->wiredCryptKey,
                         kIOHibernateAESKeySize,
                         &cryptvars->ctx.encrypt);
         aes_encrypt_key(vars->wiredCryptKey,
                         kIOHibernateAESKeySize,
                         &cryptvars->ctx.encrypt);


@@ -1514,18 +1473,23 @@ hibernate_write_image(void)
         bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
         bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
         bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
         bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
         bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
         bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));

-        bzero(gIOHibernateCryptWakeVars, sizeof(hibernate_cryptwakevars_t));

     }
     }

+#endif /* CRYPTO */

 
     hibernate_setup_for_wake();
 
     hibernate_page_list_setall(vars->page_list,
                                vars->page_list_wired,
 
     hibernate_setup_for_wake();
 
     hibernate_page_list_setall(vars->page_list,
                                vars->page_list_wired,

+                               vars->page_list_pal,
+                              false /* !preflight */,
+                              /* discard_all */
+                              ((0 == (kIOHibernateModeSleep & gIOHibernateMode)) 
+                              && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),

                                &pageCount);
 
     HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
 
                                &pageCount);
 
     HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
 

-    fileExtents = (IOPolledFileExtent *) vars->fileExtents->getBytesNoCopy();
+    fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy();

 
 #if 0
     count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
 
 #if 0
     count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);


@@ -1537,25 +1501,25 @@ hibernate_write_image(void)
     }
 #endif
 
     }
 #endif
 

-    needEncryptStart = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
-
+    needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));

     AbsoluteTime_to_scalar(&compTime) = 0;
     AbsoluteTime_to_scalar(&compTime) = 0;

-    AbsoluteTime_to_scalar(&decoTime) = 0;
+    compBytes = 0;

 
     clock_get_uptime(&allTime);
 
     clock_get_uptime(&allTime);

-
+    IOService::getPMRootDomain()->pmStatsRecordEvent( 
+                        kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);

     do 
     {
         compressedSize   = 0;
         uncompressedSize = 0;
     do 
     {
         compressedSize   = 0;
         uncompressedSize = 0;

-        iterDone         = false;
-        pageType         = 0;          // wired pages first
+        svPageCount      = 0;
+        zvPageCount      = 0;

 
 

-        IOPolledFileSeek(vars->fileVars, sizeof(IOHibernateImageHeader));
+        IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);

     
         HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n", 
                 ml_get_interrupts_enabled());
     
         HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n", 
                 ml_get_interrupts_enabled());

-        err = IOHibernatePollerOpen(vars->fileVars, kIOPolledBeforeSleepState, vars->ioBuffer);
+        err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState, true);

         HIBLOG("IOHibernatePollerOpen(%x)\n", err);
         pollerOpen = (kIOReturnSuccess == err);
         if (!pollerOpen)
         HIBLOG("IOHibernatePollerOpen(%x)\n", err);
         pollerOpen = (kIOReturnSuccess == err);
         if (!pollerOpen)


@@ -1563,77 +1527,95 @@ hibernate_write_image(void)
     
         // copy file block extent list if larger than header
     
     
         // copy file block extent list if larger than header
     

-        count = vars->fileExtents->getLength();
+        count = vars->fileVars->fileExtents->getLength();

         if (count > sizeof(header->fileExtentMap))
         {
             count -= sizeof(header->fileExtentMap);
         if (count > sizeof(header->fileExtentMap))
         {
             count -= sizeof(header->fileExtentMap);

-            err = IOPolledFileWrite(vars->fileVars,
+            err = IOHibernatePolledFileWrite(vars->fileVars,

                                     ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
             if (kIOReturnSuccess != err)
                 break;
         }
 
                                     ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
             if (kIOReturnSuccess != err)
                 break;
         }
 

+        hibernateBase = HIB_BASE; /* Defined in PAL headers */
+        hibernateEnd = (segHIBB + segSizeHIB);
+

         // copy out restore1 code
         // copy out restore1 code

-    
-        page = atop_32(sectHIBB);
-        count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
-        header->restore1CodePage = page;
+
+        for (count = 0;
+            (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+            count += segLen)
+        {
+           for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++)
+           {
+               gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
+           }
+        }
+
+        page = atop_32(kvtophys(hibernateBase));
+        count = atop_32(round_page(hibernateEnd) - hibernateBase);
+        header->restore1CodePhysPage = page;
+        header->restore1CodeVirt = hibernateBase;

         header->restore1PageCount = count;
         header->restore1PageCount = count;

-        header->restore1CodeOffset = ((uint32_t) &hibernate_machine_entrypoint)      - sectHIBB;
-        header->restore1StackOffset = ((uint32_t) &gIOHibernateRestoreStackEnd[0]) - 64 - sectHIBB;
+        header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint)      - hibernateBase;
+        header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;

 
 

-        // sum __HIB sect, with zeros for the stack
-        src = (uint8_t *) trunc_page(sectHIBB);
+        // sum __HIB seg, with zeros for the stack
+        src = (uint8_t *) trunc_page(hibernateBase);

         for (page = 0; page < count; page++)
         {
             if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
         for (page = 0; page < count; page++)
         {
             if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))

-                restore1Sum += hibernate_sum(src, page_size);
+                restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);

             else
             else

-                restore1Sum += 0x10000001;
+                restore1Sum += 0x00000000;

             src += page_size;
         }
         sum1 = restore1Sum;
     
             src += page_size;
         }
         sum1 = restore1Sum;
     

-        // write the __HIB sect, with zeros for the stack
+        // write the __HIB seg, with zeros for the stack

 
 

-        src = (uint8_t *) trunc_page(sectHIBB);
-        count = ((uint32_t) &gIOHibernateRestoreStack[0]) - trunc_page(sectHIBB);
+        src = (uint8_t *) trunc_page(hibernateBase);
+        count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);

         if (count)
         {
         if (count)
         {

-            err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
+            err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);

             if (kIOReturnSuccess != err)
                 break;
         }
             if (kIOReturnSuccess != err)
                 break;
         }

-        err = IOPolledFileWrite(vars->fileVars, 
+        err = IOHibernatePolledFileWrite(vars->fileVars, 

                                         (uint8_t *) 0,
                                         &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
                                         cryptvars);
         if (kIOReturnSuccess != err)
             break;
         src = &gIOHibernateRestoreStackEnd[0];
                                         (uint8_t *) 0,
                                         &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
                                         cryptvars);
         if (kIOReturnSuccess != err)
             break;
         src = &gIOHibernateRestoreStackEnd[0];

-        count = round_page(sectHIBB + sectSizeHIB) - ((uint32_t) src);
+        count = round_page(hibernateEnd) - ((uintptr_t) src);

         if (count)
         {
         if (count)
         {

-            err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
+            err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);

             if (kIOReturnSuccess != err)
                 break;
         }
 
             if (kIOReturnSuccess != err)
                 break;
         }
 

-        // write the preview buffer
+       if (kIOHibernateModeEncrypt & gIOHibernateMode)
+       {
+           vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
+           vars->fileVars->encryptEnd   = UINT64_MAX;
+           HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
+       }

 
 

-        addr64_t phys64;
-        IOByteCount segLen;
+        // write the preview buffer

 
 

-        if (vars->previewData)
+        if (vars->previewBuffer)

         {
             ppnum = 0;
             count = 0;
             do
             {
         {
             ppnum = 0;
             count = 0;
             do
             {

-                phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen);
+                phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);

                 pageAndCount[0] = atop_64(phys64);
                 pageAndCount[1] = atop_32(segLen);
                 pageAndCount[0] = atop_64(phys64);
                 pageAndCount[1] = atop_32(segLen);

-                err = IOPolledFileWrite(vars->fileVars, 
+                err = IOHibernatePolledFileWrite(vars->fileVars, 

                                         (const uint8_t *) &pageAndCount, sizeof(pageAndCount), 
                                         cryptvars);
                 if (kIOReturnSuccess != err)
                                         (const uint8_t *) &pageAndCount, sizeof(pageAndCount), 
                                         cryptvars);
                 if (kIOReturnSuccess != err)


@@ -1645,24 +1627,30 @@ hibernate_write_image(void)
             if (kIOReturnSuccess != err)
                 break;
 
             if (kIOReturnSuccess != err)
                 break;
 

-            src = (uint8_t *) vars->previewData->getBytesNoCopy();
-            count = vars->previewData->getLength();
+            src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
+
+                       ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
+
+            count = vars->previewBuffer->getLength();

 
             header->previewPageListSize = ppnum;
             header->previewSize = count + ppnum;
 
             for (page = 0; page < count; page += page_size)
 
             header->previewPageListSize = ppnum;
             header->previewSize = count + ppnum;
 
             for (page = 0; page < count; page += page_size)

-                sum1 += hibernate_sum(src + page, page_size);
-
-            err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
+            {
+                phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
+                sum1 += hibernate_sum_page(src + page, atop_64(phys64));
+            }
+            err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);

             if (kIOReturnSuccess != err)
                 break;
         }
 
         // mark areas for no save
             if (kIOReturnSuccess != err)
                 break;
         }
 
         // mark areas for no save

-    
+        IOMemoryDescriptor * ioBuffer;
+        ioBuffer = IOPolledFileGetIOBuffer(vars->fileVars);

         for (count = 0;
         for (count = 0;

-            (phys64 = vars->ioBuffer->getPhysicalSegment64(count, &segLen));
+            (phys64 = ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));

             count += segLen)
         {
             hibernate_set_page_state(vars->page_list, vars->page_list_wired, 
             count += segLen)
         {
             hibernate_set_page_state(vars->page_list, vars->page_list_wired, 


@@ -1672,7 +1660,7 @@ hibernate_write_image(void)
         }
     
         for (count = 0;
         }
     
         for (count = 0;

-            (phys64 = vars->srcBuffer->getPhysicalSegment64(count, &segLen));
+            (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));

             count += segLen)
         {
             hibernate_set_page_state(vars->page_list, vars->page_list_wired, 
             count += segLen)
         {
             hibernate_set_page_state(vars->page_list, vars->page_list_wired, 


@@ -1685,7 +1673,7 @@ hibernate_write_image(void)
     
         bitmap_size = vars->page_list_wired->list_size;
         src = (uint8_t *) vars->page_list_wired;
     
         bitmap_size = vars->page_list_wired->list_size;
         src = (uint8_t *) vars->page_list_wired;

-        err = IOPolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
+        err = IOHibernatePolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);

         if (kIOReturnSuccess != err)
             break;
 
         if (kIOReturnSuccess != err)
             break;
 


@@ -1694,15 +1682,16 @@ hibernate_write_image(void)
 
        hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
     
 
        hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
     

-        page = atop_32(sectHIBB);
-        count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
+
+        page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
+        count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;

         hibernate_set_page_state(vars->page_list, vars->page_list_wired,
                                         page, count,
                                         kIOHibernatePageStateFree);
         pageCount -= count;
 
         if (vars->previewBuffer) for (count = 0;
         hibernate_set_page_state(vars->page_list, vars->page_list_wired,
                                         page, count,
                                         kIOHibernatePageStateFree);
         pageCount -= count;
 
         if (vars->previewBuffer) for (count = 0;

-                                        (phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen));
+                                        (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));

                                         count += segLen)
         {
             hibernate_set_page_state(vars->page_list, vars->page_list_wired, 
                                         count += segLen)
         {
             hibernate_set_page_state(vars->page_list, vars->page_list_wired, 


@@ -1711,132 +1700,223 @@ hibernate_write_image(void)
             pageCount -= atop_32(segLen);
         }
 
             pageCount -= atop_32(segLen);
         }
 

-        src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
-    
-        ppnum     = 0;
-        pagesDone = 0;
-    
-        HIBLOG("writing %d pages\n", pageCount);
-
-        do
+        for (count = 0;
+            (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+            count += segLen)

         {
         {

-            count = hibernate_page_list_iterate(pageType ? vars->page_list : vars->page_list_wired,
-                                                    &ppnum);
-//          kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
-    
-            iterDone = !count;
+            hibernate_set_page_state(vars->page_list, vars->page_list_wired, 
+                                        atop_64(phys64), atop_32(segLen),
+                                        kIOHibernatePageStateFree);
+            pageCount -= atop_32(segLen);
+        }

 
 

-            pageAndCount[0] = ppnum;
-            pageAndCount[1] = count;
-            err = IOPolledFileWrite(vars->fileVars, 
-                                    (const uint8_t *) &pageAndCount, sizeof(pageAndCount), 
-                                    cryptvars);
-            if (kIOReturnSuccess != err)
-                break;
+        src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
+       compressed = src + page_size;
+        scratch    = compressed + page_size;

 
 

-            for (page = 0; page < count; page++)
+        pagesDone  = 0;
+        lastBlob   = 0;
+    
+        HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n", 
+               bitmap_size, header->previewSize,
+               pageCount, vars->fileVars->position);
+
+        enum
+        // pageType
+        { 
+            kWired          = 0x02,
+            kEncrypt        = 0x01,
+            kWiredEncrypt   = kWired | kEncrypt,
+            kWiredClear     = kWired,
+            kUnwiredEncrypt = kEncrypt
+        };
+
+        for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)
+        {
+           if (kUnwiredEncrypt == pageType)
+          {
+               // start unwired image
+               if (kIOHibernateModeEncrypt & gIOHibernateMode)
+               {
+                   vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
+                   vars->fileVars->encryptEnd   = UINT64_MAX;
+                   HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
+               }
+               bcopy(&cryptvars->aes_iv[0], 
+                       &gIOHibernateCryptWakeContext.aes_iv[0], 
+                       sizeof(cryptvars->aes_iv));
+               cryptvars = &gIOHibernateCryptWakeContext;
+            }
+            for (iterDone = false, ppnum = 0; !iterDone; )

             {
             {

-                err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(ppnum), page_size);
-                if (err)
+                count = hibernate_page_list_iterate((kWired & pageType) 
+                                                            ? vars->page_list_wired : vars->page_list,
+                                                        &ppnum);
+//              kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
+                iterDone = !count;
+    
+                if (count && (kWired & pageType) && needEncrypt)

                 {
                 {

-                    HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%d] %x\n", __LINE__, ppnum, err);
-                    break;
+                    uint32_t checkIndex;
+                    for (checkIndex = 0;
+                            (checkIndex < count) 
+                                && (((kEncrypt & pageType) == 0) == pmap_is_noencrypt(ppnum + checkIndex)); 
+                            checkIndex++)
+                    {}
+                    if (!checkIndex)
+                    {
+                        ppnum++;
+                        continue;
+                    }
+                    count = checkIndex;

                 }
                 }

-    
-                sum = hibernate_sum(src, page_size);
-   
-                clock_get_uptime(&startTime);
-
-                pageCompressedSize = WKdm_compress ((WK_word*) src, (WK_word*) (src + page_size), PAGE_SIZE_IN_WORDS);
-    
-                clock_get_uptime(&endTime);
-                ADD_ABSOLUTETIME(&compTime, &endTime);
-                SUB_ABSOLUTETIME(&compTime, &startTime);
-    
-                if (kIOHibernateModeEncrypt & gIOHibernateMode)
-                    pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);

 
 

-                if (pageCompressedSize > page_size)
+                switch (pageType)

                 {
                 {

-//                  HIBLOG("------------lose: %d\n", pageCompressedSize);
-                    pageCompressedSize = page_size;
+                    case kWiredEncrypt:   wiredPagesEncrypted += count; break;
+                    case kWiredClear:     wiredPagesClear     += count; break;
+                    case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;

                 }
                 }

-
-                if (pageCompressedSize != page_size)
-                    data = (src + page_size);
-                else
-                    data = src;
-
-                tag = pageCompressedSize | kIOHibernateTagSignature;
-
-                if (pageType)
-                    sum2 += sum;
+    
+                if (iterDone && (kWiredEncrypt == pageType))   {/* not yet end of wired list */}

                 else
                 else

-                    sum1 += sum;
-
-                if (needEncryptStart && (ppnum >= atop_32(sectDATAB)))

                 {
                 {

-                    // start encrypting partway into the data about to be written
-                    vars->fileVars->encryptStart = (vars->fileVars->position + AES_BLOCK_SIZE - 1) 
-                                                    & ~(AES_BLOCK_SIZE - 1);
-                    needEncryptStart = false;
+                    pageAndCount[0] = ppnum;
+                    pageAndCount[1] = count;
+                    err = IOHibernatePolledFileWrite(vars->fileVars, 
+                                            (const uint8_t *) &pageAndCount, sizeof(pageAndCount), 
+                                            cryptvars);
+                    if (kIOReturnSuccess != err)
+                        break;

                 }
                 }

-
-                err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
-                if (kIOReturnSuccess != err)
-                    break;
-
-                err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
-                if (kIOReturnSuccess != err)
-                    break;
-
-                compressedSize += pageCompressedSize;
-                if (pageCompressedSize)
-                    uncompressedSize += page_size;
-                ppnum++;
-                pagesDone++;

     
     

-                if (0 == (8191 & pagesDone))
+                for (page = ppnum; page < (ppnum + count); page++)

                 {
                 {

+                    err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
+                    if (err)
+                    {
+                        HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
+                        break;
+                    }
+        
+                    sum = hibernate_sum_page(src, page);
+                    if (kWired & pageType)
+                        sum1 += sum;
+                    else
+                        sum2 += sum;
+       
+                    clock_get_uptime(&startTime);
+                    wkresult = WKdm_compress_new((const WK_word*) src,
+                                                (WK_word*) compressed, 
+                                                (WK_word*) scratch,
+                                                page_size - 4);
+

                     clock_get_uptime(&endTime);
                     clock_get_uptime(&endTime);

-                    SUB_ABSOLUTETIME(&endTime, &allTime);
-                    absolutetime_to_nanoseconds(endTime, &nsec);
-                    progressStamp = nsec / 750000000ULL;
-                    if (progressStamp != lastProgressStamp)
+                    ADD_ABSOLUTETIME(&compTime, &endTime);
+                    SUB_ABSOLUTETIME(&compTime, &startTime);
+
+                    compBytes += page_size;
+                    pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
+
+                   if (pageCompressedSize == 0) 
+                   {
+                       pageCompressedSize = 4;
+                        data = src;
+
+                       if (*(uint32_t *)src)
+                               svPageCount++;
+                       else
+                               zvPageCount++;
+                   }
+                   else 
+                   {
+                       if (pageCompressedSize != page_size)
+                           data = compressed;
+                       else
+                           data = src;
+                   }
+    
+                    tag = pageCompressedSize | kIOHibernateTagSignature;
+                    err = IOHibernatePolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
+                    if (kIOReturnSuccess != err)
+                        break;
+    
+                    err = IOHibernatePolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
+                    if (kIOReturnSuccess != err)
+                        break;
+    
+                    compressedSize += pageCompressedSize;
+                    uncompressedSize += page_size;
+                    pagesDone++;
+    
+                    if (vars->consoleMapping && (0 == (1023 & pagesDone)))
+                    {
+                        blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
+                        if (blob != lastBlob)
+                        {
+                            ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
+                            lastBlob = blob;
+                        }
+                    }
+                    if (0 == (8191 & pagesDone))

                     {
                     {

-                        lastProgressStamp = progressStamp;
-                        HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
+                        clock_get_uptime(&endTime);
+                        SUB_ABSOLUTETIME(&endTime, &allTime);
+                        absolutetime_to_nanoseconds(endTime, &nsec);
+                        progressStamp = nsec / 750000000ULL;
+                        if (progressStamp != lastProgressStamp)
+                        {
+                            lastProgressStamp = progressStamp;
+                            HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
+                        }

                     }
                 }
                     }
                 }

+                if (kIOReturnSuccess != err)
+                    break;
+                ppnum = page;

             }
             }

+

             if (kIOReturnSuccess != err)
                 break;
             if (kIOReturnSuccess != err)
                 break;

-            if (iterDone && !pageType)
+
+            if ((kEncrypt & pageType) && vars->fileVars->encryptStart)

             {
             {

-                err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
+                vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
+                HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
+            }
+
+            if (kWiredEncrypt != pageType)
+            {
+                // end of image1/2 - fill to next block
+                err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);

                 if (kIOReturnSuccess != err)
                     break;
                 if (kIOReturnSuccess != err)
                     break;

+            }
+            if (kWiredClear == pageType)
+            {
+               // enlarge wired image for test
+//              err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);

 
 

-                iterDone = false;
-                pageType = 1;
-                ppnum = 0;
+                // end wired image
+                header->encryptStart = vars->fileVars->encryptStart;
+                header->encryptEnd   = vars->fileVars->encryptEnd;

                 image1Size = vars->fileVars->position;
                 image1Size = vars->fileVars->position;

-                if (cryptvars)
-                {
-                    bcopy(&cryptvars->aes_iv[0], 
-                            &gIOHibernateCryptWakeContext.aes_iv[0], 
-                            sizeof(cryptvars->aes_iv));
-                    cryptvars = &gIOHibernateCryptWakeContext;
-                }
-                HIBLOG("image1Size %qd\n", image1Size);
+                HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
+                        image1Size, header->encryptStart, header->encryptEnd);

             }
         }
             }
         }

-        while (!iterDone);
-        if (kIOReturnSuccess != err)
-            break;
-        err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);

         if (kIOReturnSuccess != err)
         if (kIOReturnSuccess != err)

+        {
+            if (kIOReturnOverrun == err)
+            {
+                // update actual compression ratio on not enough space (for retry)
+                gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
+            }
+
+            // update partial amount written (for IOPolledFileClose cleanup/unmap)
+            header->imageSize = vars->fileVars->position;

             break;
             break;

+        }

 
         // Header:
     
 
         // Header:
     


@@ -1844,13 +1924,16 @@ hibernate_write_image(void)
         header->image1Size   = image1Size;
         header->bitmapSize   = bitmap_size;
         header->pageCount    = pageCount;
         header->image1Size   = image1Size;
         header->bitmapSize   = bitmap_size;
         header->pageCount    = pageCount;

-        header->encryptStart = vars->fileVars->encryptStart;

     
         header->restore1Sum  = restore1Sum;
         header->image1Sum    = sum1;
         header->image2Sum    = sum2;
     
         header->restore1Sum  = restore1Sum;
         header->image1Sum    = sum1;
         header->image2Sum    = sum2;

+        header->sleepTime    = gIOLastSleepTime.tv_sec;
+
+       header->compression     = (compressedSize << 8) / uncompressedSize;
+       gIOHibernateCompression = header->compression;

     
     

-        count = vars->fileExtents->getLength();
+        count = vars->fileVars->fileExtents->getLength();

         if (count > sizeof(header->fileExtentMap))
         {
             header->fileExtentMapSize = count;
         if (count > sizeof(header->fileExtentMap))
         {
             header->fileExtentMapSize = count;


@@ -1859,113 +1942,89 @@ hibernate_write_image(void)
         else
             header->fileExtentMapSize = sizeof(header->fileExtentMap);
         bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
         else
             header->fileExtentMapSize = sizeof(header->fileExtentMap);
         bcopy(&fileExtents[0], &header->fileExtentMap[0], count);

+
+        header->deviceBase      = vars->fileVars->block0;
+        header->deviceBlockSize = vars->fileVars->blockSize;

     
         IOPolledFileSeek(vars->fileVars, 0);
     
         IOPolledFileSeek(vars->fileVars, 0);

-        err = IOPolledFileWrite(vars->fileVars,
+        err = IOHibernatePolledFileWrite(vars->fileVars,

                                     (uint8_t *) header, sizeof(IOHibernateImageHeader), 
                                     cryptvars);
         if (kIOReturnSuccess != err)
             break;
                                     (uint8_t *) header, sizeof(IOHibernateImageHeader), 
                                     cryptvars);
         if (kIOReturnSuccess != err)
             break;

-        err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
-        if (kIOReturnSuccess != err)
-            break;
-        err = IOHibernatePollerIODone(vars->fileVars);
-        if (kIOReturnSuccess != err)
-            break;
+        err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);

     }
     while (false);
     
     clock_get_uptime(&endTime);
     }
     while (false);
     
     clock_get_uptime(&endTime);

+
+    IOService::getPMRootDomain()->pmStatsRecordEvent( 
+                        kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
+

     SUB_ABSOLUTETIME(&endTime, &allTime);
     absolutetime_to_nanoseconds(endTime, &nsec);
     SUB_ABSOLUTETIME(&endTime, &allTime);
     absolutetime_to_nanoseconds(endTime, &nsec);

-    HIBLOG("all time: %qd ms, ", 
-               nsec / 1000000ULL);
+    HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);

 
     absolutetime_to_nanoseconds(compTime, &nsec);
 
     absolutetime_to_nanoseconds(compTime, &nsec);

-    HIBLOG("comp time: %qd ms, ", 
-               nsec / 1000000ULL);
-
-    absolutetime_to_nanoseconds(decoTime, &nsec);
-    HIBLOG("deco time: %qd ms, ", 
-               nsec / 1000000ULL);
-
-    HIBLOG("\nimage %qd, uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n", 
-               header->imageSize,
+    HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ", 
+               compBytes, 
+               nsec / 1000000ULL,
+               nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+    absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
+    HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ", 
+               vars->fileVars->cryptBytes, 
+               nsec / 1000000ULL, 
+               nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+    HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n", 
+               header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,

                uncompressedSize, atop_32(uncompressedSize), compressedSize,
                uncompressedSize, atop_32(uncompressedSize), compressedSize,

-               (int) ((compressedSize * 100ULL) / uncompressedSize),
+               uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,

                sum1, sum2);
 
                sum1, sum2);
 

+    HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n", 
+          svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
+

     if (pollerOpen)
     if (pollerOpen)

-        IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
+        IOPolledFilePollersClose(vars->fileVars, kIOPolledBeforeSleepState);
+
+    if (vars->consoleMapping)
+        ProgressUpdate(gIOHibernateGraphicsInfo, 
+                        vars->consoleMapping, 0, kIOHibernateProgressCount);

 
     HIBLOG("hibernate_write_image done(%x)\n", err);
 
     // should we come back via regular wake, set the state in memory.
     gIOHibernateState = kIOHibernateStateInactive;
 
 
     HIBLOG("hibernate_write_image done(%x)\n", err);
 
     // should we come back via regular wake, set the state in memory.
     gIOHibernateState = kIOHibernateStateInactive;
 

-    if ((kIOReturnSuccess == err) && !(kIOHibernateModeSleep & gIOHibernateMode))
-        return (true  /* power down */ );
-    else
-        return (false /* sleep */ );
-}
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-DECLARE_IOHIBERNATEPROGRESSALPHA
-
-static void
-ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
-{
-    uint32_t  rowBytes, pixelShift;
-    uint32_t  x, y;
-    int32_t   blob, lastBlob;
-    uint32_t  alpha, in, color, result;
-    uint8_t * out;
-    uint32_t  saveindex[kIOHibernateProgressCount] = { 0 };
-
-    pixelShift = display->depth >> 4;
-    if (pixelShift < 1)
-        return;
-
-    rowBytes = display->rowBytes;
-
-    screen += ((display->width 
-            - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
-                + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
-
-    lastBlob  = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
-
-    screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
+    KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END,
+                         wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted, 0, 0);

 
 

-    for (y = 0; y < kIOHibernateProgressHeight; y++)
+    if (kIOReturnSuccess == err)

     {
     {

-        out = screen + y * rowBytes;
-        for (blob = firstBlob; blob <= lastBlob; blob++)
-        {
-            color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
-            for (x = 0; x < kIOHibernateProgressWidth; x++)
-            {
-                alpha  = gIOHibernateProgressAlpha[y][x];
-                result = color;
-                if (alpha)
-                {
-                    if (0xff != alpha)
-                    {
-                        in = display->progressSaveUnder[blob][saveindex[blob]++];
-                        result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
-                    }
-                    if (1 == pixelShift)
-                    {
-                        result >>= 3;
-                        *((uint16_t *)out) = (result << 10) | (result << 5) | result;  // 16
-                    }
-                    else
-                        *((uint32_t *)out) = (result << 16) | (result << 8) | result;  // 32
-                }
-                out += (1 << pixelShift);
-            }
-            out += (kIOHibernateProgressSpacing << pixelShift);
-        }
+       if (kIOHibernateModeSleep & gIOHibernateMode)
+       {
+           return (kIOHibernatePostWriteSleep);
+       }
+       else if(kIOHibernateModeRestart & gIOHibernateMode)
+       {
+           return (kIOHibernatePostWriteRestart);
+       }
+       else
+       {
+           /* by default, power down */
+           return (kIOHibernatePostWriteHalt);
+       }
+    }
+    else if (kIOReturnAborted == err)
+    {
+       return (kIOHibernatePostWriteWake);
+    }
+    else
+    {
+       /* on error, sleep */
+       return (kIOHibernatePostWriteSleep);

     }
 }
 
     }
 }
 


@@ -1977,26 +2036,25 @@ hibernate_machine_init(void)
     IOReturn     err;
     uint32_t     sum;
     uint32_t     pagesDone;
     IOReturn     err;
     uint32_t     sum;
     uint32_t     pagesDone;

+    uint32_t     pagesRead = 0;
+    AbsoluteTime startTime, compTime;

     AbsoluteTime allTime, endTime;
     AbsoluteTime allTime, endTime;

-    uint64_t     nsec;
+    AbsoluteTime startIOTime, endIOTime;
+    uint64_t     nsec, nsecIO;
+    uint64_t     compBytes;

     uint32_t     lastProgressStamp = 0;
     uint32_t     progressStamp;
     uint32_t     lastProgressStamp = 0;
     uint32_t     progressStamp;

-    uint64_t    progressZeroPosition = 0;
-    uint32_t    blob, lastBlob = (uint32_t) -1L;
-    hibernate_cryptvars_t * cryptvars = 0;
+    IOPolledFileCryptVars * cryptvars = 0;

 
     IOHibernateVars * vars  = &gIOHibernateVars;
 
     IOHibernateVars * vars  = &gIOHibernateVars;

+    bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));

 
 

-    if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
+    if (!vars->fileVars || !vars->fileVars->pollers)

        return;
 
     sum = gIOHibernateCurrentHeader->actualImage1Sum;
     pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
 
        return;
 
     sum = gIOHibernateCurrentHeader->actualImage1Sum;
     pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
 

-    HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, image1Size %qx, conflictCount %d, nextFree %x\n",
-           gIOHibernateState, pagesDone, sum, gIOHibernateCurrentHeader->image1Size,
-           gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
-

     if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
     {
        HIBLOG("regular wake\n");
     if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
     {
        HIBLOG("regular wake\n");


@@ -2005,81 +2063,162 @@ hibernate_machine_init(void)
 
     HIBPRINT("diag %x %x %x %x\n",
            gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1], 
 
     HIBPRINT("diag %x %x %x %x\n",
            gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1], 

-           gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]); 
-
-    HIBPRINT("video %x %d %d %d\n",
-           gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth, 
-           gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height); 
+           gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
+
+#define t40ms(x)       (tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)
+#define tStat(x, y)    gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
+    tStat(booterStart, booterStart);
+    gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart,
+    tStat(booterDuration0, booterTime0);
+    tStat(booterDuration1, booterTime1);
+    tStat(booterDuration2, booterTime2);
+    tStat(booterDuration, booterTime);
+    tStat(booterConnectDisplayDuration, connectDisplayTime);
+    tStat(booterSplashDuration, splashTime);
+    tStat(trampolineDuration, trampolineTime);
+
+    gIOHibernateStats->image1Size  = gIOHibernateCurrentHeader->image1Size;
+    gIOHibernateStats->imageSize   = gIOHibernateCurrentHeader->imageSize;
+    gIOHibernateStats->image1Pages = pagesDone;
+
+    HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n", 
+          gIOHibernateStats->booterStart,
+          gIOHibernateStats->smcStart,
+          gIOHibernateStats->booterDuration0,
+          gIOHibernateStats->booterDuration1,
+          gIOHibernateStats->booterDuration2,
+          gIOHibernateStats->booterDuration,
+          gIOHibernateStats->booterConnectDisplayDuration,
+          gIOHibernateStats->booterSplashDuration,
+          gIOHibernateStats->trampolineDuration);
+
+    HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
+           gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
+           gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);

 
 

-    if ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)
+    if ((0 != (kIOHibernateModeSleep & gIOHibernateMode)) 
+     && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)))
+    {

         hibernate_page_list_discard(vars->page_list);
         hibernate_page_list_discard(vars->page_list);

+    }

 
 

-    boot_args *args = (boot_args *) PE_state.bootArgs;
+    cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;

 
 

-    if (vars->videoMapping 
-       && gIOHibernateGraphicsInfo->physicalAddress
-       && (args->Video.v_baseAddr == gIOHibernateGraphicsInfo->physicalAddress))
-    {
-        vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height 
-                                        * gIOHibernateGraphicsInfo->rowBytes);
-        IOMapPages(kernel_map, 
-                    vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
-                    vars->videoMapSize, kIOMapInhibitCache );
-    }
+    if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount)
+       panic("handoff overflow");

 
 

-    uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
+    IOHibernateHandoff * handoff;
+    bool                 done           = false;
+    bool                 foundCryptData = false;

 
 

-    if (gIOHibernateWakeMapSize)
+    for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
+        !done;
+        handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])

     {
     {

-       err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(gIOHibernateWakeMap), 
-                                                   gIOHibernateWakeMapSize);
-       if (kIOReturnSuccess == err)
-           hibernate_newruntime_map(src, gIOHibernateWakeMapSize, 
-                                    gIOHibernateCurrentHeader->systemTableOffset);
-       gIOHibernateWakeMap = 0;
-       gIOHibernateWakeMapSize = 0;
-    }
+//     HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
+       uint8_t * data = &handoff->data[0];
+       switch (handoff->type)
+       {
+           case kIOHibernateHandoffTypeEnd:
+               done = true;
+               break;

 
 

-    uint32_t decoOffset;
+           case kIOHibernateHandoffTypeGraphicsInfo:
+               bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
+               break;

 
 

-    clock_get_uptime(&allTime);
+           case kIOHibernateHandoffTypeCryptVars:
+               if (cryptvars)
+               {
+                   hibernate_cryptwakevars_t *
+                   wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
+                   bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
+               }
+               foundCryptData = true;
+               bzero(data, handoff->bytecount);
+               break;

 
 

-    HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
-    err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0);
-    HIBLOG("IOHibernatePollerOpen(%x)\n", err);
+           case kIOHibernateHandoffTypeMemoryMap:

 
 

-    if (gIOHibernateCurrentHeader->previewSize)
-        progressZeroPosition = gIOHibernateCurrentHeader->previewSize 
-                             + gIOHibernateCurrentHeader->fileExtentMapSize 
-                             - sizeof(gIOHibernateCurrentHeader->fileExtentMap) 
-                             + ptoa_64(gIOHibernateCurrentHeader->restore1PageCount);
+               clock_get_uptime(&allTime);

 
 

-    IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
+               hibernate_newruntime_map(data, handoff->bytecount, 
+                                        gIOHibernateCurrentHeader->systemTableOffset);

 
 

-    if (vars->videoMapping)
-    {
-        lastBlob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
-                        / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
-        ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, 0, lastBlob);
+               clock_get_uptime(&endTime);
+           
+               SUB_ABSOLUTETIME(&endTime, &allTime);
+               absolutetime_to_nanoseconds(endTime, &nsec);
+           
+               HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
+
+               break;
+
+           case kIOHibernateHandoffTypeDeviceTree:
+               {
+//                 DTEntry chosen = NULL;
+//                 HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen));
+               }
+               break;
+
+           default:
+               done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
+               break;
+       }    

     }
     }

+    if (cryptvars && !foundCryptData)
+       panic("hibernate handoff");

 
 

-    cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
-    if (kIOHibernateModeEncrypt & gIOHibernateMode)
+    HIBPRINT("video %x %d %d %d status %x\n",
+           gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth, 
+           gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus); 
+
+    if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)

     {
     {

-        cryptvars = &gIOHibernateCryptWakeContext;
-        bcopy(&gIOHibernateCryptWakeVars->aes_iv[0], 
-                &cryptvars->aes_iv[0], 
-                sizeof(cryptvars->aes_iv));
+        vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height 
+                                        * gIOHibernateGraphicsInfo->rowBytes);
+       if (vars->videoMapSize > vars->videoAllocSize) vars->videoMapSize = 0;
+       else
+       {
+           IOMapPages(kernel_map, 
+                       vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
+                       vars->videoMapSize, kIOMapInhibitCache );
+       }

     }
 
     }
 

+    if (vars->videoMapSize)
+        ProgressUpdate(gIOHibernateGraphicsInfo, 
+                        (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
+
+    uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
+    uint8_t * compressed = src + page_size;
+    uint8_t * scratch    = compressed + page_size;
+    uint32_t  decoOffset;
+
+    clock_get_uptime(&allTime);
+    AbsoluteTime_to_scalar(&compTime) = 0;
+    compBytes = 0;
+
+    HIBLOG("IOPolledFilePollersOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
+    err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledAfterSleepState, false);
+    clock_get_uptime(&startIOTime);
+    endTime = startIOTime;
+    SUB_ABSOLUTETIME(&endTime, &allTime);
+    absolutetime_to_nanoseconds(endTime, &nsec);
+    HIBLOG("IOPolledFilePollersOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
+
+    IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
+

     // kick off the read ahead
     // kick off the read ahead

-    vars->fileVars->io          = false;

     vars->fileVars->bufferHalf   = 0;
     vars->fileVars->bufferLimit  = 0;
     vars->fileVars->lastRead     = 0;
     vars->fileVars->bufferHalf   = 0;
     vars->fileVars->bufferLimit  = 0;
     vars->fileVars->lastRead     = 0;

+    vars->fileVars->readEnd      = gIOHibernateCurrentHeader->imageSize;

     vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
     vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;

+    vars->fileVars->cryptBytes   = 0;
+    AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;

 
 

-    IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
+    err = IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);

     vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
     // --
 
     vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
     // --
 


@@ -2088,14 +2227,16 @@ hibernate_machine_init(void)
     uint32_t * header = (uint32_t *) src;
     sum = 0;
 
     uint32_t * header = (uint32_t *) src;
     sum = 0;
 

-    do
+    while (kIOReturnSuccess == err)

     {
        unsigned int count;
        unsigned int page;
         uint32_t     tag;
        vm_offset_t  ppnum, compressedSize;
 
     {
        unsigned int count;
        unsigned int page;
         uint32_t     tag;
        vm_offset_t  ppnum, compressedSize;
 

-       IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
+       err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
+       if (kIOReturnSuccess != err)
+           break;

 
        ppnum = header[0];
        count = header[1];
 
        ppnum = header[0];
        count = header[1];


@@ -2107,45 +2248,55 @@ hibernate_machine_init(void)
 
        for (page = 0; page < count; page++)
        {
 
        for (page = 0; page < count; page++)
        {

-           IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
+           err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
+           if (kIOReturnSuccess != err)
+               break;

 
            compressedSize = kIOHibernateTagLength & tag;
 
            compressedSize = kIOHibernateTagLength & tag;

-           if (!compressedSize)
+           if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength))

            {
            {

-               ppnum++;
-               pagesDone++;
-               continue;
+               err = kIOReturnIPCError;
+               break;

            }
 
            }
 

-           IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
-   
-           if (compressedSize != page_size)
+           err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
+           if (kIOReturnSuccess != err) break;
+
+           if (compressedSize < page_size)

            {
                decoOffset = page_size;
            {
                decoOffset = page_size;

-               WKdm_decompress((WK_word*) src, (WK_word*) (src + decoOffset), PAGE_SIZE_IN_WORDS);
-           }
-           else
-               decoOffset = 0;
+               clock_get_uptime(&startTime);

 
 

-           sum += hibernate_sum((src + decoOffset), page_size);
+               if (compressedSize == 4) {
+                   int i;
+                   uint32_t *s, *d;
+                       
+                   s = (uint32_t *)src;
+                   d = (uint32_t *)(uintptr_t)compressed;
+
+                   for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++)
+                       *d++ = *s;
+               }
+               else 
+                   WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, compressedSize);
+               clock_get_uptime(&endTime);
+               ADD_ABSOLUTETIME(&compTime, &endTime);
+               SUB_ABSOLUTETIME(&compTime, &startTime);
+               compBytes += page_size;
+           }
+           else decoOffset = 0;

 
 

+           sum += hibernate_sum_page((src + decoOffset), ppnum);

            err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
            if (err)
            err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
            if (err)

-               HIBLOG("IOMemoryDescriptorReadToPhysical [%d] %x\n", ppnum, err);
+           {
+                   HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
+                   break;
+           }

 
            ppnum++;
            pagesDone++;
 
            ppnum++;
            pagesDone++;

-
-            if (vars->videoMapping && (0 == (255 & pagesDone)))
-            {
-                blob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
-                        / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
-                if (blob != lastBlob)
-                {
-                    ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, lastBlob, blob);
-                    lastBlob = blob;
-                }
-            }
+           pagesRead++;

 
            if (0 == (8191 & pagesDone))
            {
 
            if (0 == (8191 & pagesDone))
            {


@@ -2162,25 +2313,115 @@ hibernate_machine_init(void)
            }
        }
     }
            }
        }
     }

-    while (true);
+    if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages))
+       err = kIOReturnLockedRead;

 
 

-    gIOHibernateCurrentHeader->actualImage2Sum = sum;
+    if (kIOReturnSuccess != err)
+       panic("Hibernate restore error %x", err);

 
 

-    if (vars->fileVars->io)
-        (void) IOHibernatePollerIODone(vars->fileVars);
+    gIOHibernateCurrentHeader->actualImage2Sum = sum;
+    gIOHibernateCompression = gIOHibernateCurrentHeader->compression;

 
 

-    err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
+    clock_get_uptime(&endIOTime);

 
 

-    if (vars->videoMapping)
-        ProgressUpdate(gIOHibernateGraphicsInfo, 
-                        (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
+    err = IOPolledFilePollersClose(vars->fileVars, kIOPolledAfterSleepState);

 
     clock_get_uptime(&endTime);
 
     clock_get_uptime(&endTime);

+
+    IOService::getPMRootDomain()->pmStatsRecordEvent( 
+                        kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
+    IOService::getPMRootDomain()->pmStatsRecordEvent( 
+                        kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
+

     SUB_ABSOLUTETIME(&endTime, &allTime);
     absolutetime_to_nanoseconds(endTime, &nsec);
 
     SUB_ABSOLUTETIME(&endTime, &allTime);
     absolutetime_to_nanoseconds(endTime, &nsec);
 

-    HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %qd ms\n", 
-               pagesDone, sum, nsec / 1000000ULL);
+    SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
+    absolutetime_to_nanoseconds(endIOTime, &nsecIO);
+
+    gIOHibernateStats->kernelImageReadDuration = nsec / 1000000ULL;
+    gIOHibernateStats->imagePages              = pagesDone;
+
+    HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ", 
+               pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
+               nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
+
+    absolutetime_to_nanoseconds(compTime, &nsec);
+    HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ", 
+               compBytes, 
+               nsec / 1000000ULL,
+               nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+    absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
+    HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n", 
+               vars->fileVars->cryptBytes, 
+               nsec / 1000000ULL, 
+               nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+    KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_NONE, pagesRead, pagesDone, 0, 0, 0);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+void IOHibernateSetWakeCapabilities(uint32_t capability)
+{
+    if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
+    {
+       gIOHibernateStats->wakeCapability = capability;
+
+       if (kIOPMSystemCapabilityGraphics & capability)
+       {
+               vm_compressor_do_warmup();
+       }
+    }

 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

+
+void IOHibernateSystemRestart(void)
+{
+    static uint8_t    noteStore[32] __attribute__((aligned(32)));
+    IORegistryEntry * regEntry;
+    const OSSymbol *  sym;
+    OSData *          noteProp;
+    OSData *          data;
+    uintptr_t *       smcVars;
+    uint8_t *         smcBytes;
+    size_t            len;
+    addr64_t          element;
+
+    data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
+    if (!data) return;
+
+    smcVars = (typeof(smcVars)) data->getBytesNoCopy();
+    smcBytes = (typeof(smcBytes)) smcVars[1];
+    len = smcVars[0];
+    if (len > sizeof(noteStore)) len = sizeof(noteStore);
+    noteProp = OSData::withCapacity(3 * sizeof(element));
+    if (!noteProp) return;
+    element = len;
+    noteProp->appendBytes(&element, sizeof(element));
+    element = crc32(0, smcBytes, len);
+    noteProp->appendBytes(&element, sizeof(element));
+
+    bcopy(smcBytes, noteStore, len);
+    element = (addr64_t) &noteStore[0];
+    element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
+    noteProp->appendBytes(&element, sizeof(element));
+
+    if (!gIOOptionsEntry)
+    {
+       regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
+       gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
+       if (regEntry && !gIOOptionsEntry)
+           regEntry->release();
+    }
+
+    sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
+    if (gIOOptionsEntry && sym) gIOOptionsEntry->setProperty(sym, noteProp);
+    if (noteProp)               noteProp->release();
+    if (sym)                    sym->release();
+}
+
+
+