/*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2016 Apple Inc. All rights reserved.
*
- * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
- *
- * This file contains Original Code and/or Modifications of Original Code
- * as defined in and that are subject to the Apple Public Source License
- * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. The rights granted to you under the
- * License may not be used to create, or enable the creation or
- * redistribution of, unlawful or unlicensed copies of an Apple operating
- * system, or to circumvent, violate, or enable the circumvention or
- * violation of, any terms of an Apple operating system software license
- * agreement.
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
- * Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this
- * file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
*
- * The Original Code and all software distributed under the License are
- * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
- * Please see the License for the specific language governing rights and
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
* limitations under the License.
*
- * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
-
/*
Sleep:
by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
The image header and block list are written. The header includes the second file extent so
only the header block is needed to read the file, regardless of filesystem.
- The kernel section "__HIB" is written uncompressed to the image. This section of code and data
+ The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
(only) is used to decompress the image during wake/boot.
Some additional pages are removed from the bitmaps - the buffers used for hibernation.
The bitmaps are written to the image.
More areas are removed from the bitmaps (after they have been written to the image) - the
- section "__HIB" pages and interrupt stack.
+ segment "__HIB" pages and interrupt stack.
Each wired page is compressed and written and then each non-wired page. Compression and
disk writes are in parallel.
The image header is written to the start of the file and the polling driver closed.
#include <IOKit/pwr_mgt/IOPowerConnection.h>
#include "IOPMPowerStateQueue.h"
#include <IOKit/IOBufferMemoryDescriptor.h>
-#include <crypto/aes.h>
+#include <IOKit/AppleKeyStoreInterface.h>
+#include <libkern/crypto/aes.h>
#include <sys/uio.h>
#include <sys/conf.h>
#include <sys/stat.h>
#include <sys/fcntl.h> // (FWRITE, ...)
#include <sys/sysctl.h>
+#include <sys/kdebug.h>
+#include <stdint.h>
#include <IOKit/IOHibernatePrivate.h>
#include <IOKit/IOPolledInterface.h>
#include <IOKit/IONVRAM.h>
#include "IOHibernateInternal.h"
-#include "WKdm.h"
+#include <vm/WKdm_new.h>
+#include <vm/vm_protos.h>
#include "IOKitKernelInternal.h"
+#include <pexpert/device_tree.h>
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+#include <machine/pal_routines.h>
+#include <machine/pal_hibernate.h>
+#include <i386/tsc.h>
+#include <i386/cpuid.h>
-OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
-
-OSMetaClassDefineReservedUnused(IOPolledInterface, 0);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
+extern "C" addr64_t kvtophys(vm_offset_t va);
+extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+#define DISABLE_TRIM 0
+#define TRIM_DELAY 25000
+
+extern unsigned int save_kdebug_enable;
extern uint32_t gIOHibernateState;
uint32_t gIOHibernateMode;
static char gIOHibernateBootSignature[256+1];
static char gIOHibernateFilename[MAXPATHLEN+1];
-static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
-uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
+static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
+uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
+static uint64_t gIOHibernateCompression = 0x80; // default compression 50%
+boolean_t gIOHibernateStandbyDisabled;
static IODTNVRAM * gIOOptionsEntry;
static IORegistryEntry * gIOChosenEntry;
-static IOPolledFileIOVars gFileVars;
-static IOHibernateVars gIOHibernateVars;
-static struct kern_direct_file_io_ref_t * gIOHibernateFileRef;
-static hibernate_cryptvars_t gIOHibernateCryptWakeContext;
+static const OSSymbol * gIOHibernateBootImageKey;
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+#if defined(__i386__) || defined(__x86_64__)
-enum { kXPRamAudioVolume = 8 };
-enum { kDefaultIOSize = 128 * 1024 };
-enum { kVideoMapSize = 32 * 1024 * 1024 };
+static const OSSymbol * gIOHibernateRTCVariablesKey;
+static const OSSymbol * gIOHibernateBoot0082Key;
+static const OSSymbol * gIOHibernateBootNextKey;
+static OSData * gIOHibernateBoot0082Data;
+static OSData * gIOHibernateBootNextData;
+static OSObject * gIOHibernateBootNextSave;
-#ifndef kIOMediaPreferredBlockSizeKey
-#define kIOMediaPreferredBlockSizeKey "Preferred Block Size"
-#endif
+static IOPolledFileIOVars * gDebugImageFileVars;
+static IOLock * gDebugImageLock;
-#ifndef kIOBootPathKey
-#define kIOBootPathKey "bootpath"
-#endif
-#ifndef kIOSelectedBootDeviceKey
-#define kIOSelectedBootDeviceKey "boot-device"
-#endif
+#endif /* defined(__i386__) || defined(__x86_64__) */
+
+static IOLock * gFSLock;
+static uint32_t gFSState;
+static thread_call_t gIOHibernateTrimCalloutEntry;
+static IOPolledFileIOVars gFileVars;
+static IOHibernateVars gIOHibernateVars;
+static IOPolledFileCryptVars gIOHibernateCryptWakeContext;
+static hibernate_graphics_t _hibernateGraphics;
+static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
+static hibernate_statistics_t _hibernateStats;
+static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats;
+
+enum
+{
+ kFSIdle = 0,
+ kFSOpening = 2,
+ kFSOpened = 3,
+ kFSTimedOut = 4,
+ kFSTrimDelay = 5
+};
+
+static IOReturn IOHibernateDone(IOHibernateVars * vars);
+static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature);
+static void IOSetBootImageNVRAM(OSData * data);
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-enum { kIOHibernateMinPollersNeeded = 2 };
+enum { kDefaultIOSize = 128 * 1024 };
+enum { kVideoMapSize = 80 * 1024 * 1024 };
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
addr64_t dstAddr64;
IOByteCount dstLen;
- dstAddr64 = md->getPhysicalSegment64(offset, &dstLen);
+ dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
if (!dstAddr64)
break;
addr64_t srcAddr64;
IOByteCount dstLen;
- srcAddr64 = md->getPhysicalSegment64(offset, &dstLen);
+ srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
if (!srcAddr64)
break;
}
static vm_offset_t
-hibernate_page_list_iterate(hibernate_page_list_t * list,
- void ** iterator, vm_offset_t * ppnum)
+hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
{
- uint32_t count, idx;
+ uint32_t page = *pPage;
+ uint32_t count;
+ hibernate_bitmap_t * bitmap;
- idx = (uint32_t) *iterator;
-
- if (!idx)
- idx = hibernate_page_list_count(list, TRUE, idx);
+ while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
+ {
+ count = hibernate_page_bitmap_count(bitmap, TRUE, page);
+ if (!count)
+ break;
+ page += count;
+ if (page <= bitmap->last_page)
+ break;
+ }
- *ppnum = idx;
- count = hibernate_page_list_count(list, FALSE, idx);
- idx += count;
- idx += hibernate_page_list_count(list, TRUE, idx);
- *iterator = (void *) idx;
+ *pPage = page;
+ if (bitmap)
+ count = hibernate_page_bitmap_count(bitmap, FALSE, page);
+ else
+ count = 0;
return (count);
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-static IOReturn
-IOHibernatePollerProbe(IOPolledFileIOVars * vars, IOService * target)
+IOReturn
+IOHibernateSystemSleep(void)
{
- IOReturn err = kIOReturnError;
- int32_t idx;
- IOPolledInterface * poller;
+ IOReturn err;
+ OSData * nvramData;
+ OSObject * obj;
+ OSString * str;
+ OSNumber * num;
+ bool dsSSD, vmflush, swapPinned;
+ IOHibernateVars * vars;
+ uint64_t setFileSize = 0;
- for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
- {
- poller = (IOPolledInterface *) vars->pollers->getObject(idx);
- err = poller->probe(target);
- if (err)
- {
- HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
- break;
- }
- }
+ gIOHibernateState = kIOHibernateStateInactive;
- return (err);
-}
+ gIOHibernateDebugFlags = 0;
+ if (kIOLogHibernate & gIOKitDebug)
+ gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
-static IOReturn
-IOHibernatePollerOpen(IOPolledFileIOVars * vars, uint32_t state, IOMemoryDescriptor * md)
-{
- IOReturn err = kIOReturnError;
- int32_t idx;
- IOPolledInterface * poller;
+ if (IOService::getPMRootDomain()->getHibernateSettings(
+ &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime))
+ {
+ if (kIOHibernateModeSleep & gIOHibernateMode)
+ // default to discard clean for safe sleep
+ gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
+ | kIOHibernateModeDiscardCleanActive);
+ }
- for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
+ if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
{
- poller = (IOPolledInterface *) vars->pollers->getObject(idx);
- err = poller->open(state, md);
- if (err)
- {
- HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
- break;
- }
+ if ((str = OSDynamicCast(OSString, obj)))
+ strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
+ sizeof(gIOHibernateFilename));
+ obj->release();
}
- return (err);
-}
+ if (!gIOHibernateMode || !gIOHibernateFilename[0])
+ return (kIOReturnUnsupported);
-static IOReturn
-IOHibernatePollerClose(IOPolledFileIOVars * vars, uint32_t state)
-{
- IOReturn err = kIOReturnError;
- int32_t idx;
- IOPolledInterface * poller;
+ HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
+
+ vars = IONew(IOHibernateVars, 1);
+ if (!vars) return (kIOReturnNoMemory);
+ bzero(vars, sizeof(*vars));
- for (idx = 0;
- (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
- idx++)
+ IOLockLock(gFSLock);
+ if (!gIOHibernateTrimCalloutEntry)
{
- err = poller->close(state);
- if (err)
- HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
+ gIOHibernateTrimCalloutEntry = thread_call_allocate(&IOHibernateSystemPostWakeTrim, &gFSLock);
}
+ IOHibernateSystemPostWakeTrim(NULL, NULL);
+ if (kFSIdle != gFSState)
+ {
+ HIBLOG("hibernate file busy\n");
+ IOLockUnlock(gFSLock);
+ IODelete(vars, IOHibernateVars, 1);
+ return (kIOReturnBusy);
+ }
+ gFSState = kFSOpening;
+ IOLockUnlock(gFSLock);
- return (err);
-}
-
-static void
-IOHibernatePollerIOComplete(void * target,
- void * parameter,
- IOReturn status,
- UInt64 actualByteCount)
-{
- IOPolledFileIOVars * vars = (IOPolledFileIOVars *) parameter;
-
- vars->ioStatus = status;
-}
+ swapPinned = false;
+ do
+ {
+ vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
+ 2 * page_size + WKdm_SCRATCH_BUF_SIZE_INTERNAL, page_size);
-static IOReturn
-IOHibernatePollerIO(IOPolledFileIOVars * vars,
- uint32_t operation, uint32_t bufferOffset,
- uint64_t deviceOffset, uint64_t length)
-{
+ vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
+ ptoa_64(gIOHibernateHandoffPageCount), page_size);
- IOReturn err = kIOReturnError;
- IOPolledInterface * poller;
- IOPolledCompletion completion;
+ if (!vars->srcBuffer || !vars->handoffBuffer)
+ {
+ err = kIOReturnNoMemory;
+ break;
+ }
- completion.target = 0;
- completion.action = &IOHibernatePollerIOComplete;
- completion.parameter = vars;
+ if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey)))
+ {
+ if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMinSize = num->unsigned64BitValue();
+ obj->release();
+ }
+ if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey)))
+ {
+ if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMaxSize = num->unsigned64BitValue();
+ obj->release();
+ }
- vars->ioStatus = -1;
+ boolean_t encryptedswap = true;
+ uint32_t pageCount;
+ AbsoluteTime startTime, endTime;
+ uint64_t nsec;
- poller = (IOPolledInterface *) vars->pollers->getObject(0);
- err = poller->startIO(operation, bufferOffset, deviceOffset + vars->block0, length, completion);
- if (err)
- HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
+ bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
+ gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
+ gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
- return (err);
-}
+ vmflush = ((kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
+ err = hibernate_alloc_page_lists(&vars->page_list,
+ &vars->page_list_wired,
+ &vars->page_list_pal);
+ if (KERN_SUCCESS != err) break;
-static IOReturn
-IOHibernatePollerIODone(IOPolledFileIOVars * vars)
-{
- IOReturn err = kIOReturnError;
- int32_t idx;
- IOPolledInterface * poller;
+ err = hibernate_pin_swap(TRUE);
+ if (KERN_SUCCESS != err) break;
+ swapPinned = true;
- while (-1 == vars->ioStatus)
- {
- for (idx = 0;
- (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
- idx++)
- {
- err = poller->checkForWork();
- if (err)
- HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
+ if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode))
+ {
+ hibernate_page_list_setall(vars->page_list,
+ vars->page_list_wired,
+ vars->page_list_pal,
+ true /* preflight */,
+ vmflush /* discard */,
+ &pageCount);
+ PE_Video consoleInfo;
+ bzero(&consoleInfo, sizeof(consoleInfo));
+ IOService::getPlatform()->getConsoleInfo(&consoleInfo);
+
+ // estimate: 6% increase in pages compressed
+ // screen preview 2 images compressed 0%
+ setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
+ + vars->page_list->list_size
+ + (consoleInfo.v_width * consoleInfo.v_height * 8);
+ enum { setFileRound = 1024*1024ULL };
+ setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
+
+ HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
+ pageCount, (100ULL * gIOHibernateCompression) >> 8,
+ setFileSize, vars->fileMinSize);
+
+ if (!(kIOHibernateModeFileResize & gIOHibernateMode)
+ && (setFileSize < vars->fileMinSize))
+ {
+ setFileSize = vars->fileMinSize;
+ }
+ }
+
+ // Invalidate the image file
+ if (gDebugImageLock) {
+ IOLockLock(gDebugImageLock);
+ if (gDebugImageFileVars != 0) {
+ IOSetBootImageNVRAM(0);
+ IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
}
+ IOLockUnlock(gDebugImageLock);
}
- if (kIOReturnSuccess != vars->ioStatus)
- HIBLOG("IOPolledInterface::ioStatus 0x%x\n", vars->ioStatus);
+ err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, 0,
+ gIOHibernateCurrentHeader, sizeof(gIOHibernateCurrentHeader),
+ &vars->fileVars, &nvramData,
+ &vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
- return (vars->ioStatus);
-}
+ if (KERN_SUCCESS != err)
+ {
+ IOLockLock(gFSLock);
+ if (kFSOpening != gFSState) err = kIOReturnTimeout;
+ IOLockUnlock(gFSLock);
+ }
-IOReturn
-IOPolledInterface::checkAllForWork(void)
-{
- IOReturn err = kIOReturnNotReady;
- int32_t idx;
- IOPolledInterface * poller;
+ if (KERN_SUCCESS != err)
+ {
+ HIBLOG("IOPolledFileOpen(%x)\n", err);
+ break;
+ }
- IOHibernateVars * vars = &gIOHibernateVars;
+ // write extents for debug data usage in EFI
+ IOWriteExtentsToFile(vars->fileVars, kIOHibernateHeaderOpenSignature);
- if (!vars->fileVars || !vars->fileVars->pollers)
- return (err);
+ err = IOPolledFilePollersSetup(vars->fileVars, kIOPolledPreflightState);
+ if (KERN_SUCCESS != err) break;
- for (idx = 0;
- (poller = (IOPolledInterface *) vars->fileVars->pollers->getObject(idx));
- idx++)
- {
- err = poller->checkForWork();
- if (err)
- HIBLOG("IOPolledInterface::checkAllForWork[%d] 0x%x\n", idx, err);
- }
+ clock_get_uptime(&startTime);
+ err = hibernate_setup(gIOHibernateCurrentHeader,
+ vmflush,
+ vars->page_list, vars->page_list_wired, vars->page_list_pal);
+ clock_get_uptime(&endTime);
+ SUB_ABSOLUTETIME(&endTime, &startTime);
+ absolutetime_to_nanoseconds(endTime, &nsec);
- return (err);
-}
+ boolean_t haveSwapPin, hibFileSSD;
+ haveSwapPin = vm_swap_files_pinned();
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+ hibFileSSD = (kIOPolledFileSSD & vars->fileVars->flags);
-struct _OpenFileContext
-{
- OSData * extents;
- uint64_t size;
-};
+ HIBLOG("hibernate_setup(%d) took %qd ms, swapPin(%d) ssd(%d)\n",
+ err, nsec / 1000000ULL,
+ haveSwapPin, hibFileSSD);
+ if (KERN_SUCCESS != err) break;
-static void
-file_extent_callback(void * ref, uint64_t start, uint64_t length)
-{
- _OpenFileContext * ctx = (_OpenFileContext *) ref;
- IOPolledFileExtent extent;
+ gIOHibernateStandbyDisabled = ((!haveSwapPin || !hibFileSSD));
- extent.start = start;
- extent.length = length;
+ dsSSD = ((0 != (kIOPolledFileSSD & vars->fileVars->flags))
+ && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
- ctx->extents->appendBytes(&extent, sizeof(extent));
- ctx->size += length;
-}
+ if (dsSSD) gIOHibernateCurrentHeader->options |= kIOHibernateOptionSSD | kIOHibernateOptionColor;
+ else gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
-IOReturn
-IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
- IOPolledFileIOVars ** fileVars, OSData ** fileExtents,
- OSData ** imagePath)
-{
- IOReturn err = kIOReturnError;
- IOPolledFileIOVars * vars;
- _OpenFileContext ctx;
- OSData * extentsData;
- OSNumber * num;
- IORegistryEntry * part = 0;
- OSDictionary * matching;
- OSIterator * iter;
- dev_t hibernate_image_dev;
- uint64_t maxiobytes;
-
- vars = &gFileVars;
- do
- {
- HIBLOG("sizeof(IOHibernateImageHeader) == %ld\n", sizeof(IOHibernateImageHeader));
- if (sizeof(IOHibernateImageHeader) != 512)
- continue;
-
- vars->io = false;
- vars->buffer = (uint8_t *) ioBuffer->getBytesNoCopy();
- vars->bufferHalf = 0;
- vars->bufferOffset = 0;
- vars->bufferSize = ioBuffer->getLength() >> 1;
-
- extentsData = OSData::withCapacity(32);
-
- ctx.extents = extentsData;
- ctx.size = 0;
- vars->fileRef = kern_open_file_for_direct_io(filename,
- &file_extent_callback, &ctx,
- &hibernate_image_dev,
- &vars->block0,
- &maxiobytes);
- if (!vars->fileRef)
- {
- err = kIOReturnNoSpace;
- break;
- }
- HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx\n", filename, ctx.size,
- vars->block0, maxiobytes);
- if (ctx.size < 1*1024*1024) // check against image size estimate!
- {
- err = kIOReturnNoSpace;
- break;
- }
- if (maxiobytes < vars->bufferSize)
- vars->bufferSize = maxiobytes;
-
- vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
-
- matching = IOService::serviceMatching("IOMedia");
- num = OSNumber::withNumber(major(hibernate_image_dev), 32);
- matching->setObject(kIOBSDMajorKey, num);
- num->release();
- num = OSNumber::withNumber(minor(hibernate_image_dev), 32);
- matching->setObject(kIOBSDMinorKey, num);
- num->release();
- iter = IOService::getMatchingServices(matching);
- matching->release();
- if (iter)
+#if defined(__i386__) || defined(__x86_64__)
+ if (!uuid_is_null(vars->volumeCryptKey) &&
+ (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey)))
{
- part = (IORegistryEntry *) iter->getNextObject();
- part->retain();
- iter->release();
- }
-
- int minor, major;
- IORegistryEntry * next;
- IORegistryEntry * child;
- OSData * data;
-
- num = (OSNumber *) part->getProperty(kIOBSDMajorKey);
- if (!num)
- break;
- major = num->unsigned32BitValue();
- num = (OSNumber *) part->getProperty(kIOBSDMinorKey);
- if (!num)
- break;
- minor = num->unsigned32BitValue();
-
- hibernate_image_dev = makedev(major, minor);
+ uintptr_t smcVars[2];
+ smcVars[0] = sizeof(vars->volumeCryptKey);
+ smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
- vars->pollers = OSArray::withCapacity(4);
- if (!vars->pollers)
- break;
+ IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
+ bzero(smcVars, sizeof(smcVars));
+ }
+#endif
- vars->blockSize = 512;
- next = part;
- do
- {
- IOPolledInterface * poller;
- OSObject * obj;
- obj = next->getProperty(kIOPolledInterfaceSupportKey);
- if (kOSBooleanFalse == obj)
- {
- vars->pollers->flushCollection();
- break;
- }
- else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
- vars->pollers->setObject(poller);
- if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey))))
- vars->blockSize = num->unsigned32BitValue();
- child = next;
- }
- while ((next = child->getParentEntry(gIOServicePlane))
- && child->isParent(next, gIOServicePlane, true));
+ if (encryptedswap || !uuid_is_null(vars->volumeCryptKey))
+ gIOHibernateMode ^= kIOHibernateModeEncrypt;
- HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
- major, minor, vars->blockSize, vars->pollers->getCount());
- if (vars->pollers->getCount() < kIOHibernateMinPollersNeeded)
- continue;
+ if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
+ {
+ vars->videoAllocSize = kVideoMapSize;
+ if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize, VM_KERN_MEMORY_IOKIT))
+ vars->videoMapping = 0;
+ }
- err = IOHibernatePollerProbe(vars, (IOService *) part);
- if (kIOReturnSuccess != err)
- break;
+ // generate crypt keys
+ for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
+ vars->wiredCryptKey[i] = random();
+ for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
+ vars->cryptKey[i] = random();
- err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer);
- if (kIOReturnSuccess != err)
- break;
+ // set nvram
- *fileVars = vars;
- *fileExtents = extentsData;
-
- // make imagePath
- char str1[256];
- char str2[24];
- int len = sizeof(str1);
+ IOSetBootImageNVRAM(nvramData);
+ nvramData->release();
- if ((extentsData->getLength() >= sizeof(IOPolledFileExtent))
- && part->getPath(str1, &len, gIODTPlane))
+#if defined(__i386__) || defined(__x86_64__)
{
- // (strip the plane name)
- char * tail = strchr(str1, ':');
- if (!tail)
- tail = str1 - 1;
- data = OSData::withBytes(tail + 1, strlen(tail + 1));
- sprintf(str2, ",%qx", vars->extentMap[0]);
- data->appendBytes(str2, strlen(str2));
- *imagePath = data;
+ struct AppleRTCHibernateVars
+ {
+ uint8_t signature[4];
+ uint32_t revision;
+ uint8_t booterSignature[20];
+ uint8_t wiredCryptKey[16];
+ };
+ AppleRTCHibernateVars rtcVars;
+ OSData * data;
+
+ rtcVars.signature[0] = 'A';
+ rtcVars.signature[1] = 'A';
+ rtcVars.signature[2] = 'P';
+ rtcVars.signature[3] = 'L';
+ rtcVars.revision = 1;
+ bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
+ if (gIOHibernateBootSignature[0])
+ {
+ char c;
+ uint8_t value = 0;
+ for (uint32_t i = 0;
+ (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
+ i++)
+ {
+ if (c >= 'a') c -= 'a' - 10;
+ else if (c >= 'A') c -= 'A' - 10;
+ else if (c >= '0') c -= '0';
+ else continue;
+ value = (value << 4) | c;
+ if (i & 1) rtcVars.booterSignature[i >> 1] = value;
+ }
+ }
+ data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
+ if (data)
+ {
+ if (gIOHibernateRTCVariablesKey)
+ IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
+ data->release();
+ }
+ if (gIOChosenEntry)
+ {
+ data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
+ if (data) gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
+ // set BootNext
+ if (!gIOHibernateBoot0082Data)
+ {
+ data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
+ if (data)
+ {
+ // AppleNVRAM_EFI_LOAD_OPTION
+ struct {
+ uint32_t Attributes;
+ uint16_t FilePathLength;
+ uint16_t Desc;
+ } loadOptionHeader;
+ loadOptionHeader.Attributes = 1;
+ loadOptionHeader.FilePathLength = data->getLength();
+ loadOptionHeader.Desc = 0;
+ gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
+ if (gIOHibernateBoot0082Data)
+ {
+ gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
+ gIOHibernateBoot0082Data->appendBytes(data);
+ }
+ }
+ }
+ if (!gIOHibernateBootNextData)
+ {
+ uint16_t bits = 0x0082;
+ gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
+ }
+ if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData)
+ {
+ gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
+ gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
+ gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
+ }
+ // BootNext
+ }
}
+#endif /* !i386 && !x86_64 */
}
while (false);
- if (kIOReturnSuccess != err)
+ if (swapPinned) hibernate_pin_swap(FALSE);
+
+ IOLockLock(gFSLock);
+ if ((kIOReturnSuccess == err) && (kFSOpening != gFSState))
+ {
+ HIBLOG("hibernate file close due timeout\n");
+ err = kIOReturnTimeout;
+ }
+ if (kIOReturnSuccess == err)
{
- HIBLOG("error 0x%x opening hibernation file\n", err);
- if (vars->fileRef)
- kern_close_file_for_direct_io(vars->fileRef);
+ gFSState = kFSOpened;
+ gIOHibernateVars = *vars;
+ gFileVars = *vars->fileVars;
+ gFileVars.allocated = false;
+ gIOHibernateVars.fileVars = &gFileVars;
+ gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
+ gIOHibernateState = kIOHibernateStateHibernating;
+ }
+ else
+ {
+ IOPolledFileIOVars * fileVars = vars->fileVars;
+ IOHibernateDone(vars);
+ IOPolledFileClose(&fileVars,
+#if DISABLE_TRIM
+ 0, NULL, 0, 0, 0);
+#else
+ 0, NULL, 0, sizeof(IOHibernateImageHeader), setFileSize);
+#endif
+ gFSState = kFSIdle;
}
+ IOLockUnlock(gFSLock);
- if (part)
- part->release();
+ if (vars->fileVars) IODelete(vars->fileVars, IOPolledFileIOVars, 1);
+ IODelete(vars, IOHibernateVars, 1);
return (err);
}
-IOReturn
-IOPolledFileClose( IOPolledFileIOVars * vars )
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+static void
+IOSetBootImageNVRAM(OSData * data)
{
- if (vars->pollers)
+ IORegistryEntry * regEntry;
+
+ if (!gIOOptionsEntry)
{
- IOHibernatePollerClose(vars, kIOPolledPostflightState);
- vars->pollers->release();
+ regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
+ gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
+ if (regEntry && !gIOOptionsEntry)
+ regEntry->release();
+ }
+ if (gIOOptionsEntry && gIOHibernateBootImageKey)
+ {
+ if (data) gIOOptionsEntry->setProperty(gIOHibernateBootImageKey, data);
+ else
+ {
+ gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
+ gIOOptionsEntry->sync();
+ }
}
-
- gIOHibernateFileRef = vars->fileRef;
-
- bzero(vars, sizeof(IOPolledFileIOVars));
-
- return (kIOReturnSuccess);
}
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+/*
+ * Writes header to disk with signature, block size and file extents data.
+ * If there are more than 2 extents, then they are written on second block.
+ */
static IOReturn
-IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
+IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature)
{
- IOPolledFileExtent * extentMap;
+ IOHibernateImageHeader hdr;
+ IOItemCount count;
+ IOReturn err = kIOReturnSuccess;
+ int rc;
+ IOPolledFileExtent * fileExtents;
- extentMap = vars->extentMap;
+ fileExtents = (typeof(fileExtents)) vars->fileExtents->getBytesNoCopy();
- vars->position = position;
-
- while (position >= extentMap->length)
+ memset(&hdr, 0, sizeof(IOHibernateImageHeader));
+ count = vars->fileExtents->getLength();
+ if (count > sizeof(hdr.fileExtentMap))
{
- position -= extentMap->length;
- extentMap++;
+ hdr.fileExtentMapSize = count;
+ count = sizeof(hdr.fileExtentMap);
}
-
- vars->currentExtent = extentMap;
- vars->extentRemaining = extentMap->length - position;
- vars->extentPosition = vars->position - position;
-
- if (vars->bufferSize <= vars->extentRemaining)
- vars->bufferLimit = vars->bufferSize;
else
- vars->bufferLimit = vars->extentRemaining;
-
- return (kIOReturnSuccess);
-}
+ hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap);
-static IOReturn
-IOPolledFileWrite(IOPolledFileIOVars * vars,
- const uint8_t * bytes, IOByteCount size,
- hibernate_cryptvars_t * cryptvars)
-{
- IOReturn err = kIOReturnSuccess;
- IOByteCount copy;
- bool flush = false;
+ bcopy(fileExtents, &hdr.fileExtentMap[0], count);
- do
+ // copy file block extent list if larger than header
+ if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap))
{
- if (!bytes && !size)
- {
- // seek to end of block & flush
- size = vars->position & (vars->blockSize - 1);
- if (size)
- size = vars->blockSize - size;
- flush = true;
- // use some garbage for the fill
- bytes = vars->buffer + vars->bufferOffset;
- }
+ count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap);
+ rc = kern_write_file(vars->fileRef, vars->blockSize,
+ (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)),
+ count, IO_SKIP_ENCRYPTION);
+ if (rc != 0) {
+ HIBLOG("kern_write_file returned %d\n", rc);
+ err = kIOReturnIOError;
+ goto exit;
+ }
+ }
+ hdr.signature = signature;
+ hdr.deviceBlockSize = vars->blockSize;
+
+ rc = kern_write_file(vars->fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION);
+ if (rc != 0) {
+ HIBLOG("kern_write_file returned %d\n", rc);
+ err = kIOReturnIOError;
+ goto exit;
+ }
- copy = vars->bufferLimit - vars->bufferOffset;
- if (copy > size)
- copy = size;
- else
- flush = true;
+exit:
+ return err;
+}
- if (bytes)
- {
- bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
- bytes += copy;
- }
- else
- bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
-
- size -= copy;
- vars->bufferOffset += copy;
- vars->position += copy;
+extern "C" boolean_t root_is_CF_drive;
- if (flush && vars->bufferOffset)
- {
- uint64_t offset = (vars->position - vars->bufferOffset
- - vars->extentPosition + vars->currentExtent->start);
- uint32_t length = (vars->bufferOffset);
+void
+IOOpenDebugDataFile(const char *fname, uint64_t size)
+{
+ IOReturn err;
+ OSData * imagePath = NULL;
+ uint64_t padding;
- if (cryptvars && vars->encryptStart && (vars->position > vars->encryptStart))
- {
- uint32_t encryptLen, encryptStart;
- encryptLen = vars->position - vars->encryptStart;
- if (encryptLen > length)
- encryptLen = length;
- encryptStart = length - encryptLen;
-
- // encrypt the buffer
- aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
- &cryptvars->aes_iv[0],
- encryptLen / AES_BLOCK_SIZE,
- vars->buffer + vars->bufferHalf + encryptStart,
- &cryptvars->ctx.encrypt);
- // save initial vector for following encrypts
- bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
- &cryptvars->aes_iv[0],
- AES_BLOCK_SIZE);
- }
+ if (!gDebugImageLock) {
+ gDebugImageLock = IOLockAlloc();
+ }
- if (vars->io)
- {
- err = IOHibernatePollerIODone(vars);
- if (kIOReturnSuccess != err)
- break;
- }
+ if (root_is_CF_drive) return;
-if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
-//if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
+ // Try to get a lock, but don't block for getting lock
+ if (!IOLockTryLock(gDebugImageLock)) {
+ HIBLOG("IOOpenDebugDataFile: Failed to get lock\n");
+ return;
+ }
- err = IOHibernatePollerIO(vars, kIOPolledWrite, vars->bufferHalf, offset, length);
- if (kIOReturnSuccess != err)
- break;
- vars->io = true;
+ if (gDebugImageFileVars || !fname || !size) {
+ HIBLOG("IOOpenDebugDataFile: conditions failed\n");
+ goto exit;
+ }
- vars->extentRemaining -= vars->bufferOffset;
- if (!vars->extentRemaining)
- {
- vars->currentExtent++;
- vars->extentRemaining = vars->currentExtent->length;
- vars->extentPosition = vars->position;
- if (!vars->extentRemaining)
- {
- err = kIOReturnOverrun;
- break;
- }
- }
+ padding = (PAGE_SIZE*2); // allocate couple more pages for header and fileextents
+ err = IOPolledFileOpen(fname, size+padding, 32ULL*1024*1024*1024,
+ NULL, 0,
+ &gDebugImageFileVars, &imagePath, NULL, 0);
- vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
- vars->bufferOffset = 0;
- if (vars->bufferSize <= vars->extentRemaining)
- vars->bufferLimit = vars->bufferSize;
- else
- vars->bufferLimit = vars->extentRemaining;
+ if ((kIOReturnSuccess == err) && imagePath)
+ {
+ if ((gDebugImageFileVars->fileSize < (size+padding)) ||
+ (gDebugImageFileVars->fileExtents->getLength() > PAGE_SIZE)) {
+ // Can't use the file
+ IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
+ HIBLOG("IOOpenDebugDataFile: too many file extents\n");
+ goto exit;
+ }
- flush = false;
- }
+ // write extents for debug data usage in EFI
+ IOWriteExtentsToFile(gDebugImageFileVars, kIOHibernateHeaderOpenSignature);
+ IOSetBootImageNVRAM(imagePath);
}
- while (size);
- return (err);
+exit:
+ IOLockUnlock(gDebugImageLock);
+
+ if (imagePath) imagePath->release();
+ return;
}
-static IOReturn
-IOPolledFileRead(IOPolledFileIOVars * vars,
- uint8_t * bytes, IOByteCount size,
- hibernate_cryptvars_t * cryptvars)
+void
+IOCloseDebugDataFile()
{
- IOReturn err = kIOReturnSuccess;
- IOByteCount copy;
+ IOSetBootImageNVRAM(0);
-// bytesWritten += size;
+ if (gDebugImageLock) {
+ IOLockLock(gDebugImageLock);
+ if (gDebugImageFileVars != 0) {
+ IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
+ }
+ IOLockUnlock(gDebugImageLock);
+ }
- do
- {
- copy = vars->bufferLimit - vars->bufferOffset;
- if (copy > size)
- copy = size;
- if (bytes)
- {
- bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
- bytes += copy;
- }
- size -= copy;
- vars->bufferOffset += copy;
-// vars->position += copy;
+}
- if (vars->bufferOffset == vars->bufferLimit)
- {
- if (vars->io)
- {
- err = IOHibernatePollerIODone(vars);
- if (kIOReturnSuccess != err)
- break;
- }
- else
- cryptvars = 0;
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
+DECLARE_IOHIBERNATEPROGRESSALPHA
- vars->position += vars->lastRead;
- vars->extentRemaining -= vars->lastRead;
- vars->bufferLimit = vars->lastRead;
+static void
+ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
+{
+ uint32_t rowBytes, pixelShift;
+ uint32_t x, y;
+ int32_t blob;
+ uint32_t alpha, in, color, result;
+ uint8_t * out;
+ uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
- if (!vars->extentRemaining)
- {
- vars->currentExtent++;
- vars->extentRemaining = vars->currentExtent->length;
- vars->extentPosition = vars->position;
- if (!vars->extentRemaining)
+ rowBytes = display->rowBytes;
+ pixelShift = display->depth >> 4;
+ if (pixelShift < 1) return;
+
+ screen += ((display->width
+ - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
+ + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
+
+ for (y = 0; y < kIOHibernateProgressHeight; y++)
+ {
+ out = screen + y * rowBytes;
+ for (blob = 0; blob < kIOHibernateProgressCount; blob++)
+ {
+ color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
+ for (x = 0; x < kIOHibernateProgressWidth; x++)
+ {
+ alpha = gIOHibernateProgressAlpha[y][x];
+ result = color;
+ if (alpha)
{
- err = kIOReturnOverrun;
- break;
+ if (0xff != alpha)
+ {
+ if (1 == pixelShift)
+ {
+ in = *((uint16_t *)out) & 0x1f; // 16
+ in = (in << 3) | (in >> 2);
+ }
+ else
+ in = *((uint32_t *)out) & 0xff; // 32
+ saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
+ result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
+ }
+ if (1 == pixelShift)
+ {
+ result >>= 3;
+ *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
+ }
+ else
+ *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
}
- }
+ out += (1 << pixelShift);
+ }
+ out += (kIOHibernateProgressSpacing << pixelShift);
+ }
+ }
+}
+
- if (vars->extentRemaining <= vars->bufferSize)
- vars->lastRead = vars->extentRemaining;
- else
- vars->lastRead = vars->bufferSize;
+static void
+ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
+{
+ uint32_t rowBytes, pixelShift;
+ uint32_t x, y;
+ int32_t blob, lastBlob;
+ uint32_t alpha, in, color, result;
+ uint8_t * out;
+ uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
- uint64_t offset = (vars->position
- - vars->extentPosition + vars->currentExtent->start);
- uint64_t length = (vars->lastRead);
+ pixelShift = display->depth >> 4;
+ if (pixelShift < 1)
+ return;
-//if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
+ rowBytes = display->rowBytes;
- err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
- if (kIOReturnSuccess != err)
- break;
- vars->io = true;
+ screen += ((display->width
+ - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
+ + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
+
+ lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
- vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
- vars->bufferOffset = 0;
+ screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
- if (cryptvars)
+ for (y = 0; y < kIOHibernateProgressHeight; y++)
+ {
+ out = screen + y * rowBytes;
+ for (blob = firstBlob; blob <= lastBlob; blob++)
+ {
+ color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
+ for (x = 0; x < kIOHibernateProgressWidth; x++)
{
- uint8_t thisVector[AES_BLOCK_SIZE];
- // save initial vector for following decrypts
- bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
- bcopy(vars->buffer + vars->bufferHalf + vars->lastRead - AES_BLOCK_SIZE,
- &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
- // decrypt the buffer
- aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
- &thisVector[0],
- vars->lastRead / AES_BLOCK_SIZE,
- vars->buffer + vars->bufferHalf,
- &cryptvars->ctx.decrypt);
+ alpha = gIOHibernateProgressAlpha[y][x];
+ result = color;
+ if (alpha)
+ {
+ if (0xff != alpha)
+ {
+ in = display->progressSaveUnder[blob][saveindex[blob]++];
+ result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
+ }
+ if (1 == pixelShift)
+ {
+ result >>= 3;
+ *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
+ }
+ else
+ *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
+ }
+ out += (1 << pixelShift);
}
- }
+ out += (kIOHibernateProgressSpacing << pixelShift);
+ }
}
- while (size);
-
- return (err);
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
+
IOReturn
-IOHibernateSystemSleep(void)
+IOHibernateIOKitSleep(void)
{
- IOReturn err;
- OSData * data;
- OSObject * obj;
- OSString * str;
- OSNumber * num;
-
- IOHibernateVars * vars = &gIOHibernateVars;
-
- if (vars->fileVars && vars->fileVars->fileRef)
- // already on the way down
- return (kIOReturnSuccess);
+ IOReturn ret = kIOReturnSuccess;
+ IOLockLock(gFSLock);
+ if (kFSOpening == gFSState)
+ {
+ gFSState = kFSTimedOut;
+ HIBLOG("hibernate file open timed out\n");
+ ret = kIOReturnTimeout;
+ }
+ IOLockUnlock(gFSLock);
+ return (ret);
+}
- gIOHibernateState = kIOHibernateStateInactive;
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
- if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateModeKey)))
- {
- if ((num = OSDynamicCast(OSNumber, obj)))
- gIOHibernateMode = num->unsigned32BitValue();
- if (kIOHibernateModeSleep & gIOHibernateMode)
- // default to discard clean for safe sleep
- gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
- | kIOHibernateModeDiscardCleanActive);
+IOReturn
+IOHibernateSystemHasSlept(void)
+{
+ IOReturn ret = kIOReturnSuccess;
+ IOHibernateVars * vars = &gIOHibernateVars;
+ OSObject * obj = 0;
+ OSData * data;
- obj->release();
- }
- if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeRatioKey)))
+ IOLockLock(gFSLock);
+ if ((kFSOpened != gFSState) && gIOHibernateMode)
{
- if ((num = OSDynamicCast(OSNumber, obj)))
- gIOHibernateFreeRatio = num->unsigned32BitValue();
- obj->release();
+ ret = kIOReturnTimeout;
}
- if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeTimeKey)))
- {
- if ((num = OSDynamicCast(OSNumber, obj)))
- gIOHibernateFreeTime = num->unsigned32BitValue();
+ IOLockUnlock(gFSLock);
+ if (kIOReturnSuccess != ret) return (ret);
+
+ if (gIOHibernateMode) obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
+ vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
+ if (obj && !vars->previewBuffer)
obj->release();
- }
- if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
+
+ vars->consoleMapping = NULL;
+ if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
{
- if ((str = OSDynamicCast(OSString, obj)))
- strcpy(gIOHibernateFilename, str->getCStringNoCopy());
- obj->release();
+ vars->previewBuffer->release();
+ vars->previewBuffer = 0;
}
- if (!gIOHibernateMode || !gIOHibernateFilename[0])
- return (kIOReturnUnsupported);
-
- HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
-
- do
+ if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
+ && vars->previewBuffer
+ && (data = OSDynamicCast(OSData,
+ IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
{
- vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(0, 4 * page_size, page_size);
- vars->ioBuffer = IOBufferMemoryDescriptor::withOptions(0, 2 * kDefaultIOSize, page_size);
+ UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
+ HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
- if (!vars->srcBuffer || !vars->ioBuffer)
- {
- err = kIOReturnNoMemory;
- break;
- }
+ IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
- err = IOPolledFileOpen(gIOHibernateFilename, vars->ioBuffer,
- &vars->fileVars, &vars->fileExtents, &data);
- if (KERN_SUCCESS != err)
- {
- HIBLOG("IOPolledFileOpen(%x)\n", err);
- break;
- }
- if (vars->fileVars->fileRef)
+ if (kIOHibernatePreviewUpdates & flags)
{
- // invalidate the image file
- gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
- int err = kern_write_file(vars->fileVars->fileRef, 0,
- (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
- if (KERN_SUCCESS != err)
- HIBLOG("kern_write_file(%d)\n", err);
+ PE_Video consoleInfo;
+ hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
+
+ IOService::getPlatform()->getConsoleInfo(&consoleInfo);
+
+ graphicsInfo->width = consoleInfo.v_width;
+ graphicsInfo->height = consoleInfo.v_height;
+ graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
+ graphicsInfo->depth = consoleInfo.v_depth;
+ vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
+
+ HIBPRINT("video %p %d %d %d\n",
+ vars->consoleMapping, graphicsInfo->depth,
+ graphicsInfo->width, graphicsInfo->height);
+ if (vars->consoleMapping)
+ ProgressInit(graphicsInfo, vars->consoleMapping,
+ &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
}
+ }
- bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
-
- boolean_t encryptedswap;
- err = hibernate_setup(gIOHibernateCurrentHeader,
- gIOHibernateFreeRatio, gIOHibernateFreeTime,
- &vars->page_list, &vars->page_list_wired, &encryptedswap);
- if (KERN_SUCCESS != err)
- {
- HIBLOG("hibernate_setup(%d)\n", err);
- break;
- }
-
- if (encryptedswap)
- gIOHibernateMode ^= kIOHibernateModeEncrypt;
-
- vars->videoAllocSize = kVideoMapSize;
- if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
- vars->videoMapping = 0;
-
- // generate crypt keys
- for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
- vars->wiredCryptKey[i] = random();
- for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
- vars->cryptKey[i] = random();
+ if (gIOOptionsEntry)
+ gIOOptionsEntry->sync();
- // set nvram
+ return (ret);
+}
- IORegistryEntry * regEntry;
- if (!gIOOptionsEntry)
- {
- regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
- gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
- if (regEntry && !gIOOptionsEntry)
- regEntry->release();
- }
- if (!gIOChosenEntry)
- gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
- if (gIOOptionsEntry)
+static DeviceTreeNode *
+MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry)
+{
+ DeviceTreeNodeProperty * prop;
+ DeviceTreeNode * child;
+ IORegistryEntry * childRegEntry;
+ const char * nameProp;
+ unsigned int propLen, idx;
+
+ prop = (DeviceTreeNodeProperty *) (entry + 1);
+ for (idx = 0; idx < entry->nProperties; idx++)
+ {
+ if (regEntry && (0 != strcmp("name", prop->name)))
{
- const OSSymbol * sym;
- size_t len;
- char valueString[16];
-
- sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
- if (sym)
- {
- gIOOptionsEntry->setProperty(sym, data);
- sym->release();
- }
- data->release();
-
- vars->saveBootDevice = gIOOptionsEntry->copyProperty(kIOSelectedBootDeviceKey);
- if (gIOChosenEntry)
- {
- OSData * bootDevice = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOBootPathKey));
- if (bootDevice)
- {
- sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
- OSString * str2 = OSString::withCStringNoCopy((const char *) bootDevice->getBytesNoCopy());
- if (sym && str2)
- gIOOptionsEntry->setProperty(sym, str2);
- if (sym)
- sym->release();
- if (str2)
- str2->release();
- }
-
- data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMemorySignatureKey));
- if (data)
- {
- vars->haveFastBoot = true;
-
- len = sprintf(valueString, "0x%lx", *((UInt32 *)data->getBytesNoCopy()));
- data = OSData::withBytes(valueString, len + 1);
- sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
- if (sym && data)
- gIOOptionsEntry->setProperty(sym, data);
- if (sym)
- sym->release();
- if (data)
- data->release();
- }
- data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
- if (data)
- gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
- }
-
- if (kIOHibernateModeEncrypt & gIOHibernateMode)
- {
- data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
- sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
- if (sym && data)
- gIOOptionsEntry->setProperty(sym, data);
- if (sym)
- sym->release();
- if (data)
- data->release();
- if (gIOHibernateBootSignature[0])
- {
- data = OSData::withCapacity(16);
- sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
- if (sym && data)
- {
- char c;
- uint8_t value;
- for (uint32_t i = 0; (c = gIOHibernateBootSignature[i]); i++)
- {
- if (c >= 'a')
- c -= 'a' - 10;
- else if (c >= 'A')
- c -= 'A' - 10;
- else if (c >= '0')
- c -= '0';
- else
- continue;
- value = (value << 4) | c;
- if (i & 1)
- data->appendBytes(&value, sizeof(value));
- }
- gIOOptionsEntry->setProperty(sym, data);
- }
- if (sym)
- sym->release();
- if (data)
- data->release();
- }
- }
-
- if (!vars->haveFastBoot)
- {
- // set boot volume to zero
- IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
- if (platform && (kIOReturnSuccess == platform->readXPRAM(kXPRamAudioVolume,
- &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume))))
- {
- uint8_t newVolume;
- newVolume = vars->saveBootAudioVolume & 0xf8;
- platform->writeXPRAM(kXPRamAudioVolume,
- &newVolume, sizeof(newVolume));
- }
- }
+ regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
+// HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
}
- // --
-
- gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
- gIOHibernateState = kIOHibernateStateHibernating;
+ prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
}
- while (false);
- return (err);
+ child = (DeviceTreeNode *) prop;
+ for (idx = 0; idx < entry->nChildren; idx++)
+ {
+ if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen))
+ panic("no name");
+ childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
+// HIBPRINT("%s == %p\n", nameProp, childRegEntry);
+ child = MergeDeviceTree(child, childRegEntry);
+ }
+ return (child);
}
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
IOReturn
-IOHibernateSystemHasSlept(void)
+IOHibernateSystemWake(void)
{
- IOHibernateVars * vars = &gIOHibernateVars;
-
- if ((vars->previewData = OSDynamicCast(OSData,
- IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewBufferKey))))
+ if (kFSOpened == gFSState)
{
- vars->previewBuffer = IOMemoryDescriptor::withAddress(
- (void *) vars->previewData->getBytesNoCopy(),
- vars->previewData->getLength(),
- kIODirectionInOut);
-
- if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
- {
- vars->previewBuffer->release();
- vars->previewBuffer = 0;
- }
- if (!vars->previewBuffer)
- vars->previewData = 0;
+ IOPolledFilePollersClose(gIOHibernateVars.fileVars, kIOPolledPostflightState);
+ IOHibernateDone(&gIOHibernateVars);
+ }
+ else
+ {
+ IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
+ IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
}
- if (gIOOptionsEntry)
- gIOOptionsEntry->sync();
-
return (kIOReturnSuccess);
}
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-IOReturn
-IOHibernateSystemWake(void)
+static IOReturn
+IOHibernateDone(IOHibernateVars * vars)
{
- IOHibernateVars * vars = &gIOHibernateVars;
-
- hibernate_teardown(vars->page_list, vars->page_list_wired);
+ hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
if (vars->videoMapping)
{
IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
if (vars->videoAllocSize)
// dealloc range
- kmem_free(kernel_map, trunc_page_32(vars->videoMapping), vars->videoAllocSize);
+ kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
}
if (vars->previewBuffer)
vars->previewBuffer = 0;
}
- if (vars->fileVars)
+ if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
+ {
+ IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
+ gIOHibernateCurrentHeader->options, 32);
+ }
+ else
+ {
+ IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
+ }
+
+ if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
+ && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus))
{
- IOPolledFileClose(vars->fileVars);
+ IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
+ &gIOHibernateGraphicsInfo->gfxStatus,
+ sizeof(gIOHibernateGraphicsInfo->gfxStatus));
+ }
+ else
+ {
+ IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
}
// invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
- OSData * data = OSData::withCapacity(4);
- if (gIOOptionsEntry && data)
- {
- const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
- if (sym)
- {
- gIOOptionsEntry->setProperty(sym, data);
- sym->release();
- }
- sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
- if (sym)
- {
- if (vars->saveBootDevice)
+#if defined(__i386__) || defined(__x86_64__)
+ IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
+ IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
+
+ /*
+ * Hibernate variable is written to NVRAM on platforms in which RtcRam
+ * is not backed by coin cell. Remove Hibernate data from NVRAM.
+ */
+ if (gIOOptionsEntry) {
+
+ if (gIOHibernateRTCVariablesKey) {
+ if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
+ gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
+ }
+ }
+
+ if (gIOHibernateBootNextKey)
{
- gIOOptionsEntry->setProperty(sym, vars->saveBootDevice);
- vars->saveBootDevice->release();
+ if (gIOHibernateBootNextSave)
+ {
+ gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
+ gIOHibernateBootNextSave->release();
+ gIOHibernateBootNextSave = NULL;
+ }
+ else
+ gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
}
- sym->release();
- }
- sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
- if (sym)
- {
- gIOOptionsEntry->setProperty(sym, data);
- sym->release();
- }
- sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
- if (sym)
- {
- gIOOptionsEntry->removeProperty(sym);
- sym->release();
- }
- }
- if (data)
- data->release();
+ if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) gIOOptionsEntry->sync();
+ }
+#endif
- if (gIOOptionsEntry)
+ if (vars->srcBuffer) vars->srcBuffer->release();
+ bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
+ if (vars->handoffBuffer)
{
- if (!vars->haveFastBoot)
+ if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
{
- // reset boot audio volume
- IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
- if (platform)
- platform->writeXPRAM(kXPRamAudioVolume,
- &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume));
+ IOHibernateHandoff * handoff;
+ bool done = false;
+ for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
+ !done;
+ handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
+ {
+ HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
+ uint8_t * data = &handoff->data[0];
+ switch (handoff->type)
+ {
+ case kIOHibernateHandoffTypeEnd:
+ done = true;
+ break;
+
+ case kIOHibernateHandoffTypeDeviceTree:
+ MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
+ break;
+
+ case kIOHibernateHandoffTypeKeyStore:
+#if defined(__i386__) || defined(__x86_64__)
+ {
+ IOBufferMemoryDescriptor *
+ md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
+ if (md)
+ {
+ IOSetKeyStoreData(md);
+ }
+ }
+#endif
+ break;
+
+ default:
+ done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
+ break;
+ }
+ }
}
-
- // sync now to hardware if the booter has not
- if (kIOHibernateStateInactive == gIOHibernateState)
- gIOOptionsEntry->sync();
- else
- // just sync the variables in case a later panic syncs nvram (it won't sync variables)
- gIOOptionsEntry->syncOFVariables();
+ vars->handoffBuffer->release();
}
- if (vars->srcBuffer)
- vars->srcBuffer->release();
- if (vars->ioBuffer)
- vars->ioBuffer->release();
- if (vars->fileExtents)
- vars->fileExtents->release();
-
bzero(vars, sizeof(*vars));
// gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
return (kIOReturnSuccess);
}
+void
+IOHibernateSystemPostWakeTrim(void * p1, void * p2)
+{
+ // invalidate & close the image file
+ if (p1) IOLockLock(gFSLock);
+ if (kFSTrimDelay == gFSState)
+ {
+ IOPolledFileIOVars * vars = &gFileVars;
+ IOPolledFileClose(&vars,
+#if DISABLE_TRIM
+ 0, NULL, 0, 0, 0);
+#else
+ 0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader),
+ sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize);
+#endif
+ gFSState = kFSIdle;
+ }
+ if (p1) IOLockUnlock(gFSLock);
+}
+
IOReturn
IOHibernateSystemPostWake(void)
{
- if (gIOHibernateFileRef)
+ gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
+ IOLockLock(gFSLock);
+ if (kFSTrimDelay == gFSState) IOHibernateSystemPostWakeTrim(NULL, NULL);
+ else if (kFSOpened != gFSState) gFSState = kFSIdle;
+ else
{
- kern_close_file_for_direct_io(gIOHibernateFileRef);
- gIOHibernateFileRef = 0;
+ AbsoluteTime deadline;
+
+ gFSState = kFSTrimDelay;
+ clock_interval_to_deadline(TRIM_DELAY, kMillisecondScale, &deadline );
+ thread_call_enter1_delayed(gIOHibernateTrimCalloutEntry, NULL, deadline);
}
+ IOLockUnlock(gFSLock);
+
+ // IOCloseDebugDataFile() calls IOSetBootImageNVRAM() unconditionally
+ IOCloseDebugDataFile( );
return (kIOReturnSuccess);
}
+uint32_t IOHibernateWasScreenLocked(void)
+{
+ uint32_t ret = 0;
+ if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry)
+ {
+ OSData *
+ data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
+ if (data)
+ {
+ ret = ((uint32_t *)data->getBytesNoCopy())[0];
+ gIOChosenEntry->setProperty(kIOBooterScreenLockStateKey, data);
+ }
+ }
+ else gIOChosenEntry->removeProperty(kIOBooterScreenLockStateKey);
+
+ return (ret);
+}
+
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
+ CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
+ gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
+SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
+ CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
+ gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
+ CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
+ &gIOHibernateMode, 0, "");
+SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
+ CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
+ &_hibernateStats, hibernate_statistics_t, "");
+
+SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
+ CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+ &_hibernateStats.graphicsReadyTime, 0, "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
+ CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+ &_hibernateStats.wakeNotificationTime, 0, "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
+ CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+ &_hibernateStats.lockScreenReadyTime, 0, "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
+ CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+ &_hibernateStats.hidReadyTime, 0, "");
+
+
void
IOHibernateSystemInit(IOPMrootDomain * rootDomain)
{
+ gIOHibernateBootImageKey = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
+
+#if defined(__i386__) || defined(__x86_64__)
+ gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
+ gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
+ gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
+ gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
+#endif /* defined(__i386__) || defined(__x86_64__) */
+
OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
if (data)
{
data->release();
}
- if (PE_parse_boot_arg("hfile", gIOHibernateFilename))
+ if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))
gIOHibernateMode = kIOHibernateModeOn;
else
gIOHibernateFilename[0] = 0;
- static SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
- CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
- gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
sysctl_register_oid(&sysctl__kern_hibernatefile);
-
- static SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
- CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
- gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
sysctl_register_oid(&sysctl__kern_bootsignature);
-
- static SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
- CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
- &gIOHibernateMode, 0, "");
sysctl_register_oid(&sysctl__kern_hibernatemode);
+ sysctl_register_oid(&sysctl__kern_hibernatestatistics);
+ sysctl_register_oid(&sysctl__kern_hibernategraphicsready);
+ sysctl_register_oid(&sysctl__kern_hibernatewakenotification);
+ sysctl_register_oid(&sysctl__kern_hibernatelockscreenready);
+ sysctl_register_oid(&sysctl__kern_hibernatehidready);
+
+ gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
+
+ gFSLock = IOLockAlloc();
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-static void
-hibernate_setup_for_wake(void)
+static IOReturn
+IOHibernatePolledFileWrite(IOPolledFileIOVars * vars,
+ const uint8_t * bytes, IOByteCount size,
+ IOPolledFileCryptVars * cryptvars)
{
-#if __ppc__
- // go slow (state needed for wake)
- ml_set_processor_speed(1);
-#endif
+ IOReturn err;
+
+ err = IOPolledFileWrite(vars, bytes, size, cryptvars);
+ if ((kIOReturnSuccess == err) && hibernate_should_abort()) err = kIOReturnAborted;
+
+ return (err);
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-extern "C" boolean_t
+extern "C" uint32_t
hibernate_write_image(void)
{
IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
IOHibernateVars * vars = &gIOHibernateVars;
IOPolledFileExtent * fileExtents;
+ _static_assert_1_arg(sizeof(IOHibernateImageHeader) == 512);
+
uint32_t pageCount, pagesDone;
IOReturn err;
- vm_offset_t ppnum;
- IOItemCount page, count;
+ vm_offset_t ppnum, page;
+ IOItemCount count;
uint8_t * src;
uint8_t * data;
+ uint8_t * compressed;
+ uint8_t * scratch;
IOByteCount pageCompressedSize;
uint64_t compressedSize, uncompressedSize;
uint64_t image1Size = 0;
uint32_t bitmap_size;
- bool iterDone, pollerOpen, needEncryptStart;
+ bool iterDone, pollerOpen, needEncrypt;
uint32_t restore1Sum, sum, sum1, sum2;
+ int wkresult;
uint32_t tag;
uint32_t pageType;
uint32_t pageAndCount[2];
+ addr64_t phys64;
+ IOByteCount segLen;
+ uintptr_t hibernateBase;
+ uintptr_t hibernateEnd;
AbsoluteTime startTime, endTime;
- AbsoluteTime allTime, compTime, decoTime;
+ AbsoluteTime allTime, compTime;
+ uint64_t compBytes;
uint64_t nsec;
uint32_t lastProgressStamp = 0;
uint32_t progressStamp;
+ uint32_t blob, lastBlob = (uint32_t) -1L;
+
+ uint32_t wiredPagesEncrypted;
+ uint32_t dirtyPagesEncrypted;
+ uint32_t wiredPagesClear;
+ uint32_t svPageCount;
+ uint32_t zvPageCount;
- hibernate_cryptvars_t _cryptvars;
- hibernate_cryptvars_t * cryptvars = 0;
+ IOPolledFileCryptVars _cryptvars;
+ IOPolledFileCryptVars * cryptvars = 0;
- if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
- return (false /* sleep */ );
+ wiredPagesEncrypted = 0;
+ dirtyPagesEncrypted = 0;
+ wiredPagesClear = 0;
+ svPageCount = 0;
+ zvPageCount = 0;
+
+ if (!vars->fileVars
+ || !vars->fileVars->pollers
+ || !(kIOHibernateModeOn & gIOHibernateMode)) return (kIOHibernatePostWriteSleep);
+
+ if (kIOHibernateModeSleep & gIOHibernateMode)
+ kdebug_enable = save_kdebug_enable;
+
+ KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START, 0, 0, 0, 0, 0);
+ IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
restore1Sum = sum1 = sum2 = 0;
+#if CRYPTO
// encryption data. "iv" is the "initial vector".
if (kIOHibernateModeEncrypt & gIOHibernateMode)
{
0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
cryptvars = &gIOHibernateCryptWakeContext;
- bzero(cryptvars, sizeof(hibernate_cryptvars_t));
+ bzero(cryptvars, sizeof(IOPolledFileCryptVars));
aes_encrypt_key(vars->cryptKey,
kIOHibernateAESKeySize,
&cryptvars->ctx.encrypt);
&cryptvars->ctx.decrypt);
cryptvars = &_cryptvars;
- bzero(cryptvars, sizeof(hibernate_cryptvars_t));
+ bzero(cryptvars, sizeof(IOPolledFileCryptVars));
+ for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++)
+ vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
+ bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
aes_encrypt_key(vars->wiredCryptKey,
kIOHibernateAESKeySize,
&cryptvars->ctx.encrypt);
bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
- bzero(gIOHibernateCryptWakeVars, sizeof(hibernate_cryptwakevars_t));
}
-
- hibernate_setup_for_wake();
+#endif /* CRYPTO */
hibernate_page_list_setall(vars->page_list,
vars->page_list_wired,
+ vars->page_list_pal,
+ false /* !preflight */,
+ /* discard_all */
+ ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
+ && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
&pageCount);
HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
- fileExtents = (IOPolledFileExtent *) vars->fileExtents->getBytesNoCopy();
+ fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy();
#if 0
count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
}
#endif
- needEncryptStart = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
-
+ needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
AbsoluteTime_to_scalar(&compTime) = 0;
- AbsoluteTime_to_scalar(&decoTime) = 0;
+ compBytes = 0;
clock_get_uptime(&allTime);
-
+ IOService::getPMRootDomain()->pmStatsRecordEvent(
+ kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
do
{
compressedSize = 0;
uncompressedSize = 0;
- iterDone = false;
- pageType = 0; // wired pages first
+ svPageCount = 0;
+ zvPageCount = 0;
- IOPolledFileSeek(vars->fileVars, sizeof(IOHibernateImageHeader));
+ IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
ml_get_interrupts_enabled());
- err = IOHibernatePollerOpen(vars->fileVars, kIOPolledBeforeSleepState, vars->ioBuffer);
+ err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState,
+ // abortable if not low battery
+ !IOService::getPMRootDomain()->mustHibernate());
HIBLOG("IOHibernatePollerOpen(%x)\n", err);
pollerOpen = (kIOReturnSuccess == err);
if (!pollerOpen)
// copy file block extent list if larger than header
- count = vars->fileExtents->getLength();
+ count = vars->fileVars->fileExtents->getLength();
if (count > sizeof(header->fileExtentMap))
{
count -= sizeof(header->fileExtentMap);
- err = IOPolledFileWrite(vars->fileVars,
+ err = IOHibernatePolledFileWrite(vars->fileVars,
((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
if (kIOReturnSuccess != err)
break;
}
+ hibernateBase = HIB_BASE; /* Defined in PAL headers */
+ hibernateEnd = (segHIBB + segSizeHIB);
+
// copy out restore1 code
-
- page = atop_32(sectHIBB);
- count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
- header->restore1CodePage = page;
+
+ for (count = 0;
+ (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+ count += segLen)
+ {
+ for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++)
+ {
+ gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
+ }
+ }
+
+ page = atop_32(kvtophys(hibernateBase));
+ count = atop_32(round_page(hibernateEnd) - hibernateBase);
+ header->restore1CodePhysPage = page;
+ header->restore1CodeVirt = hibernateBase;
header->restore1PageCount = count;
- header->restore1CodeOffset = ((uint32_t) &hibernate_machine_entrypoint) - sectHIBB;
- header->restore1StackOffset = ((uint32_t) &gIOHibernateRestoreStackEnd[0]) - 64 - sectHIBB;
+ header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
+ header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
- // sum __HIB sect, with zeros for the stack
- src = (uint8_t *) trunc_page(sectHIBB);
+ // sum __HIB seg, with zeros for the stack
+ src = (uint8_t *) trunc_page(hibernateBase);
for (page = 0; page < count; page++)
{
if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
- restore1Sum += hibernate_sum(src, page_size);
+ restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
else
- restore1Sum += 0x10000001;
+ restore1Sum += 0x00000000;
src += page_size;
}
sum1 = restore1Sum;
- // write the __HIB sect, with zeros for the stack
+ // write the __HIB seg, with zeros for the stack
- src = (uint8_t *) trunc_page(sectHIBB);
- count = ((uint32_t) &gIOHibernateRestoreStack[0]) - trunc_page(sectHIBB);
+ src = (uint8_t *) trunc_page(hibernateBase);
+ count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
if (count)
{
- err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
+ err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
if (kIOReturnSuccess != err)
break;
}
- err = IOPolledFileWrite(vars->fileVars,
+ err = IOHibernatePolledFileWrite(vars->fileVars,
(uint8_t *) 0,
&gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
cryptvars);
if (kIOReturnSuccess != err)
break;
src = &gIOHibernateRestoreStackEnd[0];
- count = round_page(sectHIBB + sectSizeHIB) - ((uint32_t) src);
+ count = round_page(hibernateEnd) - ((uintptr_t) src);
if (count)
{
- err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
+ err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
if (kIOReturnSuccess != err)
break;
}
- // write the preview buffer
+ if (kIOHibernateModeEncrypt & gIOHibernateMode)
+ {
+ vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
+ vars->fileVars->encryptEnd = UINT64_MAX;
+ HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
+ }
- addr64_t phys64;
- IOByteCount segLen;
+ // write the preview buffer
- if (vars->previewData)
+ if (vars->previewBuffer)
{
ppnum = 0;
count = 0;
do
{
- phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen);
+ phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
pageAndCount[0] = atop_64(phys64);
pageAndCount[1] = atop_32(segLen);
- err = IOPolledFileWrite(vars->fileVars,
+ err = IOHibernatePolledFileWrite(vars->fileVars,
(const uint8_t *) &pageAndCount, sizeof(pageAndCount),
cryptvars);
if (kIOReturnSuccess != err)
if (kIOReturnSuccess != err)
break;
- src = (uint8_t *) vars->previewData->getBytesNoCopy();
- count = vars->previewData->getLength();
+ src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
+
+ ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
+
+ count = vars->previewBuffer->getLength();
header->previewPageListSize = ppnum;
header->previewSize = count + ppnum;
for (page = 0; page < count; page += page_size)
- sum1 += hibernate_sum(src + page, page_size);
-
- err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
+ {
+ phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
+ sum1 += hibernate_sum_page(src + page, atop_64(phys64));
+ }
+ err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
if (kIOReturnSuccess != err)
break;
}
// mark areas for no save
-
+ IOMemoryDescriptor * ioBuffer;
+ ioBuffer = IOPolledFileGetIOBuffer(vars->fileVars);
for (count = 0;
- (phys64 = vars->ioBuffer->getPhysicalSegment64(count, &segLen));
+ (phys64 = ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
count += segLen)
{
hibernate_set_page_state(vars->page_list, vars->page_list_wired,
}
for (count = 0;
- (phys64 = vars->srcBuffer->getPhysicalSegment64(count, &segLen));
+ (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
count += segLen)
{
hibernate_set_page_state(vars->page_list, vars->page_list_wired,
bitmap_size = vars->page_list_wired->list_size;
src = (uint8_t *) vars->page_list_wired;
- err = IOPolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
+ err = IOHibernatePolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
if (kIOReturnSuccess != err)
break;
// mark more areas for no save, but these are not available
// for trashing during restore
+
+ hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
-#if !__i386__
- page = atop_32(sectHIBB);
- count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
-#else
- // XXX
- page = atop_32(sectHIBB & 0x3FFFFFFF);
- count = atop_32(round_page((sectHIBB + sectSizeHIB) & 0x3FFFFFFF)) - page;
-#endif
+
+ page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
+ count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
hibernate_set_page_state(vars->page_list, vars->page_list_wired,
page, count,
kIOHibernatePageStateFree);
pageCount -= count;
-
-
if (vars->previewBuffer) for (count = 0;
- (phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen));
+ (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
count += segLen)
{
hibernate_set_page_state(vars->page_list, vars->page_list_wired,
pageCount -= atop_32(segLen);
}
- src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
-
- void * iter = 0;
- pagesDone = 0;
-
- HIBLOG("writing %d pages\n", pageCount);
-
- do
+ for (count = 0;
+ (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+ count += segLen)
{
- count = hibernate_page_list_iterate(pageType ? vars->page_list : vars->page_list_wired,
- &iter, &ppnum);
-// kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
-
- iterDone = !count;
-
- pageAndCount[0] = ppnum;
- pageAndCount[1] = count;
- err = IOPolledFileWrite(vars->fileVars,
- (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
- cryptvars);
- if (kIOReturnSuccess != err)
- break;
+ hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+ atop_64(phys64), atop_32(segLen),
+ kIOHibernatePageStateFree);
+ pageCount -= atop_32(segLen);
+ }
- for (page = 0; page < count; page++)
- {
- err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(ppnum), page_size);
- if (err)
- {
- HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%d] %x\n", __LINE__, ppnum, err);
- break;
- }
-
- sum = hibernate_sum(src, page_size);
-
- clock_get_uptime(&startTime);
+ src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
+ compressed = src + page_size;
+ scratch = compressed + page_size;
- pageCompressedSize = WKdm_compress ((WK_word*) src, (WK_word*) (src + page_size), PAGE_SIZE_IN_WORDS);
+ pagesDone = 0;
+ lastBlob = 0;
- clock_get_uptime(&endTime);
- ADD_ABSOLUTETIME(&compTime, &endTime);
- SUB_ABSOLUTETIME(&compTime, &startTime);
-
- if (kIOHibernateModeEncrypt & gIOHibernateMode)
- pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
+ HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
+ bitmap_size, header->previewSize,
+ pageCount, vars->fileVars->position);
+
+ enum
+ // pageType
+ {
+ kWired = 0x02,
+ kEncrypt = 0x01,
+ kWiredEncrypt = kWired | kEncrypt,
+ kWiredClear = kWired,
+ kUnwiredEncrypt = kEncrypt
+ };
+
+ bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features()));
+#define _pmap_is_noencrypt(x) (cpuAES ? false : pmap_is_noencrypt((x)))
+
+ for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)
+ {
+ if (kUnwiredEncrypt == pageType)
+ {
+ // start unwired image
+ if (kIOHibernateModeEncrypt & gIOHibernateMode)
+ {
+ vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
+ vars->fileVars->encryptEnd = UINT64_MAX;
+ HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
+ }
+ bcopy(&cryptvars->aes_iv[0],
+ &gIOHibernateCryptWakeContext.aes_iv[0],
+ sizeof(cryptvars->aes_iv));
+ cryptvars = &gIOHibernateCryptWakeContext;
+ }
+ for (iterDone = false, ppnum = 0; !iterDone; )
+ {
+ count = hibernate_page_list_iterate((kWired & pageType)
+ ? vars->page_list_wired : vars->page_list,
+ &ppnum);
+// kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
+ iterDone = !count;
- if (pageCompressedSize > page_size)
+ if (count && (kWired & pageType) && needEncrypt)
{
-// HIBLOG("------------lose: %d\n", pageCompressedSize);
- pageCompressedSize = page_size;
+ uint32_t checkIndex;
+ for (checkIndex = 0;
+ (checkIndex < count)
+ && (((kEncrypt & pageType) == 0) == _pmap_is_noencrypt(ppnum + checkIndex));
+ checkIndex++)
+ {}
+ if (!checkIndex)
+ {
+ ppnum++;
+ continue;
+ }
+ count = checkIndex;
}
- if (pageCompressedSize != page_size)
- data = (src + page_size);
- else
- data = src;
-
- tag = pageCompressedSize | kIOHibernateTagSignature;
-
- if (pageType)
- sum2 += sum;
+ switch (pageType)
+ {
+ case kWiredEncrypt: wiredPagesEncrypted += count; break;
+ case kWiredClear: wiredPagesClear += count; break;
+ case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
+ }
+
+ if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */}
else
- sum1 += sum;
-
- if (needEncryptStart && (ppnum >= atop_32(sectDATAB)))
{
- // start encrypting partway into the data about to be written
- vars->fileVars->encryptStart = (vars->fileVars->position + AES_BLOCK_SIZE - 1)
- & ~(AES_BLOCK_SIZE - 1);
- needEncryptStart = false;
+ pageAndCount[0] = ppnum;
+ pageAndCount[1] = count;
+ err = IOHibernatePolledFileWrite(vars->fileVars,
+ (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
+ cryptvars);
+ if (kIOReturnSuccess != err)
+ break;
}
-
- err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
- if (kIOReturnSuccess != err)
- break;
-
- err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
- if (kIOReturnSuccess != err)
- break;
-
- compressedSize += pageCompressedSize;
- if (pageCompressedSize)
- uncompressedSize += page_size;
- ppnum++;
- pagesDone++;
- if (0 == (8191 & pagesDone))
+ for (page = ppnum; page < (ppnum + count); page++)
{
+ err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
+ if (err)
+ {
+ HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
+ break;
+ }
+
+ sum = hibernate_sum_page(src, page);
+ if (kWired & pageType)
+ sum1 += sum;
+ else
+ sum2 += sum;
+
+ clock_get_uptime(&startTime);
+ wkresult = WKdm_compress_new((const WK_word*) src,
+ (WK_word*) compressed,
+ (WK_word*) scratch,
+ page_size - 4);
+
clock_get_uptime(&endTime);
- SUB_ABSOLUTETIME(&endTime, &allTime);
- absolutetime_to_nanoseconds(endTime, &nsec);
- progressStamp = nsec / 750000000ULL;
- if (progressStamp != lastProgressStamp)
+ ADD_ABSOLUTETIME(&compTime, &endTime);
+ SUB_ABSOLUTETIME(&compTime, &startTime);
+
+ compBytes += page_size;
+ pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
+
+ if (pageCompressedSize == 0)
+ {
+ pageCompressedSize = 4;
+ data = src;
+
+ if (*(uint32_t *)src)
+ svPageCount++;
+ else
+ zvPageCount++;
+ }
+ else
+ {
+ if (pageCompressedSize != page_size)
+ data = compressed;
+ else
+ data = src;
+ }
+
+ tag = pageCompressedSize | kIOHibernateTagSignature;
+ err = IOHibernatePolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
+ if (kIOReturnSuccess != err)
+ break;
+
+ err = IOHibernatePolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
+ if (kIOReturnSuccess != err)
+ break;
+
+ compressedSize += pageCompressedSize;
+ uncompressedSize += page_size;
+ pagesDone++;
+
+ if (vars->consoleMapping && (0 == (1023 & pagesDone)))
+ {
+ blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
+ if (blob != lastBlob)
+ {
+ ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
+ lastBlob = blob;
+ }
+ }
+ if (0 == (8191 & pagesDone))
{
- lastProgressStamp = progressStamp;
- HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
+ clock_get_uptime(&endTime);
+ SUB_ABSOLUTETIME(&endTime, &allTime);
+ absolutetime_to_nanoseconds(endTime, &nsec);
+ progressStamp = nsec / 750000000ULL;
+ if (progressStamp != lastProgressStamp)
+ {
+ lastProgressStamp = progressStamp;
+ HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
+ }
}
}
+ if (kIOReturnSuccess != err)
+ break;
+ ppnum = page;
}
+
if (kIOReturnSuccess != err)
break;
- if (iterDone && !pageType)
+
+ if ((kEncrypt & pageType) && vars->fileVars->encryptStart)
+ {
+ vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
+ HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
+ }
+
+ if (kWiredEncrypt != pageType)
{
- err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
+ // end of image1/2 - fill to next block
+ err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
if (kIOReturnSuccess != err)
break;
+ }
+ if (kWiredClear == pageType)
+ {
+ // enlarge wired image for test
+// err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
- iterDone = false;
- pageType = 1;
- iter = 0;
+ // end wired image
+ header->encryptStart = vars->fileVars->encryptStart;
+ header->encryptEnd = vars->fileVars->encryptEnd;
image1Size = vars->fileVars->position;
- if (cryptvars)
- {
- bcopy(&cryptvars->aes_iv[0],
- &gIOHibernateCryptWakeContext.aes_iv[0],
- sizeof(cryptvars->aes_iv));
- cryptvars = &gIOHibernateCryptWakeContext;
- }
- HIBLOG("image1Size %qd\n", image1Size);
+ HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
+ image1Size, header->encryptStart, header->encryptEnd);
}
}
- while (!iterDone);
- if (kIOReturnSuccess != err)
- break;
- err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
if (kIOReturnSuccess != err)
+ {
+ if (kIOReturnOverrun == err)
+ {
+ // update actual compression ratio on not enough space (for retry)
+ gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
+ }
+
+ // update partial amount written (for IOPolledFileClose cleanup/unmap)
+ header->imageSize = vars->fileVars->position;
break;
+ }
// Header:
header->image1Size = image1Size;
header->bitmapSize = bitmap_size;
header->pageCount = pageCount;
- header->encryptStart = vars->fileVars->encryptStart;
header->restore1Sum = restore1Sum;
header->image1Sum = sum1;
header->image2Sum = sum2;
+ header->sleepTime = gIOLastSleepTime.tv_sec;
+
+ header->compression = (compressedSize << 8) / uncompressedSize;
+ gIOHibernateCompression = header->compression;
- count = vars->fileExtents->getLength();
+ count = vars->fileVars->fileExtents->getLength();
if (count > sizeof(header->fileExtentMap))
{
header->fileExtentMapSize = count;
else
header->fileExtentMapSize = sizeof(header->fileExtentMap);
bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
+
+ header->deviceBase = vars->fileVars->block0;
+ header->deviceBlockSize = vars->fileVars->blockSize;
IOPolledFileSeek(vars->fileVars, 0);
- err = IOPolledFileWrite(vars->fileVars,
+ err = IOHibernatePolledFileWrite(vars->fileVars,
(uint8_t *) header, sizeof(IOHibernateImageHeader),
cryptvars);
if (kIOReturnSuccess != err)
break;
- err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
- if (kIOReturnSuccess != err)
- break;
- err = IOHibernatePollerIODone(vars->fileVars);
- if (kIOReturnSuccess != err)
- break;
+ err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
}
while (false);
clock_get_uptime(&endTime);
+
+ IOService::getPMRootDomain()->pmStatsRecordEvent(
+ kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
+
SUB_ABSOLUTETIME(&endTime, &allTime);
absolutetime_to_nanoseconds(endTime, &nsec);
- HIBLOG("all time: %qd ms, ",
- nsec / 1000000ULL);
+ HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
absolutetime_to_nanoseconds(compTime, &nsec);
- HIBLOG("comp time: %qd ms, ",
- nsec / 1000000ULL);
-
- absolutetime_to_nanoseconds(decoTime, &nsec);
- HIBLOG("deco time: %qd ms, ",
- nsec / 1000000ULL);
-
- HIBLOG("\nimage %qd, uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
- header->imageSize,
+ HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
+ compBytes,
+ nsec / 1000000ULL,
+ nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+ absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
+ HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
+ vars->fileVars->cryptBytes,
+ nsec / 1000000ULL,
+ nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+ HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
+ header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
uncompressedSize, atop_32(uncompressedSize), compressedSize,
- (int) ((compressedSize * 100ULL) / uncompressedSize),
+ uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
sum1, sum2);
+ HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
+ svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
+
if (pollerOpen)
- IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
+ IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted );
+
+ if (vars->consoleMapping)
+ ProgressUpdate(gIOHibernateGraphicsInfo,
+ vars->consoleMapping, 0, kIOHibernateProgressCount);
HIBLOG("hibernate_write_image done(%x)\n", err);
// should we come back via regular wake, set the state in memory.
gIOHibernateState = kIOHibernateStateInactive;
- if ((kIOReturnSuccess == err) && !(kIOHibernateModeSleep & gIOHibernateMode))
- return (true /* power down */ );
- else
- return (false /* sleep */ );
-}
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-DECLARE_IOHIBERNATEPROGRESSALPHA
-
-static void
-ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
-{
- uint32_t rowBytes, pixelShift;
- uint32_t x, y;
- int32_t blob, lastBlob;
- uint32_t alpha, in, color, result;
- uint8_t * out;
- uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
-
- pixelShift = display->depth >> 4;
- if (pixelShift < 1)
- return;
-
- rowBytes = display->rowBytes;
-
- screen += ((display->width
- - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
- + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
-
- lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
-
- screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
+ KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END,
+ wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted, 0, 0);
- for (y = 0; y < kIOHibernateProgressHeight; y++)
+ if (kIOReturnSuccess == err)
{
- out = screen + y * rowBytes;
- for (blob = firstBlob; blob <= lastBlob; blob++)
- {
- color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
- for (x = 0; x < kIOHibernateProgressWidth; x++)
- {
- alpha = gIOHibernateProgressAlpha[y][x];
- result = color;
- if (alpha)
- {
- if (0xff != alpha)
- {
- in = display->progressSaveUnder[blob][saveindex[blob]++];
- result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
- }
- if (1 == pixelShift)
- {
- result >>= 3;
- *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
- }
- else
- *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
- }
- out += (1 << pixelShift);
- }
- out += (kIOHibernateProgressSpacing << pixelShift);
- }
+ if (kIOHibernateModeSleep & gIOHibernateMode)
+ {
+ return (kIOHibernatePostWriteSleep);
+ }
+ else if(kIOHibernateModeRestart & gIOHibernateMode)
+ {
+ return (kIOHibernatePostWriteRestart);
+ }
+ else
+ {
+ /* by default, power down */
+ return (kIOHibernatePostWriteHalt);
+ }
+ }
+ else if (kIOReturnAborted == err)
+ {
+ return (kIOHibernatePostWriteWake);
+ }
+ else
+ {
+ /* on error, sleep */
+ return (kIOHibernatePostWriteSleep);
}
}
IOReturn err;
uint32_t sum;
uint32_t pagesDone;
+ uint32_t pagesRead = 0;
+ AbsoluteTime startTime, compTime;
AbsoluteTime allTime, endTime;
- uint64_t nsec;
+ AbsoluteTime startIOTime, endIOTime;
+ uint64_t nsec, nsecIO;
+ uint64_t compBytes;
uint32_t lastProgressStamp = 0;
uint32_t progressStamp;
- uint64_t progressZeroPosition = 0;
- uint32_t blob, lastBlob = (uint32_t) -1L;
- hibernate_cryptvars_t * cryptvars = 0;
+ IOPolledFileCryptVars * cryptvars = 0;
IOHibernateVars * vars = &gIOHibernateVars;
+ bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
- if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
+ if (!vars->fileVars || !vars->fileVars->pollers)
return;
- if ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)
- hibernate_page_list_discard(vars->page_list);
-
-
sum = gIOHibernateCurrentHeader->actualImage1Sum;
pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
- HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, image1Size %qx, conflictCount %d, nextFree %x\n",
- gIOHibernateState, pagesDone, sum, gIOHibernateCurrentHeader->image1Size,
- gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
-
if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
{
HIBLOG("regular wake\n");
HIBPRINT("diag %x %x %x %x\n",
gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
- gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
+ gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
+
+#define t40ms(x) (tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)
+#define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
+ tStat(booterStart, booterStart);
+ gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart;
+ tStat(booterDuration0, booterTime0);
+ tStat(booterDuration1, booterTime1);
+ tStat(booterDuration2, booterTime2);
+ tStat(booterDuration, booterTime);
+ tStat(booterConnectDisplayDuration, connectDisplayTime);
+ tStat(booterSplashDuration, splashTime);
+ tStat(trampolineDuration, trampolineTime);
+
+ gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size;
+ gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize;
+ gIOHibernateStats->image1Pages = pagesDone;
+
+ HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
+ gIOHibernateStats->booterStart,
+ gIOHibernateStats->smcStart,
+ gIOHibernateStats->booterDuration0,
+ gIOHibernateStats->booterDuration1,
+ gIOHibernateStats->booterDuration2,
+ gIOHibernateStats->booterDuration,
+ gIOHibernateStats->booterConnectDisplayDuration,
+ gIOHibernateStats->booterSplashDuration,
+ gIOHibernateStats->trampolineDuration);
+
+ HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
+ gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
+ gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
+
+ if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
+ && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)))
+ {
+ hibernate_page_list_discard(vars->page_list);
+ }
+
+ cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
+
+ if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount)
+ panic("handoff overflow");
+
+ IOHibernateHandoff * handoff;
+ bool done = false;
+ bool foundCryptData = false;
+
+ for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
+ !done;
+ handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
+ {
+// HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
+ uint8_t * data = &handoff->data[0];
+ switch (handoff->type)
+ {
+ case kIOHibernateHandoffTypeEnd:
+ done = true;
+ break;
+
+ case kIOHibernateHandoffTypeGraphicsInfo:
+ if (handoff->bytecount == sizeof(*gIOHibernateGraphicsInfo))
+ {
+ bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
+ }
+ break;
+
+ case kIOHibernateHandoffTypeCryptVars:
+ if (cryptvars)
+ {
+ hibernate_cryptwakevars_t *
+ wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
+ bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
+ }
+ foundCryptData = true;
+ bzero(data, handoff->bytecount);
+ break;
+
+ case kIOHibernateHandoffTypeMemoryMap:
+
+ clock_get_uptime(&allTime);
- HIBPRINT("video %x %d %d %d\n",
+ hibernate_newruntime_map(data, handoff->bytecount,
+ gIOHibernateCurrentHeader->systemTableOffset);
+
+ clock_get_uptime(&endTime);
+
+ SUB_ABSOLUTETIME(&endTime, &allTime);
+ absolutetime_to_nanoseconds(endTime, &nsec);
+
+ HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
+
+ break;
+
+ case kIOHibernateHandoffTypeDeviceTree:
+ {
+// DTEntry chosen = NULL;
+// HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen));
+ }
+ break;
+
+ default:
+ done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
+ break;
+ }
+ }
+ if (cryptvars && !foundCryptData)
+ panic("hibernate handoff");
+
+ HIBPRINT("video 0x%llx %d %d %d status %x\n",
gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
- gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height);
+ gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)
{
vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
* gIOHibernateGraphicsInfo->rowBytes);
- IOMapPages(kernel_map,
- vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
- vars->videoMapSize, kIOMapInhibitCache );
+ if (vars->videoMapSize > vars->videoAllocSize) vars->videoMapSize = 0;
+ else
+ {
+ IOMapPages(kernel_map,
+ vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
+ vars->videoMapSize, kIOMapInhibitCache );
+ }
}
- uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();;
- uint32_t decoOffset;
+ if (vars->videoMapSize)
+ ProgressUpdate(gIOHibernateGraphicsInfo,
+ (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
+
+ uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
+ uint8_t * compressed = src + page_size;
+ uint8_t * scratch = compressed + page_size;
+ uint32_t decoOffset;
clock_get_uptime(&allTime);
+ AbsoluteTime_to_scalar(&compTime) = 0;
+ compBytes = 0;
- HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
- err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0);
- HIBLOG("IOHibernatePollerOpen(%x)\n", err);
-
- if (gIOHibernateCurrentHeader->previewSize)
- progressZeroPosition = gIOHibernateCurrentHeader->previewSize
- + gIOHibernateCurrentHeader->fileExtentMapSize
- - sizeof(gIOHibernateCurrentHeader->fileExtentMap)
- + ptoa_64(gIOHibernateCurrentHeader->restore1PageCount);
+ HIBLOG("IOPolledFilePollersOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
+ err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledAfterSleepState, false);
+ clock_get_uptime(&startIOTime);
+ endTime = startIOTime;
+ SUB_ABSOLUTETIME(&endTime, &allTime);
+ absolutetime_to_nanoseconds(endTime, &nsec);
+ HIBLOG("IOPolledFilePollersOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
- if (vars->videoMapping)
- {
- lastBlob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
- / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
- ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, 0, lastBlob);
- }
-
- cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
- if (kIOHibernateModeEncrypt & gIOHibernateMode)
- {
- cryptvars = &gIOHibernateCryptWakeContext;
- bcopy(&gIOHibernateCryptWakeVars->aes_iv[0],
- &cryptvars->aes_iv[0],
- sizeof(cryptvars->aes_iv));
- }
-
// kick off the read ahead
- vars->fileVars->io = false;
vars->fileVars->bufferHalf = 0;
vars->fileVars->bufferLimit = 0;
vars->fileVars->lastRead = 0;
+ vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
+ vars->fileVars->cryptBytes = 0;
+ AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
- IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
+ err = IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
// --
uint32_t * header = (uint32_t *) src;
sum = 0;
- do
+ while (kIOReturnSuccess == err)
{
unsigned int count;
unsigned int page;
uint32_t tag;
vm_offset_t ppnum, compressedSize;
- IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
+ err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
+ if (kIOReturnSuccess != err)
+ break;
ppnum = header[0];
count = header[1];
for (page = 0; page < count; page++)
{
- IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
+ err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
+ if (kIOReturnSuccess != err)
+ break;
compressedSize = kIOHibernateTagLength & tag;
- if (!compressedSize)
+ if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength))
{
- ppnum++;
- pagesDone++;
- continue;
+ err = kIOReturnIPCError;
+ break;
}
- IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
-
- if (compressedSize != page_size)
+ err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
+ if (kIOReturnSuccess != err) break;
+
+ if (compressedSize < page_size)
{
decoOffset = page_size;
- WKdm_decompress((WK_word*) src, (WK_word*) (src + decoOffset), PAGE_SIZE_IN_WORDS);
- }
- else
- decoOffset = 0;
+ clock_get_uptime(&startTime);
- sum += hibernate_sum((src + decoOffset), page_size);
+ if (compressedSize == 4) {
+ int i;
+ uint32_t *s, *d;
+
+ s = (uint32_t *)src;
+ d = (uint32_t *)(uintptr_t)compressed;
+
+ for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++)
+ *d++ = *s;
+ }
+ else
+ WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, compressedSize);
+ clock_get_uptime(&endTime);
+ ADD_ABSOLUTETIME(&compTime, &endTime);
+ SUB_ABSOLUTETIME(&compTime, &startTime);
+ compBytes += page_size;
+ }
+ else decoOffset = 0;
+ sum += hibernate_sum_page((src + decoOffset), ppnum);
err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
if (err)
- HIBLOG("IOMemoryDescriptorReadToPhysical [%d] %x\n", ppnum, err);
+ {
+ HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
+ break;
+ }
ppnum++;
pagesDone++;
-
- if (vars->videoMapping && (0 == (255 & pagesDone)))
- {
- blob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
- / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
- if (blob != lastBlob)
- {
- ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, lastBlob, blob);
- lastBlob = blob;
- }
- }
+ pagesRead++;
if (0 == (8191 & pagesDone))
{
}
}
}
- while (true);
+ if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages))
+ err = kIOReturnLockedRead;
- gIOHibernateCurrentHeader->actualImage2Sum = sum;
+ if (kIOReturnSuccess != err)
+ panic("Hibernate restore error %x", err);
- if (vars->fileVars->io)
- (void) IOHibernatePollerIODone(vars->fileVars);
+ gIOHibernateCurrentHeader->actualImage2Sum = sum;
+ gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
- err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
+ clock_get_uptime(&endIOTime);
- if (vars->videoMapping)
- ProgressUpdate(gIOHibernateGraphicsInfo,
- (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
+ err = IOPolledFilePollersClose(vars->fileVars, kIOPolledAfterSleepState);
clock_get_uptime(&endTime);
+
+ IOService::getPMRootDomain()->pmStatsRecordEvent(
+ kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
+ IOService::getPMRootDomain()->pmStatsRecordEvent(
+ kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
+
SUB_ABSOLUTETIME(&endTime, &allTime);
absolutetime_to_nanoseconds(endTime, &nsec);
- HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %qd ms\n",
- pagesDone, sum, nsec / 1000000ULL);
+ SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
+ absolutetime_to_nanoseconds(endIOTime, &nsecIO);
+
+ gIOHibernateStats->kernelImageReadDuration = nsec / 1000000ULL;
+ gIOHibernateStats->imagePages = pagesDone;
+
+ HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
+ pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
+ nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
+
+ absolutetime_to_nanoseconds(compTime, &nsec);
+ HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
+ compBytes,
+ nsec / 1000000ULL,
+ nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+ absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
+ HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
+ vars->fileVars->cryptBytes,
+ nsec / 1000000ULL,
+ nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+ KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_NONE, pagesRead, pagesDone, 0, 0, 0);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+void IOHibernateSetWakeCapabilities(uint32_t capability)
+{
+ if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
+ {
+ gIOHibernateStats->wakeCapability = capability;
+
+ if (kIOPMSystemCapabilityGraphics & capability)
+ {
+ vm_compressor_do_warmup();
+ }
+ }
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+void IOHibernateSystemRestart(void)
+{
+ static uint8_t noteStore[32] __attribute__((aligned(32)));
+ IORegistryEntry * regEntry;
+ const OSSymbol * sym;
+ OSData * noteProp;
+ OSData * data;
+ uintptr_t * smcVars;
+ uint8_t * smcBytes;
+ size_t len;
+ addr64_t element;
+
+ data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
+ if (!data) return;
+
+ smcVars = (typeof(smcVars)) data->getBytesNoCopy();
+ smcBytes = (typeof(smcBytes)) smcVars[1];
+ len = smcVars[0];
+ if (len > sizeof(noteStore)) len = sizeof(noteStore);
+ noteProp = OSData::withCapacity(3 * sizeof(element));
+ if (!noteProp) return;
+ element = len;
+ noteProp->appendBytes(&element, sizeof(element));
+ element = crc32(0, smcBytes, len);
+ noteProp->appendBytes(&element, sizeof(element));
+
+ bcopy(smcBytes, noteStore, len);
+ element = (addr64_t) ¬eStore[0];
+ element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
+ noteProp->appendBytes(&element, sizeof(element));
+
+ if (!gIOOptionsEntry)
+ {
+ regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
+ gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
+ if (regEntry && !gIOOptionsEntry)
+ regEntry->release();
+ }
+
+ sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
+ if (gIOOptionsEntry && sym) gIOOptionsEntry->setProperty(sym, noteProp);
+ if (noteProp) noteProp->release();
+ if (sym) sym->release();
+}
+
+
+
projects / apple / xnu.git / blobdiff