X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/8a3053a07cee346dca737a5670e546fd26a7c9d6..4d15aeb193b2c68f1d38666c317f8d3734f5f083:/iokit/Kernel/IOHibernateIO.cpp?ds=inline diff --git a/iokit/Kernel/IOHibernateIO.cpp b/iokit/Kernel/IOHibernateIO.cpp index 851adac60..953c6d445 100644 --- a/iokit/Kernel/IOHibernateIO.cpp +++ b/iokit/Kernel/IOHibernateIO.cpp @@ -1,8 +1,8 @@ /* - * Copyright (c) 2004-2008 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2004-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,11 +22,10 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ - /* Sleep: @@ -160,18 +159,21 @@ to restrict I/O ops. #include // (FWRITE, ...) #include #include +#include #include #include #include #include "IOHibernateInternal.h" #include +#include #include "IOKitKernelInternal.h" #include #include #include #include +#include extern "C" addr64_t kvtophys(vm_offset_t va); extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va); @@ -179,7 +181,7 @@ extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va); /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #define DISABLE_TRIM 0 -#define TRIM_DELAY 5000 +#define TRIM_DELAY 25000 extern unsigned int save_kdebug_enable; extern uint32_t gIOHibernateState; @@ -189,25 +191,33 @@ static char gIOHibernateFilename[MAXPATHLEN+1]; static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent) uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms) static uint64_t gIOHibernateCompression = 0x80; // default compression 50% +boolean_t gIOHibernateStandbyDisabled; static IODTNVRAM * gIOOptionsEntry; static IORegistryEntry * gIOChosenEntry; + +static const OSSymbol * gIOHibernateBootImageKey; + #if defined(__i386__) || defined(__x86_64__) -static const OSSymbol * gIOCreateEFIDevicePathSymbol; + static const OSSymbol * gIOHibernateRTCVariablesKey; static const OSSymbol * gIOHibernateBoot0082Key; static const OSSymbol * gIOHibernateBootNextKey; static OSData * gIOHibernateBoot0082Data; static OSData * gIOHibernateBootNextData; static OSObject * gIOHibernateBootNextSave; -#endif + +static IOPolledFileIOVars * gDebugImageFileVars; +static IOLock * gDebugImageLock; + +#endif /* defined(__i386__) || defined(__x86_64__) */ static IOLock * gFSLock; static uint32_t gFSState; +static thread_call_t gIOHibernateTrimCalloutEntry; static IOPolledFileIOVars gFileVars; static IOHibernateVars gIOHibernateVars; -static struct kern_direct_file_io_ref_t * gIOHibernateFileRef; -static hibernate_cryptvars_t gIOHibernateCryptWakeContext; +static IOPolledFileCryptVars gIOHibernateCryptWakeContext; static hibernate_graphics_t _hibernateGraphics; static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics; static hibernate_statistics_t _hibernateStats; @@ -215,31 +225,22 @@ static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats; enum { - kFSIdle = 0, - kFSOpening = 2, - kFSOpened = 3, - kFSTimedOut = 4, + kFSIdle = 0, + kFSOpening = 2, + kFSOpened = 3, + kFSTimedOut = 4, + kFSTrimDelay = 5 }; static IOReturn IOHibernateDone(IOHibernateVars * vars); +static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature); +static void IOSetBootImageNVRAM(OSData * data); /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -enum { kXPRamAudioVolume = 8 }; enum { kDefaultIOSize = 128 * 1024 }; enum { kVideoMapSize = 80 * 1024 * 1024 }; -#ifndef kIOMediaPreferredBlockSizeKey -#define kIOMediaPreferredBlockSizeKey "Preferred Block Size" -#endif - -#ifndef kIOBootPathKey -#define kIOBootPathKey "bootpath" -#endif -#ifndef kIOSelectedBootDeviceKey -#define kIOSelectedBootDeviceKey "boot-device" -#endif - /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ // copy from phys addr to MD @@ -368,783 +369,18 @@ hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage) count = hibernate_page_bitmap_count(bitmap, TRUE, page); if (!count) break; - page += count; - if (page <= bitmap->last_page) - break; - } - - *pPage = page; - if (bitmap) - count = hibernate_page_bitmap_count(bitmap, FALSE, page); - else - count = 0; - - return (count); -} - -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -static IOReturn -IOHibernatePollerProbe(IOPolledFileIOVars * vars, IOService * target) -{ - IOReturn err = kIOReturnError; - int32_t idx; - IOPolledInterface * poller; - - for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--) - { - poller = (IOPolledInterface *) vars->pollers->getObject(idx); - err = poller->probe(target); - if (err) - { - HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err); - break; - } - } - - return (err); -} - -static IOReturn -IOHibernatePollerOpen(IOPolledFileIOVars * vars, uint32_t state, IOMemoryDescriptor * md) -{ - IOReturn err = kIOReturnError; - int32_t idx; - IOPolledInterface * poller; - - for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--) - { - poller = (IOPolledInterface *) vars->pollers->getObject(idx); - err = poller->open(state, md); - if (err) - { - HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err); - break; - } - } - - return (err); -} - -static IOReturn -IOHibernatePollerClose(IOPolledFileIOVars * vars, uint32_t state) -{ - IOReturn err = kIOReturnError; - int32_t idx; - IOPolledInterface * poller; - - for (idx = 0; - (poller = (IOPolledInterface *) vars->pollers->getObject(idx)); - idx++) - { - err = poller->close(state); - if (err) - HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err); - } - - return (err); -} - -static void -IOHibernatePollerIOComplete(void * target, - void * parameter, - IOReturn status, - UInt64 actualByteCount) -{ - IOPolledFileIOVars * vars = (IOPolledFileIOVars *) parameter; - - vars->ioStatus = status; -} - -static IOReturn -IOHibernatePollerIO(IOPolledFileIOVars * vars, - uint32_t operation, uint32_t bufferOffset, - uint64_t deviceOffset, uint64_t length) -{ - - IOReturn err = kIOReturnError; - IOPolledInterface * poller; - IOPolledCompletion completion; - - completion.target = 0; - completion.action = &IOHibernatePollerIOComplete; - completion.parameter = vars; - - vars->ioStatus = -1; - - poller = (IOPolledInterface *) vars->pollers->getObject(0); - err = poller->startIO(operation, bufferOffset, deviceOffset + vars->block0, length, completion); - if (err) - HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err); - - return (err); -} - -static IOReturn -IOHibernatePollerIODone(IOPolledFileIOVars * vars, bool abortable) -{ - IOReturn err = kIOReturnSuccess; - int32_t idx = 0; - IOPolledInterface * poller; - - while (-1 == vars->ioStatus) - { - for (idx = 0; - (poller = (IOPolledInterface *) vars->pollers->getObject(idx)); - idx++) - { - IOReturn newErr; - newErr = poller->checkForWork(); - if ((newErr == kIOReturnAborted) && !abortable) - newErr = kIOReturnSuccess; - if (kIOReturnSuccess == err) - err = newErr; - } - } - - if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort()) - { - err = kIOReturnAborted; - HIBLOG("IOPolledInterface::checkForWork sw abort\n"); - } - - if (err) - { - HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err); - } - else - { - err = vars->ioStatus; - if (kIOReturnSuccess != err) - HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err); - } - - return (err); -} - -IOReturn -IOPolledInterface::checkAllForWork(void) -{ - IOReturn err = kIOReturnNotReady; - int32_t idx; - IOPolledInterface * poller; - - IOHibernateVars * vars = &gIOHibernateVars; - - if (!vars->fileVars || !vars->fileVars->pollers) - return (err); - - for (idx = 0; - (poller = (IOPolledInterface *) vars->fileVars->pollers->getObject(idx)); - idx++) - { - err = poller->checkForWork(); - if (err) - HIBLOG("IOPolledInterface::checkAllForWork[%d] 0x%x\n", idx, err); - } - - return (err); -} - -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -struct _OpenFileContext -{ - OSData * extents; - uint64_t size; -}; - -static void -file_extent_callback(void * ref, uint64_t start, uint64_t length) -{ - _OpenFileContext * ctx = (_OpenFileContext *) ref; - IOPolledFileExtent extent; - - extent.start = start; - extent.length = length; - - HIBLOG("[0x%qx, 0x%qx]\n", start, length); - - ctx->extents->appendBytes(&extent, sizeof(extent)); - ctx->size += length; -} - -static IOService * -IOCopyMediaForDev(dev_t device) -{ - OSDictionary * matching; - OSNumber * num; - OSIterator * iter; - IOService * result = 0; - - matching = IOService::serviceMatching("IOMedia"); - if (!matching) - return (0); - do - { - num = OSNumber::withNumber(major(device), 32); - if (!num) - break; - matching->setObject(kIOBSDMajorKey, num); - num->release(); - num = OSNumber::withNumber(minor(device), 32); - if (!num) - break; - matching->setObject(kIOBSDMinorKey, num); - num->release(); - if (!num) - break; - iter = IOService::getMatchingServices(matching); - if (iter) - { - result = (IOService *) iter->getNextObject(); - result->retain(); - iter->release(); - } - } - while (false); - matching->release(); - - return (result); -} - -IOReturn -IOPolledFileOpen( const char * filename, uint64_t setFileSize, - IOBufferMemoryDescriptor * ioBuffer, - IOPolledFileIOVars ** fileVars, OSData ** fileExtents, - OSData ** imagePath, uint8_t * volumeCryptKey) -{ - IOReturn err = kIOReturnSuccess; - IOPolledFileIOVars * vars; - _OpenFileContext ctx; - OSData * extentsData; - OSNumber * num; - IOService * part = 0; - OSString * keyUUID = 0; - OSString * keyStoreUUID = 0; - dev_t block_dev; - dev_t hibernate_image_dev; - uint64_t maxiobytes; - AbsoluteTime startTime, endTime; - uint64_t nsec; - - vars = IONew(IOPolledFileIOVars, 1); - if (!vars) return (kIOReturnNoMemory); - bzero(vars, sizeof(*vars)); - - do - { - vars->io = false; - vars->buffer = (uint8_t *) ioBuffer->getBytesNoCopy(); - vars->bufferHalf = 0; - vars->bufferOffset = 0; - vars->bufferSize = ioBuffer->getLength() >> 1; - - extentsData = OSData::withCapacity(32); - ctx.extents = extentsData; - ctx.size = 0; - clock_get_uptime(&startTime); - vars->fileRef = kern_open_file_for_direct_io(filename, - &file_extent_callback, &ctx, - setFileSize, - // write file: - 0, (caddr_t) gIOHibernateCurrentHeader, - sizeof(IOHibernateImageHeader), - // results - &block_dev, - &hibernate_image_dev, - &vars->block0, - &maxiobytes, - &vars->flags); -#if 0 - uint32_t msDelay = (131071 & random()); - HIBLOG("sleep %d\n", msDelay); - IOSleep(msDelay); -#endif - clock_get_uptime(&endTime); - SUB_ABSOLUTETIME(&endTime, &startTime); - absolutetime_to_nanoseconds(endTime, &nsec); - - if (!vars->fileRef) err = kIOReturnNoSpace; - - IOLockLock(gFSLock); - if (kFSOpening != gFSState) err = kIOReturnTimeout; - IOLockUnlock(gFSLock); - - HIBLOG("kern_open_file_for_direct_io(%d) took %qd ms\n", err, nsec / 1000000ULL); - if (kIOReturnSuccess != err) break; - - if (kIOHibernateModeSSDInvert & gIOHibernateMode) - vars->flags ^= kIOHibernateOptionSSD; - - HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx ssd %d\n", filename, ctx.size, - vars->block0, maxiobytes, kIOHibernateOptionSSD & vars->flags); - if (ctx.size < 1*1024*1024) // check against image size estimate! - { - err = kIOReturnNoSpace; - break; - } - - vars->fileSize = ctx.size; - if (maxiobytes < vars->bufferSize) vars->bufferSize = maxiobytes; - - vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy(); - - part = IOCopyMediaForDev(block_dev); - if (!part) - { - err = kIOReturnNotFound; - break; - } - err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false, - (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL); - if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID) - { -// IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy()); - uuid_t volumeKeyUUID; - aks_volume_key_t vek; - static IOService * sKeyStore; - static const OSSymbol * sAKSGetKey; - - if (!sAKSGetKey) - sAKSGetKey = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY); - if (!sKeyStore) - sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane); - if (sKeyStore) - err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID); - else - err = kIOReturnNoResources; - if (kIOReturnSuccess == err) - err = sKeyStore->callPlatformFunction(sAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL); - if (kIOReturnSuccess != err) - IOLog("volume key err 0x%x\n", err); - else - { - size_t bytes = (kIOHibernateAESKeySize / 8); - if (vek.key.keybytecount < bytes) - bytes = vek.key.keybytecount; - bcopy(&vek.key.keybytes[0], volumeCryptKey, bytes); - } - bzero(&vek, sizeof(vek)); - } - part->release(); - - part = IOCopyMediaForDev(hibernate_image_dev); - if (!part) - { - err = kIOReturnNotFound; - break; - } - - IORegistryEntry * next; - IORegistryEntry * child; - IOService * service; - OSData * data; - - vars->pollers = OSArray::withCapacity(4); - if (!vars->pollers) - { - err = kIOReturnNoMemory; - break; - } - - vars->blockSize = 512; - next = part; - do - { - IOPolledInterface * poller; - OSObject * obj; - - obj = next->getProperty(kIOPolledInterfaceSupportKey); - if (kOSBooleanFalse == obj) - { - vars->pollers->flushCollection(); - break; - } - else if ((poller = OSDynamicCast(IOPolledInterface, obj))) - vars->pollers->setObject(poller); - - if ((service = OSDynamicCast(IOService, next)) - && service->getDeviceMemory() - && !vars->pollers->getCount()) break; - - if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey)))) - vars->blockSize = num->unsigned32BitValue(); - child = next; - } - while ((next = child->getParentEntry(gIOServicePlane)) - && child->isParent(next, gIOServicePlane, true)); - - if (vars->blockSize < 4096) vars->blockSize = 4096; - - HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n", - major(hibernate_image_dev), minor(hibernate_image_dev), (long)vars->blockSize, - vars->pollers->getCount()); - - if (!vars->pollers->getCount()) - { - err = kIOReturnUnsupported; - continue; - } - if (vars->blockSize < sizeof(IOHibernateImageHeader)) - { - err = kIOReturnError; - continue; - } - - err = IOHibernatePollerProbe(vars, (IOService *) part); - if (kIOReturnSuccess != err) break; - - err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer); - if (kIOReturnSuccess != err) break; - - vars->media = part; - next = part; - while (next) - { - next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue); - next = next->getParentEntry(gIOServicePlane); - } - - *fileVars = vars; - *fileExtents = extentsData; - - // make imagePath - - if ((extentsData->getLength() >= sizeof(IOPolledFileExtent))) - { - char str2[24 + sizeof(uuid_string_t) + 2]; - -#if defined(__i386__) || defined(__x86_64__) - if (!gIOCreateEFIDevicePathSymbol) - gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath"); - - if (keyUUID) - snprintf(str2, sizeof(str2), "%qx:%s", - vars->extentMap[0].start, keyUUID->getCStringNoCopy()); - else - snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start); - - err = IOService::getPlatform()->callPlatformFunction( - gIOCreateEFIDevicePathSymbol, false, - (void *) part, (void *) str2, - (void *) (uintptr_t) true, (void *) &data); -#else - char str1[256]; - int len = sizeof(str1); - - if (!part->getPath(str1, &len, gIODTPlane)) - err = kIOReturnNotFound; - else - { - snprintf(str2, sizeof(str2), ",%qx", vars->extentMap[0].start); - // (strip the plane name) - char * tail = strchr(str1, ':'); - if (!tail) - tail = str1 - 1; - data = OSData::withBytes(tail + 1, strlen(tail + 1)); - data->appendBytes(str2, strlen(str2)); - } -#endif - if (kIOReturnSuccess == err) - *imagePath = data; - else - HIBLOG("error 0x%x getting path\n", err); - } - } - while (false); - - if (kIOReturnSuccess != err) - { - HIBLOG("error 0x%x opening hibernation file\n", err); - if (vars->fileRef) - { - kern_close_file_for_direct_io(vars->fileRef, 0, 0, 0, 0, 0); - vars->fileRef = NULL; - } - } - - if (part) - part->release(); - - return (err); -} - -IOReturn -IOPolledFileClose( IOPolledFileIOVars * vars ) -{ - if (vars->pollers) - { - IOHibernatePollerClose(vars, kIOPolledPostflightState); - vars->pollers->release(); - } - - bzero(vars, sizeof(IOPolledFileIOVars)); - - return (kIOReturnSuccess); -} - -static IOReturn -IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position) -{ - IOPolledFileExtent * extentMap; - - extentMap = vars->extentMap; - - vars->position = position; - - while (position >= extentMap->length) - { - position -= extentMap->length; - extentMap++; - } - - vars->currentExtent = extentMap; - vars->extentRemaining = extentMap->length - position; - vars->extentPosition = vars->position - position; - - if (vars->bufferSize <= vars->extentRemaining) - vars->bufferLimit = vars->bufferSize; - else - vars->bufferLimit = vars->extentRemaining; - - return (kIOReturnSuccess); -} - -static IOReturn -IOPolledFileWrite(IOPolledFileIOVars * vars, - const uint8_t * bytes, IOByteCount size, - hibernate_cryptvars_t * cryptvars) -{ - IOReturn err = kIOReturnSuccess; - IOByteCount copy; - bool flush = false; - - do - { - if (!bytes && !size) - { - // seek to end of block & flush - size = vars->position & (vars->blockSize - 1); - if (size) - size = vars->blockSize - size; - flush = true; - // use some garbage for the fill - bytes = vars->buffer + vars->bufferOffset; - } - - copy = vars->bufferLimit - vars->bufferOffset; - if (copy > size) - copy = size; - else - flush = true; - - if (bytes) - { - bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy); - bytes += copy; - } - else - bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy); - - size -= copy; - vars->bufferOffset += copy; - vars->position += copy; - - if (flush && vars->bufferOffset) - { - uint64_t offset = (vars->position - vars->bufferOffset - - vars->extentPosition + vars->currentExtent->start); - uint32_t length = (vars->bufferOffset); - -#if CRYPTO - if (cryptvars && vars->encryptStart - && (vars->position > vars->encryptStart) - && ((vars->position - length) < vars->encryptEnd)) - { - AbsoluteTime startTime, endTime; - - uint64_t encryptLen, encryptStart; - encryptLen = vars->position - vars->encryptStart; - if (encryptLen > length) - encryptLen = length; - encryptStart = length - encryptLen; - if (vars->position > vars->encryptEnd) - encryptLen -= (vars->position - vars->encryptEnd); - - clock_get_uptime(&startTime); - - // encrypt the buffer - aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart, - &cryptvars->aes_iv[0], - encryptLen / AES_BLOCK_SIZE, - vars->buffer + vars->bufferHalf + encryptStart, - &cryptvars->ctx.encrypt); - - clock_get_uptime(&endTime); - ADD_ABSOLUTETIME(&vars->cryptTime, &endTime); - SUB_ABSOLUTETIME(&vars->cryptTime, &startTime); - vars->cryptBytes += encryptLen; - - // save initial vector for following encrypts - bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE, - &cryptvars->aes_iv[0], - AES_BLOCK_SIZE); - } -#endif /* CRYPTO */ - - if (vars->io) - { - err = IOHibernatePollerIODone(vars, true); - if (kIOReturnSuccess != err) - break; - } - -if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position); -//if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length); - - err = IOHibernatePollerIO(vars, kIOPolledWrite, vars->bufferHalf, offset, length); - if (kIOReturnSuccess != err) - break; - vars->io = true; - - vars->extentRemaining -= vars->bufferOffset; - if (!vars->extentRemaining) - { - vars->currentExtent++; - vars->extentRemaining = vars->currentExtent->length; - vars->extentPosition = vars->position; - if (!vars->extentRemaining) - { - err = kIOReturnOverrun; - break; - } - } - - vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize; - vars->bufferOffset = 0; - if (vars->bufferSize <= vars->extentRemaining) - vars->bufferLimit = vars->bufferSize; - else - vars->bufferLimit = vars->extentRemaining; - - flush = false; - } - } - while (size); - - return (err); -} - -static IOReturn -IOPolledFileRead(IOPolledFileIOVars * vars, - uint8_t * bytes, IOByteCount size, - hibernate_cryptvars_t * cryptvars) -{ - IOReturn err = kIOReturnSuccess; - IOByteCount copy; - -// bytesWritten += size; - - do - { - copy = vars->bufferLimit - vars->bufferOffset; - if (copy > size) - copy = size; - - if (bytes) - { - bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy); - bytes += copy; - } - size -= copy; - vars->bufferOffset += copy; -// vars->position += copy; - - if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd)) - { - if (vars->io) - { - err = IOHibernatePollerIODone(vars, false); - if (kIOReturnSuccess != err) - break; - } - else - cryptvars = 0; - -if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position); - - vars->position += vars->lastRead; - vars->extentRemaining -= vars->lastRead; - vars->bufferLimit = vars->lastRead; - - if (!vars->extentRemaining) - { - vars->currentExtent++; - vars->extentRemaining = vars->currentExtent->length; - vars->extentPosition = vars->position; - if (!vars->extentRemaining) - { - err = kIOReturnOverrun; - break; - } - } - - uint64_t length; - uint64_t lastReadLength = vars->lastRead; - uint64_t offset = (vars->position - - vars->extentPosition + vars->currentExtent->start); - if (vars->extentRemaining <= vars->bufferSize) - length = vars->extentRemaining; - else - length = vars->bufferSize; - if ((length + vars->position) > vars->readEnd) - length = vars->readEnd - vars->position; - - vars->lastRead = length; - if (length) - { -//if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length); - err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length); - if (kIOReturnSuccess != err) - break; - vars->io = true; - } - - vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize; - vars->bufferOffset = 0; - -#if CRYPTO - if (cryptvars) - { - uint8_t thisVector[AES_BLOCK_SIZE]; - AbsoluteTime startTime, endTime; - - // save initial vector for following decrypts - bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE); - bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE, - &cryptvars->aes_iv[0], AES_BLOCK_SIZE); - - // decrypt the buffer - clock_get_uptime(&startTime); - - aes_decrypt_cbc(vars->buffer + vars->bufferHalf, - &thisVector[0], - lastReadLength / AES_BLOCK_SIZE, - vars->buffer + vars->bufferHalf, - &cryptvars->ctx.decrypt); - - clock_get_uptime(&endTime); - ADD_ABSOLUTETIME(&vars->cryptTime, &endTime); - SUB_ABSOLUTETIME(&vars->cryptTime, &startTime); - vars->cryptBytes += lastReadLength; - } -#endif /* CRYPTO */ - } + page += count; + if (page <= bitmap->last_page) + break; } - while (size); - return (err); + *pPage = page; + if (bitmap) + count = hibernate_page_bitmap_count(bitmap, FALSE, page); + else + count = 0; + + return (count); } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -1153,18 +389,16 @@ IOReturn IOHibernateSystemSleep(void) { IOReturn err; - OSData * data; + OSData * nvramData; OSObject * obj; OSString * str; OSNumber * num; - bool dsSSD, vmflush; + bool dsSSD, vmflush, swapPinned; IOHibernateVars * vars; + uint64_t setFileSize = 0; gIOHibernateState = kIOHibernateStateInactive; - if (!gIOChosenEntry) - gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane); - gIOHibernateDebugFlags = 0; if (kIOLogHibernate & gIOKitDebug) gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs; @@ -1196,6 +430,11 @@ IOHibernateSystemSleep(void) bzero(vars, sizeof(*vars)); IOLockLock(gFSLock); + if (!gIOHibernateTrimCalloutEntry) + { + gIOHibernateTrimCalloutEntry = thread_call_allocate(&IOHibernateSystemPostWakeTrim, &gFSLock); + } + IOHibernateSystemPostWakeTrim(NULL, NULL); if (kFSIdle != gFSState) { HIBLOG("hibernate file busy\n"); @@ -1206,17 +445,16 @@ IOHibernateSystemSleep(void) gFSState = kFSOpening; IOLockUnlock(gFSLock); + swapPinned = false; do { vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, - 2 * page_size + WKdm_SCRATCH_BUF_SIZE, page_size); - vars->ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, - 2 * kDefaultIOSize, page_size); + 2 * page_size + WKdm_SCRATCH_BUF_SIZE_INTERNAL, page_size); vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, ptoa_64(gIOHibernateHandoffPageCount), page_size); - if (!vars->srcBuffer || !vars->ioBuffer || !vars->handoffBuffer) + if (!vars->srcBuffer || !vars->handoffBuffer) { err = kIOReturnNoMemory; break; @@ -1242,13 +480,15 @@ IOHibernateSystemSleep(void) gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags; gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature; - vmflush = (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)); - uint64_t setFileSize = 0; + vmflush = ((kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey))); err = hibernate_alloc_page_lists(&vars->page_list, &vars->page_list_wired, &vars->page_list_pal); - if (KERN_SUCCESS != err) - break; + if (KERN_SUCCESS != err) break; + + err = hibernate_pin_swap(TRUE); + if (KERN_SUCCESS != err) break; + swapPinned = true; if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode)) { @@ -1263,13 +503,13 @@ IOHibernateSystemSleep(void) IOService::getPlatform()->getConsoleInfo(&consoleInfo); // estimate: 6% increase in pages compressed - // screen preview 2 images compressed 50% + // screen preview 2 images compressed 0% setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8) + vars->page_list->list_size + (consoleInfo.v_width * consoleInfo.v_height * 8); enum { setFileRound = 1024*1024ULL }; setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1)); - + HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n", pageCount, (100ULL * gIOHibernateCompression) >> 8, setFileSize, vars->fileMinSize); @@ -1281,11 +521,27 @@ IOHibernateSystemSleep(void) } } - // open & invalidate the image file + // Invalidate the image file + if (gDebugImageLock) { + IOLockLock(gDebugImageLock); + if (gDebugImageFileVars != 0) { + IOSetBootImageNVRAM(0); + IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0); + } + IOLockUnlock(gDebugImageLock); + } + + err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, 0, + gIOHibernateCurrentHeader, sizeof(gIOHibernateCurrentHeader), + &vars->fileVars, &nvramData, + &vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey)); - err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, vars->ioBuffer, - &vars->fileVars, &vars->fileExtents, &data, - &vars->volumeCryptKey[0]); + if (KERN_SUCCESS != err) + { + IOLockLock(gFSLock); + if (kFSOpening != gFSState) err = kIOReturnTimeout; + IOLockUnlock(gFSLock); + } if (KERN_SUCCESS != err) { @@ -1293,53 +549,60 @@ IOHibernateSystemSleep(void) break; } + // write extents for debug data usage in EFI + IOWriteExtentsToFile(vars->fileVars, kIOHibernateHeaderOpenSignature); + + err = IOPolledFilePollersSetup(vars->fileVars, kIOPolledPreflightState); + if (KERN_SUCCESS != err) break; + clock_get_uptime(&startTime); err = hibernate_setup(gIOHibernateCurrentHeader, - gIOHibernateFreeRatio, gIOHibernateFreeTime, vmflush, vars->page_list, vars->page_list_wired, vars->page_list_pal); clock_get_uptime(&endTime); SUB_ABSOLUTETIME(&endTime, &startTime); absolutetime_to_nanoseconds(endTime, &nsec); - HIBLOG("hibernate_setup(%d) took %qd ms\n", err, nsec / 1000000ULL); - dsSSD = ((0 != (kIOHibernateOptionSSD & vars->fileVars->flags)) + boolean_t haveSwapPin, hibFileSSD; + haveSwapPin = vm_swap_files_pinned(); + + hibFileSSD = (kIOPolledFileSSD & vars->fileVars->flags); + + HIBLOG("hibernate_setup(%d) took %qd ms, swapPin(%d) ssd(%d)\n", + err, nsec / 1000000ULL, + haveSwapPin, hibFileSSD); + if (KERN_SUCCESS != err) break; + + gIOHibernateStandbyDisabled = ((!haveSwapPin || !hibFileSSD)); + + dsSSD = ((0 != (kIOPolledFileSSD & vars->fileVars->flags)) && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey))); - if (dsSSD) - { - gIOHibernateCurrentHeader->options |= - kIOHibernateOptionSSD - | kIOHibernateOptionColor; + + if (dsSSD) gIOHibernateCurrentHeader->options |= kIOHibernateOptionSSD | kIOHibernateOptionColor; + else gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress; + #if defined(__i386__) || defined(__x86_64__) - if (!uuid_is_null(vars->volumeCryptKey) && - (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey))) - { - uintptr_t smcVars[2]; - smcVars[0] = sizeof(vars->volumeCryptKey); - smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0]; + if (!uuid_is_null(vars->volumeCryptKey) && + (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey))) + { + uintptr_t smcVars[2]; + smcVars[0] = sizeof(vars->volumeCryptKey); + smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0]; - IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars)); - bzero(smcVars, sizeof(smcVars)); - } + IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars)); + bzero(smcVars, sizeof(smcVars)); + } #endif - } - else - { - gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress; - } - if (KERN_SUCCESS != err) - break; - if (encryptedswap || !uuid_is_null(vars->volumeCryptKey)) gIOHibernateMode ^= kIOHibernateModeEncrypt; if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options) { vars->videoAllocSize = kVideoMapSize; - if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize)) + if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize, VM_KERN_MEMORY_IOKIT)) vars->videoMapping = 0; } @@ -1351,28 +614,11 @@ IOHibernateSystemSleep(void) // set nvram - IORegistryEntry * regEntry; - if (!gIOOptionsEntry) - { - regEntry = IORegistryEntry::fromPath("/options", gIODTPlane); - gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry); - if (regEntry && !gIOOptionsEntry) - regEntry->release(); - } - - if (gIOOptionsEntry) - { - const OSSymbol * sym; - - sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey); - if (sym) - { - gIOOptionsEntry->setProperty(sym, data); - sym->release(); - } - data->release(); + IOSetBootImageNVRAM(nvramData); + nvramData->release(); #if defined(__i386__) || defined(__x86_64__) + { struct AppleRTCHibernateVars { uint8_t signature[4]; @@ -1381,6 +627,7 @@ IOHibernateSystemSleep(void) uint8_t wiredCryptKey[16]; }; AppleRTCHibernateVars rtcVars; + OSData * data; rtcVars.signature[0] = 'A'; rtcVars.signature[1] = 'A'; @@ -1396,167 +643,94 @@ IOHibernateSystemSleep(void) (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1)); i++) { - if (c >= 'a') - c -= 'a' - 10; - else if (c >= 'A') - c -= 'A' - 10; - else if (c >= '0') - c -= '0'; - else - continue; + if (c >= 'a') c -= 'a' - 10; + else if (c >= 'A') c -= 'A' - 10; + else if (c >= '0') c -= '0'; + else continue; value = (value << 4) | c; - if (i & 1) - rtcVars.booterSignature[i >> 1] = value; + if (i & 1) rtcVars.booterSignature[i >> 1] = value; } } data = OSData::withBytes(&rtcVars, sizeof(rtcVars)); if (data) { - if (!gIOHibernateRTCVariablesKey) - gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey); if (gIOHibernateRTCVariablesKey) IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data); - - if( gIOOptionsEntry ) - { - if( gIOHibernateMode & kIOHibernateModeSwitch ) - { - const OSSymbol *sym; - sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSwitchVarsKey); - if( sym ) - { - gIOOptionsEntry->setProperty(sym, data); /* intentional insecure backup of rtc boot vars */ - sym->release(); - } - } - } - data->release(); } if (gIOChosenEntry) { data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey)); - if (data) - gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy()); + if (data) gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy()); + // set BootNext + if (!gIOHibernateBoot0082Data) { - // set BootNext - - if (!gIOHibernateBoot0082Data) + data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path")); + if (data) { - data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path")); - if (data) + // AppleNVRAM_EFI_LOAD_OPTION + struct { + uint32_t Attributes; + uint16_t FilePathLength; + uint16_t Desc; + } loadOptionHeader; + loadOptionHeader.Attributes = 1; + loadOptionHeader.FilePathLength = data->getLength(); + loadOptionHeader.Desc = 0; + gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength); + if (gIOHibernateBoot0082Data) { - // AppleNVRAM_EFI_LOAD_OPTION - struct { - uint32_t Attributes; - uint16_t FilePathLength; - uint16_t Desc; - } loadOptionHeader; - loadOptionHeader.Attributes = 1; - loadOptionHeader.FilePathLength = data->getLength(); - loadOptionHeader.Desc = 0; - gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength); - if (gIOHibernateBoot0082Data) - { - gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader)); - gIOHibernateBoot0082Data->appendBytes(data); - } + gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader)); + gIOHibernateBoot0082Data->appendBytes(data); } } - if (!gIOHibernateBoot0082Key) - gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082"); - if (!gIOHibernateBootNextKey) - gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext"); - if (!gIOHibernateBootNextData) - { - uint16_t bits = 0x0082; - gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits)); - } - if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData) - { - gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey); - gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data); - gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData); - } - } - } -#else /* !i386 && !x86_64 */ - if (kIOHibernateModeEncrypt & gIOHibernateMode) - { - data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey)); - sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey); - if (sym && data) - gIOOptionsEntry->setProperty(sym, data); - if (sym) - sym->release(); - if (data) - data->release(); - if (false && gIOHibernateBootSignature[0]) - { - data = OSData::withCapacity(16); - sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey); - if (sym && data) - { - char c; - uint8_t value = 0; - for (uint32_t i = 0; (c = gIOHibernateBootSignature[i]); i++) - { - if (c >= 'a') - c -= 'a' - 10; - else if (c >= 'A') - c -= 'A' - 10; - else if (c >= '0') - c -= '0'; - else - continue; - value = (value << 4) | c; - if (i & 1) - data->appendBytes(&value, sizeof(value)); - } - gIOOptionsEntry->setProperty(sym, data); - } - if (sym) - sym->release(); - if (data) - data->release(); - } - } - if (!vars->haveFastBoot) - { - // set boot volume to zero - IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform()); - if (platform && (kIOReturnSuccess == platform->readXPRAM(kXPRamAudioVolume, - &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume)))) - { - uint8_t newVolume; - newVolume = vars->saveBootAudioVolume & 0xf8; - platform->writeXPRAM(kXPRamAudioVolume, - &newVolume, sizeof(newVolume)); - } + } + if (!gIOHibernateBootNextData) + { + uint16_t bits = 0x0082; + gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits)); + } + if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData) + { + gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey); + gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data); + gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData); + } + // BootNext } -#endif /* !i386 && !x86_64 */ } - // -- - +#endif /* !i386 && !x86_64 */ } while (false); + if (swapPinned) hibernate_pin_swap(FALSE); + IOLockLock(gFSLock); - if ((kIOReturnSuccess == err) && (kFSOpening == gFSState)) + if ((kIOReturnSuccess == err) && (kFSOpening != gFSState)) + { + HIBLOG("hibernate file close due timeout\n"); + err = kIOReturnTimeout; + } + if (kIOReturnSuccess == err) { gFSState = kFSOpened; gIOHibernateVars = *vars; gFileVars = *vars->fileVars; + gFileVars.allocated = false; gIOHibernateVars.fileVars = &gFileVars; - gIOHibernateFileRef = gFileVars.fileRef; gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature; gIOHibernateState = kIOHibernateStateHibernating; } else { - HIBLOG("hibernate file close due timeout\n"); - if (vars->fileVars && vars->fileVars->fileRef) kern_close_file_for_direct_io(vars->fileVars->fileRef, 0, 0, 0, 0, 0); + IOPolledFileIOVars * fileVars = vars->fileVars; IOHibernateDone(vars); + IOPolledFileClose(&fileVars, +#if DISABLE_TRIM + 0, NULL, 0, 0, 0); +#else + 0, NULL, 0, sizeof(IOHibernateImageHeader), setFileSize); +#endif gFSState = kFSIdle; } IOLockUnlock(gFSLock); @@ -1569,6 +743,155 @@ IOHibernateSystemSleep(void) /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +static void +IOSetBootImageNVRAM(OSData * data) +{ + IORegistryEntry * regEntry; + + if (!gIOOptionsEntry) + { + regEntry = IORegistryEntry::fromPath("/options", gIODTPlane); + gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry); + if (regEntry && !gIOOptionsEntry) + regEntry->release(); + } + if (gIOOptionsEntry && gIOHibernateBootImageKey) + { + if (data) gIOOptionsEntry->setProperty(gIOHibernateBootImageKey, data); + else + { + gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey); + gIOOptionsEntry->sync(); + } + } +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +/* + * Writes header to disk with signature, block size and file extents data. + * If there are more than 2 extents, then they are written on second block. + */ +static IOReturn +IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature) +{ + IOHibernateImageHeader hdr; + IOItemCount count; + IOReturn err = kIOReturnSuccess; + int rc; + IOPolledFileExtent * fileExtents; + + fileExtents = (typeof(fileExtents)) vars->fileExtents->getBytesNoCopy(); + + memset(&hdr, 0, sizeof(IOHibernateImageHeader)); + count = vars->fileExtents->getLength(); + if (count > sizeof(hdr.fileExtentMap)) + { + hdr.fileExtentMapSize = count; + count = sizeof(hdr.fileExtentMap); + } + else + hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap); + + bcopy(fileExtents, &hdr.fileExtentMap[0], count); + + // copy file block extent list if larger than header + if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap)) + { + count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap); + rc = kern_write_file(vars->fileRef, vars->blockSize, + (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)), + count, IO_SKIP_ENCRYPTION); + if (rc != 0) { + HIBLOG("kern_write_file returned %d\n", rc); + err = kIOReturnIOError; + goto exit; + } + } + hdr.signature = signature; + hdr.deviceBlockSize = vars->blockSize; + + rc = kern_write_file(vars->fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION); + if (rc != 0) { + HIBLOG("kern_write_file returned %d\n", rc); + err = kIOReturnIOError; + goto exit; + } + +exit: + return err; +} + +extern "C" boolean_t root_is_CF_drive; + +void +IOOpenDebugDataFile(const char *fname, uint64_t size) +{ + IOReturn err; + OSData * imagePath = NULL; + uint64_t padding; + + if (!gDebugImageLock) { + gDebugImageLock = IOLockAlloc(); + } + + if (root_is_CF_drive) return; + + // Try to get a lock, but don't block for getting lock + if (!IOLockTryLock(gDebugImageLock)) { + HIBLOG("IOOpenDebugDataFile: Failed to get lock\n"); + return; + } + + if (gDebugImageFileVars || !fname || !size) { + HIBLOG("IOOpenDebugDataFile: conditions failed\n"); + goto exit; + } + + padding = (PAGE_SIZE*2); // allocate couple more pages for header and fileextents + err = IOPolledFileOpen(fname, size+padding, 32ULL*1024*1024*1024, + NULL, 0, + &gDebugImageFileVars, &imagePath, NULL, 0); + + if ((kIOReturnSuccess == err) && imagePath) + { + if ((gDebugImageFileVars->fileSize < (size+padding)) || + (gDebugImageFileVars->fileExtents->getLength() > PAGE_SIZE)) { + // Can't use the file + IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0); + HIBLOG("IOOpenDebugDataFile: too many file extents\n"); + goto exit; + } + + // write extents for debug data usage in EFI + IOWriteExtentsToFile(gDebugImageFileVars, kIOHibernateHeaderOpenSignature); + IOSetBootImageNVRAM(imagePath); + } + +exit: + IOLockUnlock(gDebugImageLock); + + if (imagePath) imagePath->release(); + return; +} + +void +IOCloseDebugDataFile() +{ + IOSetBootImageNVRAM(0); + + if (gDebugImageLock) { + IOLockLock(gDebugImageLock); + if (gDebugImageFileVars != 0) { + IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0); + } + IOLockUnlock(gDebugImageLock); + } + + +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + DECLARE_IOHIBERNATEPROGRESSALPHA static void @@ -1809,6 +1132,7 @@ IOHibernateSystemWake(void) { if (kFSOpened == gFSState) { + IOPolledFilePollersClose(gIOHibernateVars.fileVars, kIOPolledPostflightState); IOHibernateDone(&gIOHibernateVars); } else @@ -1822,8 +1146,6 @@ IOHibernateSystemWake(void) static IOReturn IOHibernateDone(IOHibernateVars * vars) { - IORegistryEntry * next; - hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal); if (vars->videoMapping) @@ -1864,17 +1186,6 @@ IOHibernateDone(IOHibernateVars * vars) IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey); } - if (vars->fileVars) - { - if ((next = vars->fileVars->media)) do - { - next->removeProperty(kIOPolledInterfaceActiveKey); - next = next->getParentEntry(gIOServicePlane); - } - while (next); - IOPolledFileClose(vars->fileVars); - } - // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched #if defined(__i386__) || defined(__x86_64__) @@ -1908,10 +1219,7 @@ IOHibernateDone(IOHibernateVars * vars) } #endif - if (vars->srcBuffer) - vars->srcBuffer->release(); - if (vars->ioBuffer) - vars->ioBuffer->release(); + if (vars->srcBuffer) vars->srcBuffer->release(); bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0])); if (vars->handoffBuffer) { @@ -1956,8 +1264,6 @@ IOHibernateDone(IOHibernateVars * vars) } vars->handoffBuffer->release(); } - if (vars->fileExtents) - vars->fileExtents->release(); bzero(vars, sizeof(*vars)); @@ -1966,54 +1272,63 @@ IOHibernateDone(IOHibernateVars * vars) return (kIOReturnSuccess); } -IOReturn -IOHibernateSystemPostWake(void) +void +IOHibernateSystemPostWakeTrim(void * p1, void * p2) { - struct kern_direct_file_io_ref_t * fileRef; - - if (kFSOpened == gFSState) + // invalidate & close the image file + if (p1) IOLockLock(gFSLock); + if (kFSTrimDelay == gFSState) { - // invalidate & close the image file - gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature; - if ((fileRef = gIOHibernateFileRef)) - { - gIOHibernateFileRef = 0; - IOSleep(TRIM_DELAY); - kern_close_file_for_direct_io(fileRef, + IOPolledFileIOVars * vars = &gFileVars; + IOPolledFileClose(&vars, #if DISABLE_TRIM - 0, 0, 0, 0, 0); + 0, NULL, 0, 0, 0); #else - 0, (caddr_t) gIOHibernateCurrentHeader, - sizeof(IOHibernateImageHeader), - 0, - gIOHibernateCurrentHeader->imageSize); + 0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader), + sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize); #endif - } - gFSState = kFSIdle; + gFSState = kFSIdle; + } + if (p1) IOLockUnlock(gFSLock); +} + +IOReturn +IOHibernateSystemPostWake(void) +{ + gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature; + IOLockLock(gFSLock); + if (kFSTrimDelay == gFSState) IOHibernateSystemPostWakeTrim(NULL, NULL); + else if (kFSOpened != gFSState) gFSState = kFSIdle; + else + { + AbsoluteTime deadline; + + gFSState = kFSTrimDelay; + clock_interval_to_deadline(TRIM_DELAY, kMillisecondScale, &deadline ); + thread_call_enter1_delayed(gIOHibernateTrimCalloutEntry, NULL, deadline); } + IOLockUnlock(gFSLock); + + // IOCloseDebugDataFile() calls IOSetBootImageNVRAM() unconditionally + IOCloseDebugDataFile( ); return (kIOReturnSuccess); } -bool IOHibernateWasScreenLocked(void) +uint32_t IOHibernateWasScreenLocked(void) { - bool ret = false; + uint32_t ret = 0; if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry) { OSData * data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey)); - if (data) switch (*((uint32_t *)data->getBytesNoCopy())) + if (data) { - case kIOScreenLockLocked: - case kIOScreenLockFileVaultDialog: - ret = true; - break; - case kIOScreenLockNoLock: - case kIOScreenLockUnlocked: - default: - ret = false; - break; - } + ret = ((uint32_t *)data->getBytesNoCopy())[0]; + gIOChosenEntry->setProperty(kIOBooterScreenLockStateKey, data); + } } + else gIOChosenEntry->removeProperty(kIOBooterScreenLockStateKey); + return (ret); } @@ -2030,25 +1345,34 @@ SYSCTL_UINT(_kern, OID_AUTO, hibernatemode, &gIOHibernateMode, 0, ""); SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics, CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED, - gIOHibernateStats, hibernate_statistics_t, ""); + &_hibernateStats, hibernate_statistics_t, ""); SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready, CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY, - &gIOHibernateStats->graphicsReadyTime, 0, ""); + &_hibernateStats.graphicsReadyTime, 0, ""); SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification, CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY, - &gIOHibernateStats->wakeNotificationTime, 0, ""); + &_hibernateStats.wakeNotificationTime, 0, ""); SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready, CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY, - &gIOHibernateStats->lockScreenReadyTime, 0, ""); + &_hibernateStats.lockScreenReadyTime, 0, ""); SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready, CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY, - &gIOHibernateStats->hidReadyTime, 0, ""); + &_hibernateStats.hidReadyTime, 0, ""); void IOHibernateSystemInit(IOPMrootDomain * rootDomain) { + gIOHibernateBootImageKey = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey); + +#if defined(__i386__) || defined(__x86_64__) + gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey); + gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082"); + gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext"); + gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey); +#endif /* defined(__i386__) || defined(__x86_64__) */ + OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState)); if (data) { @@ -2070,58 +1394,27 @@ IOHibernateSystemInit(IOPMrootDomain * rootDomain) sysctl_register_oid(&sysctl__kern_hibernatelockscreenready); sysctl_register_oid(&sysctl__kern_hibernatehidready); - gFSLock = IOLockAlloc(); -} - - -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane); -static void -hibernate_setup_for_wake(void) -{ + gFSLock = IOLockAlloc(); } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#define C_ASSERT(e) typedef char __C_ASSERT__[(e) ? 1 : -1] - -static bool -no_encrypt_page(vm_offset_t ppnum) -{ - if (pmap_is_noencrypt((ppnum_t)ppnum) == TRUE) - { - return true; - } - return false; -} - -static void -hibernate_pal_callback(void *vars_arg, vm_offset_t addr) -{ - IOHibernateVars *vars = (IOHibernateVars *)vars_arg; - /* Make sure it's not in either of the save lists */ - hibernate_set_page_state(vars->page_list, vars->page_list_wired, atop_64(addr), 1, kIOHibernatePageStateFree); - - /* Set it in the bitmap of pages owned by the PAL */ - hibernate_page_bitset(vars->page_list_pal, TRUE, atop_64(addr)); -} - -static struct hibernate_cryptvars_t *local_cryptvars; - -extern "C" int -hibernate_pal_write(void *buffer, size_t size) +static IOReturn +IOHibernatePolledFileWrite(IOPolledFileIOVars * vars, + const uint8_t * bytes, IOByteCount size, + IOPolledFileCryptVars * cryptvars) { - IOHibernateVars * vars = &gIOHibernateVars; + IOReturn err; - IOReturn err = IOPolledFileWrite(vars->fileVars, (const uint8_t *)buffer, size, local_cryptvars); - if (kIOReturnSuccess != err) { - kprintf("epic hibernate fail! %d\n", err); - return err; - } + err = IOPolledFileWrite(vars, bytes, size, cryptvars); + if ((kIOReturnSuccess == err) && hibernate_should_abort()) err = kIOReturnAborted; - return 0; + return (err); } +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ extern "C" uint32_t hibernate_write_image(void) @@ -2130,7 +1423,7 @@ hibernate_write_image(void) IOHibernateVars * vars = &gIOHibernateVars; IOPolledFileExtent * fileExtents; - C_ASSERT(sizeof(IOHibernateImageHeader) == 512); + _static_assert_1_arg(sizeof(IOHibernateImageHeader) == 512); uint32_t pageCount, pagesDone; IOReturn err; @@ -2140,8 +1433,7 @@ hibernate_write_image(void) uint8_t * data; uint8_t * compressed; uint8_t * scratch; - void * zerosCompressed; - IOByteCount pageCompressedSize, zerosCompressedLen; + IOByteCount pageCompressedSize; uint64_t compressedSize, uncompressedSize; uint64_t image1Size = 0; uint32_t bitmap_size; @@ -2153,6 +1445,8 @@ hibernate_write_image(void) uint32_t pageAndCount[2]; addr64_t phys64; IOByteCount segLen; + uintptr_t hibernateBase; + uintptr_t hibernateEnd; AbsoluteTime startTime, endTime; AbsoluteTime allTime, compTime; @@ -2165,18 +1459,21 @@ hibernate_write_image(void) uint32_t wiredPagesEncrypted; uint32_t dirtyPagesEncrypted; uint32_t wiredPagesClear; - uint32_t zeroPageCount; + uint32_t svPageCount; + uint32_t zvPageCount; - hibernate_cryptvars_t _cryptvars; - hibernate_cryptvars_t * cryptvars = 0; + IOPolledFileCryptVars _cryptvars; + IOPolledFileCryptVars * cryptvars = 0; wiredPagesEncrypted = 0; dirtyPagesEncrypted = 0; wiredPagesClear = 0; - zeroPageCount = 0; + svPageCount = 0; + zvPageCount = 0; - if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents) - return (false /* sleep */ ); + if (!vars->fileVars + || !vars->fileVars->pollers + || !(kIOHibernateModeOn & gIOHibernateMode)) return (kIOHibernatePostWriteSleep); if (kIOHibernateModeSleep & gIOHibernateMode) kdebug_enable = save_kdebug_enable; @@ -2186,8 +1483,6 @@ hibernate_write_image(void) restore1Sum = sum1 = sum2 = 0; - hibernate_pal_prepare(); - #if CRYPTO // encryption data. "iv" is the "initial vector". if (kIOHibernateModeEncrypt & gIOHibernateMode) @@ -2197,7 +1492,7 @@ hibernate_write_image(void) 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda }; cryptvars = &gIOHibernateCryptWakeContext; - bzero(cryptvars, sizeof(hibernate_cryptvars_t)); + bzero(cryptvars, sizeof(IOPolledFileCryptVars)); aes_encrypt_key(vars->cryptKey, kIOHibernateAESKeySize, &cryptvars->ctx.encrypt); @@ -2206,7 +1501,7 @@ hibernate_write_image(void) &cryptvars->ctx.decrypt); cryptvars = &_cryptvars; - bzero(cryptvars, sizeof(hibernate_cryptvars_t)); + bzero(cryptvars, sizeof(IOPolledFileCryptVars)); for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++) vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount]; bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey)); @@ -2217,13 +1512,9 @@ hibernate_write_image(void) bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE); bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey)); bzero(&vars->cryptKey[0], sizeof(vars->cryptKey)); - - local_cryptvars = cryptvars; } #endif /* CRYPTO */ - hibernate_setup_for_wake(); - hibernate_page_list_setall(vars->page_list, vars->page_list_wired, vars->page_list_pal, @@ -2235,7 +1526,7 @@ hibernate_write_image(void) HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount); - fileExtents = (IOPolledFileExtent *) vars->fileExtents->getBytesNoCopy(); + fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy(); #if 0 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent); @@ -2258,13 +1549,16 @@ hibernate_write_image(void) { compressedSize = 0; uncompressedSize = 0; - zeroPageCount = 0; + svPageCount = 0; + zvPageCount = 0; IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize); HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled()); - err = IOHibernatePollerOpen(vars->fileVars, kIOPolledBeforeSleepState, vars->ioBuffer); + err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState, + // abortable if not low battery + !IOService::getPMRootDomain()->mustHibernate()); HIBLOG("IOHibernatePollerOpen(%x)\n", err); pollerOpen = (kIOReturnSuccess == err); if (!pollerOpen) @@ -2272,21 +1566,17 @@ hibernate_write_image(void) // copy file block extent list if larger than header - count = vars->fileExtents->getLength(); + count = vars->fileVars->fileExtents->getLength(); if (count > sizeof(header->fileExtentMap)) { count -= sizeof(header->fileExtentMap); - err = IOPolledFileWrite(vars->fileVars, + err = IOHibernatePolledFileWrite(vars->fileVars, ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars); if (kIOReturnSuccess != err) break; } - uintptr_t hibernateBase; - uintptr_t hibernateEnd; - hibernateBase = HIB_BASE; /* Defined in PAL headers */ - hibernateEnd = (segHIBB + segSizeHIB); // copy out restore1 code @@ -2327,11 +1617,11 @@ hibernate_write_image(void) count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase); if (count) { - err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars); + err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars); if (kIOReturnSuccess != err) break; } - err = IOPolledFileWrite(vars->fileVars, + err = IOHibernatePolledFileWrite(vars->fileVars, (uint8_t *) 0, &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0], cryptvars); @@ -2341,7 +1631,7 @@ hibernate_write_image(void) count = round_page(hibernateEnd) - ((uintptr_t) src); if (count) { - err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars); + err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars); if (kIOReturnSuccess != err) break; } @@ -2364,7 +1654,7 @@ hibernate_write_image(void) phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone); pageAndCount[0] = atop_64(phys64); pageAndCount[1] = atop_32(segLen); - err = IOPolledFileWrite(vars->fileVars, + err = IOHibernatePolledFileWrite(vars->fileVars, (const uint8_t *) &pageAndCount, sizeof(pageAndCount), cryptvars); if (kIOReturnSuccess != err) @@ -2390,15 +1680,16 @@ hibernate_write_image(void) phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone); sum1 += hibernate_sum_page(src + page, atop_64(phys64)); } - err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars); + err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars); if (kIOReturnSuccess != err) break; } // mark areas for no save - + IOMemoryDescriptor * ioBuffer; + ioBuffer = IOPolledFileGetIOBuffer(vars->fileVars); for (count = 0; - (phys64 = vars->ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone)); + (phys64 = ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone)); count += segLen) { hibernate_set_page_state(vars->page_list, vars->page_list_wired, @@ -2421,7 +1712,7 @@ hibernate_write_image(void) bitmap_size = vars->page_list_wired->list_size; src = (uint8_t *) vars->page_list_wired; - err = IOPolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars); + err = IOHibernatePolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars); if (kIOReturnSuccess != err) break; @@ -2458,20 +1749,10 @@ hibernate_write_image(void) pageCount -= atop_32(segLen); } - (void)hibernate_pal_callback; - src = (uint8_t *) vars->srcBuffer->getBytesNoCopy(); compressed = src + page_size; scratch = compressed + page_size; - // compress a zero page - bzero(src, page_size); - zerosCompressed = vars->handoffBuffer->getBytesNoCopy(); - zerosCompressedLen = WKdm_compress_new((WK_word*) src, - (WK_word*) zerosCompressed, - (WK_word*) scratch, - page_size - 4); - pagesDone = 0; lastBlob = 0; @@ -2489,6 +1770,9 @@ hibernate_write_image(void) kUnwiredEncrypt = kEncrypt }; + bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features())); +#define _pmap_is_noencrypt(x) (cpuAES ? false : pmap_is_noencrypt((x))) + for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--) { if (kUnwiredEncrypt == pageType) @@ -2512,13 +1796,13 @@ hibernate_write_image(void) &ppnum); // kprintf("[%d](%x : %x)\n", pageType, ppnum, count); iterDone = !count; - + if (count && (kWired & pageType) && needEncrypt) { uint32_t checkIndex; for (checkIndex = 0; (checkIndex < count) - && (((kEncrypt & pageType) == 0) == no_encrypt_page(ppnum + checkIndex)); + && (((kEncrypt & pageType) == 0) == _pmap_is_noencrypt(ppnum + checkIndex)); checkIndex++) {} if (!checkIndex) @@ -2541,7 +1825,7 @@ hibernate_write_image(void) { pageAndCount[0] = ppnum; pageAndCount[1] = count; - err = IOPolledFileWrite(vars->fileVars, + err = IOHibernatePolledFileWrite(vars->fileVars, (const uint8_t *) &pageAndCount, sizeof(pageAndCount), cryptvars); if (kIOReturnSuccess != err) @@ -2564,7 +1848,7 @@ hibernate_write_image(void) sum2 += sum; clock_get_uptime(&startTime); - wkresult = WKdm_compress_new((WK_word*) src, + wkresult = WKdm_compress_new((const WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, page_size - 4); @@ -2576,27 +1860,30 @@ hibernate_write_image(void) compBytes += page_size; pageCompressedSize = (-1 == wkresult) ? page_size : wkresult; - if ((pageCompressedSize == zerosCompressedLen) - && !bcmp(compressed, zerosCompressed, zerosCompressedLen)) + if (pageCompressedSize == 0) { - pageCompressedSize = 0; - zeroPageCount++; - } - - if (kIOHibernateModeEncrypt & gIOHibernateMode) - pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1); - - if (pageCompressedSize != page_size) - data = compressed; - else + pageCompressedSize = 4; data = src; + + if (*(uint32_t *)src) + svPageCount++; + else + zvPageCount++; + } + else + { + if (pageCompressedSize != page_size) + data = compressed; + else + data = src; + } tag = pageCompressedSize | kIOHibernateTagSignature; - err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars); + err = IOHibernatePolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars); if (kIOReturnSuccess != err) break; - err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars); + err = IOHibernatePolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars); if (kIOReturnSuccess != err) break; @@ -2643,14 +1930,14 @@ hibernate_write_image(void) if (kWiredEncrypt != pageType) { // end of image1/2 - fill to next block - err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars); + err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars); if (kIOReturnSuccess != err) break; } if (kWiredClear == pageType) { // enlarge wired image for test -// err = IOPolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars); +// err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars); // end wired image header->encryptStart = vars->fileVars->encryptStart; @@ -2664,9 +1951,12 @@ hibernate_write_image(void) { if (kIOReturnOverrun == err) { - // update actual compression ratio on not enough space + // update actual compression ratio on not enough space (for retry) gIOHibernateCompression = (compressedSize << 8) / uncompressedSize; } + + // update partial amount written (for IOPolledFileClose cleanup/unmap) + header->imageSize = vars->fileVars->position; break; } @@ -2685,7 +1975,7 @@ hibernate_write_image(void) header->compression = (compressedSize << 8) / uncompressedSize; gIOHibernateCompression = header->compression; - count = vars->fileExtents->getLength(); + count = vars->fileVars->fileExtents->getLength(); if (count > sizeof(header->fileExtentMap)) { header->fileExtentMapSize = count; @@ -2699,17 +1989,12 @@ hibernate_write_image(void) header->deviceBlockSize = vars->fileVars->blockSize; IOPolledFileSeek(vars->fileVars, 0); - err = IOPolledFileWrite(vars->fileVars, + err = IOHibernatePolledFileWrite(vars->fileVars, (uint8_t *) header, sizeof(IOHibernateImageHeader), cryptvars); if (kIOReturnSuccess != err) break; - err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars); - if (kIOReturnSuccess != err) - break; - err = IOHibernatePollerIODone(vars->fileVars, true); - if (kIOReturnSuccess != err) - break; + err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars); } while (false); @@ -2740,14 +2025,11 @@ hibernate_write_image(void) uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0, sum1, sum2); - HIBLOG("zeroPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n", - zeroPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted); - - if (vars->fileVars->io) - (void) IOHibernatePollerIODone(vars->fileVars, false); + HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n", + svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted); if (pollerOpen) - IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState); + IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted ); if (vars->consoleMapping) ProgressUpdate(gIOHibernateGraphicsInfo, @@ -2804,12 +2086,12 @@ hibernate_machine_init(void) uint64_t compBytes; uint32_t lastProgressStamp = 0; uint32_t progressStamp; - hibernate_cryptvars_t * cryptvars = 0; + IOPolledFileCryptVars * cryptvars = 0; IOHibernateVars * vars = &gIOHibernateVars; bzero(gIOHibernateStats, sizeof(hibernate_statistics_t)); - if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents) + if (!vars->fileVars || !vars->fileVars->pollers) return; sum = gIOHibernateCurrentHeader->actualImage1Sum; @@ -2828,7 +2110,7 @@ hibernate_machine_init(void) #define t40ms(x) (tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000) #define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y); tStat(booterStart, booterStart); - gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart, + gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart; tStat(booterDuration0, booterTime0); tStat(booterDuration1, booterTime1); tStat(booterDuration2, booterTime2); @@ -2884,7 +2166,10 @@ hibernate_machine_init(void) break; case kIOHibernateHandoffTypeGraphicsInfo: - bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo)); + if (handoff->bytecount == sizeof(*gIOHibernateGraphicsInfo)) + { + bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo)); + } break; case kIOHibernateHandoffTypeCryptVars: @@ -2929,7 +2214,7 @@ hibernate_machine_init(void) if (cryptvars && !foundCryptData) panic("hibernate handoff"); - HIBPRINT("video %x %d %d %d status %x\n", + HIBPRINT("video 0x%llx %d %d %d status %x\n", gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth, gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus); @@ -2959,18 +2244,17 @@ hibernate_machine_init(void) AbsoluteTime_to_scalar(&compTime) = 0; compBytes = 0; - HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled()); - err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0); + HIBLOG("IOPolledFilePollersOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled()); + err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledAfterSleepState, false); clock_get_uptime(&startIOTime); endTime = startIOTime; SUB_ABSOLUTETIME(&endTime, &allTime); absolutetime_to_nanoseconds(endTime, &nsec); - HIBLOG("IOHibernatePollerOpen(%x) %qd ms\n", err, nsec / 1000000ULL); + HIBLOG("IOPolledFilePollersOpen(%x) %qd ms\n", err, nsec / 1000000ULL); IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size); // kick off the read ahead - vars->fileVars->io = false; vars->fileVars->bufferHalf = 0; vars->fileVars->bufferLimit = 0; vars->fileVars->lastRead = 0; @@ -3020,30 +2304,39 @@ hibernate_machine_init(void) break; } - if (!compressedSize) bzero_phys(ptoa_64(ppnum), page_size); - else + err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars); + if (kIOReturnSuccess != err) break; + + if (compressedSize < page_size) { - err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars); - if (kIOReturnSuccess != err) break; - if (compressedSize < page_size) - { - decoOffset = page_size; - clock_get_uptime(&startTime); - WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, page_size); - clock_get_uptime(&endTime); - ADD_ABSOLUTETIME(&compTime, &endTime); - SUB_ABSOLUTETIME(&compTime, &startTime); - compBytes += page_size; + decoOffset = page_size; + clock_get_uptime(&startTime); + + if (compressedSize == 4) { + int i; + uint32_t *s, *d; + + s = (uint32_t *)src; + d = (uint32_t *)(uintptr_t)compressed; + + for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++) + *d++ = *s; } - else decoOffset = 0; + else + WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, compressedSize); + clock_get_uptime(&endTime); + ADD_ABSOLUTETIME(&compTime, &endTime); + SUB_ABSOLUTETIME(&compTime, &startTime); + compBytes += page_size; + } + else decoOffset = 0; - sum += hibernate_sum_page((src + decoOffset), ppnum); - err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size); - if (err) - { + sum += hibernate_sum_page((src + decoOffset), ppnum); + err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size); + if (err) + { HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err); break; - } } ppnum++; @@ -3074,12 +2367,9 @@ hibernate_machine_init(void) gIOHibernateCurrentHeader->actualImage2Sum = sum; gIOHibernateCompression = gIOHibernateCurrentHeader->compression; - if (vars->fileVars->io) - (void) IOHibernatePollerIODone(vars->fileVars, false); - clock_get_uptime(&endIOTime); - err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState); + err = IOPolledFilePollersClose(vars->fileVars, kIOPolledAfterSleepState); clock_get_uptime(&endTime); @@ -3128,7 +2418,6 @@ void IOHibernateSetWakeCapabilities(uint32_t capability) { vm_compressor_do_warmup(); } - } }