X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/89b3af67bb32e691275bf6fa803d1834b2284115..ebb1b9f42b62218f29061826217bb0f71cd375a6:/iokit/Kernel/IOHibernateIO.cpp diff --git a/iokit/Kernel/IOHibernateIO.cpp b/iokit/Kernel/IOHibernateIO.cpp index a7621a2ce..a4d7dbb4d 100644 --- a/iokit/Kernel/IOHibernateIO.cpp +++ b/iokit/Kernel/IOHibernateIO.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2004-2008 Apple Computer, Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -151,6 +151,7 @@ to restrict I/O ops. #include #include "IOPMPowerStateQueue.h" #include +#include #include #include @@ -158,37 +159,24 @@ to restrict I/O ops. #include #include // (FWRITE, ...) #include +#include #include #include #include #include "IOHibernateInternal.h" -#include "WKdm.h" +#include #include "IOKitKernelInternal.h" +#include -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +#include +#include -OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject); - -OSMetaClassDefineReservedUnused(IOPolledInterface, 0); -OSMetaClassDefineReservedUnused(IOPolledInterface, 1); -OSMetaClassDefineReservedUnused(IOPolledInterface, 2); -OSMetaClassDefineReservedUnused(IOPolledInterface, 3); -OSMetaClassDefineReservedUnused(IOPolledInterface, 4); -OSMetaClassDefineReservedUnused(IOPolledInterface, 5); -OSMetaClassDefineReservedUnused(IOPolledInterface, 6); -OSMetaClassDefineReservedUnused(IOPolledInterface, 7); -OSMetaClassDefineReservedUnused(IOPolledInterface, 8); -OSMetaClassDefineReservedUnused(IOPolledInterface, 9); -OSMetaClassDefineReservedUnused(IOPolledInterface, 10); -OSMetaClassDefineReservedUnused(IOPolledInterface, 11); -OSMetaClassDefineReservedUnused(IOPolledInterface, 12); -OSMetaClassDefineReservedUnused(IOPolledInterface, 13); -OSMetaClassDefineReservedUnused(IOPolledInterface, 14); -OSMetaClassDefineReservedUnused(IOPolledInterface, 15); +extern "C" addr64_t kvtophys(vm_offset_t va); /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +extern unsigned int save_kdebug_enable; extern uint32_t gIOHibernateState; uint32_t gIOHibernateMode; static char gIOHibernateBootSignature[256+1]; @@ -198,14 +186,22 @@ uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (m static IODTNVRAM * gIOOptionsEntry; static IORegistryEntry * gIOChosenEntry; -#ifdef __i386__ +#if defined(__i386__) || defined(__x86_64__) static const OSSymbol * gIOCreateEFIDevicePathSymbol; +static const OSSymbol * gIOHibernateRTCVariablesKey; +static const OSSymbol * gIOHibernateBoot0082Key; +static const OSSymbol * gIOHibernateBootNextKey; +static OSData * gIOHibernateBoot0082Data; +static OSData * gIOHibernateBootNextData; +static OSObject * gIOHibernateBootNextSave; #endif static IOPolledFileIOVars gFileVars; static IOHibernateVars gIOHibernateVars; static struct kern_direct_file_io_ref_t * gIOHibernateFileRef; static hibernate_cryptvars_t gIOHibernateCryptWakeContext; +static hibernate_graphics_t _hibernateGraphics; +static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics; /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -243,7 +239,7 @@ IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md, addr64_t dstAddr64; IOByteCount dstLen; - dstAddr64 = md->getPhysicalSegment64(offset, &dstLen); + dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone); if (!dstAddr64) break; @@ -281,7 +277,7 @@ IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md, addr64_t srcAddr64; IOByteCount dstLen; - srcAddr64 = md->getPhysicalSegment64(offset, &dstLen); + srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone); if (!srcAddr64) break; @@ -468,28 +464,45 @@ IOHibernatePollerIO(IOPolledFileIOVars * vars, } static IOReturn -IOHibernatePollerIODone(IOPolledFileIOVars * vars) +IOHibernatePollerIODone(IOPolledFileIOVars * vars, bool abortable) { - IOReturn err = kIOReturnError; - int32_t idx; + IOReturn err = kIOReturnSuccess; + int32_t idx = 0; IOPolledInterface * poller; while (-1 == vars->ioStatus) { - for (idx = 0; - (poller = (IOPolledInterface *) vars->pollers->getObject(idx)); + for (idx = 0; + (poller = (IOPolledInterface *) vars->pollers->getObject(idx)); idx++) { - err = poller->checkForWork(); - if (err) - HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err); + IOReturn newErr; + newErr = poller->checkForWork(); + if ((newErr == kIOReturnAborted) && !abortable) + newErr = kIOReturnSuccess; + if (kIOReturnSuccess == err) + err = newErr; } } - if (kIOReturnSuccess != vars->ioStatus) - HIBLOG("IOPolledInterface::ioStatus 0x%x\n", vars->ioStatus); + if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort()) + { + err = kIOReturnAborted; + HIBLOG("IOPolledInterface::checkForWork sw abort\n"); + } + + if (err) + { + HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err); + } + else + { + err = vars->ioStatus; + if (kIOReturnSuccess != err) + HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err); + } - return (vars->ioStatus); + return (err); } IOReturn @@ -537,19 +550,59 @@ file_extent_callback(void * ref, uint64_t start, uint64_t length) ctx->size += length; } +static IOService * +IOCopyMediaForDev(dev_t device) +{ + OSDictionary * matching; + OSNumber * num; + OSIterator * iter; + IOService * result = 0; + + matching = IOService::serviceMatching("IOMedia"); + if (!matching) + return (0); + do + { + num = OSNumber::withNumber(major(device), 32); + if (!num) + break; + matching->setObject(kIOBSDMajorKey, num); + num->release(); + num = OSNumber::withNumber(minor(device), 32); + if (!num) + break; + matching->setObject(kIOBSDMinorKey, num); + num->release(); + if (!num) + break; + iter = IOService::getMatchingServices(matching); + if (iter) + { + result = (IOService *) iter->getNextObject(); + result->retain(); + iter->release(); + } + } + while (false); + matching->release(); + + return (result); +} + IOReturn IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer, IOPolledFileIOVars ** fileVars, OSData ** fileExtents, - OSData ** imagePath) + OSData ** imagePath, uint8_t * volumeCryptKey) { IOReturn err = kIOReturnError; IOPolledFileIOVars * vars; _OpenFileContext ctx; OSData * extentsData; OSNumber * num; - IORegistryEntry * part = 0; - OSDictionary * matching; - OSIterator * iter; + IOService * part = 0; + OSString * keyUUID = 0; + OSString * keyStoreUUID = 0; + dev_t block_dev; dev_t hibernate_image_dev; uint64_t maxiobytes; @@ -572,16 +625,25 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer, ctx.size = 0; vars->fileRef = kern_open_file_for_direct_io(filename, &file_extent_callback, &ctx, + &block_dev, &hibernate_image_dev, &vars->block0, - &maxiobytes); + &maxiobytes, + &vars->flags, + 0, (caddr_t) gIOHibernateCurrentHeader, + sizeof(IOHibernateImageHeader)); if (!vars->fileRef) { err = kIOReturnNoSpace; break; } - HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx\n", filename, ctx.size, - vars->block0, maxiobytes); + gIOHibernateFileRef = vars->fileRef; + + if (kIOHibernateModeSSDInvert & gIOHibernateMode) + vars->flags ^= kIOHibernateOptionSSD; + + HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx ssd %d\n", filename, ctx.size, + vars->block0, maxiobytes, kIOHibernateOptionSSD & vars->flags); if (ctx.size < 1*1024*1024) // check against image size estimate! { err = kIOReturnNoSpace; @@ -592,39 +654,52 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer, vars->bufferSize = maxiobytes; vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy(); - - matching = IOService::serviceMatching("IOMedia"); - num = OSNumber::withNumber(major(hibernate_image_dev), 32); - matching->setObject(kIOBSDMajorKey, num); - num->release(); - num = OSNumber::withNumber(minor(hibernate_image_dev), 32); - matching->setObject(kIOBSDMinorKey, num); - num->release(); - iter = IOService::getMatchingServices(matching); - matching->release(); - if (iter) - { - part = (IORegistryEntry *) iter->getNextObject(); - part->retain(); - iter->release(); - } - - int minor, major; + + part = IOCopyMediaForDev(block_dev); + if (!part) + break; + + err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false, + (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL); + if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID) + { +// IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy()); + uuid_t volumeKeyUUID; + aks_volume_key_t vek; + static IOService * sKeyStore; + static const OSSymbol * sAKSGetKey; + + if (!sAKSGetKey) + sAKSGetKey = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY); + if (!sKeyStore) + sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane); + if (sKeyStore) + err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID); + else + err = kIOReturnNoResources; + if (kIOReturnSuccess == err) + err = sKeyStore->callPlatformFunction(sAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL); + if (kIOReturnSuccess != err) + IOLog("volume key err 0x%x\n", err); + else + { + size_t bytes = (kIOHibernateAESKeySize / 8); + if (vek.key.keybytecount < bytes) + bytes = vek.key.keybytecount; + bcopy(&vek.key.keybytes[0], volumeCryptKey, bytes); + } + bzero(&vek, sizeof(vek)); + } + part->release(); + + part = IOCopyMediaForDev(hibernate_image_dev); + if (!part) + break; + IORegistryEntry * next; IORegistryEntry * child; OSData * data; - num = (OSNumber *) part->getProperty(kIOBSDMajorKey); - if (!num) - break; - major = num->unsigned32BitValue(); - num = (OSNumber *) part->getProperty(kIOBSDMinorKey); - if (!num) - break; - minor = num->unsigned32BitValue(); - - hibernate_image_dev = makedev(major, minor); - vars->pollers = OSArray::withCapacity(4); if (!vars->pollers) break; @@ -652,7 +727,7 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer, && child->isParent(next, gIOServicePlane, true)); HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n", - major, minor, vars->blockSize, vars->pollers->getCount()); + major(hibernate_image_dev), minor(hibernate_image_dev), (long)vars->blockSize, vars->pollers->getCount()); if (vars->pollers->getCount() < kIOHibernateMinPollersNeeded) continue; @@ -671,18 +746,22 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer, if ((extentsData->getLength() >= sizeof(IOPolledFileExtent))) { - char str2[24]; + char str2[24 + sizeof(uuid_string_t) + 2]; -#if __i386__ +#if defined(__i386__) || defined(__x86_64__) if (!gIOCreateEFIDevicePathSymbol) gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath"); - sprintf(str2, "%qx", vars->extentMap[0]); + if (keyUUID) + snprintf(str2, sizeof(str2), "%qx:%s", + vars->extentMap[0].start, keyUUID->getCStringNoCopy()); + else + snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start); err = IOService::getPlatform()->callPlatformFunction( gIOCreateEFIDevicePathSymbol, false, - (void *) part, (void *) str2, (void *) true, - (void *) &data); + (void *) part, (void *) str2, + (void *) (uintptr_t) true, (void *) &data); #else char str1[256]; int len = sizeof(str1); @@ -691,7 +770,7 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer, err = kIOReturnNotFound; else { - sprintf(str2, ",%qx", vars->extentMap[0]); + snprintf(str2, sizeof(str2), ",%qx", vars->extentMap[0].start); // (strip the plane name) char * tail = strchr(str1, ':'); if (!tail) @@ -712,7 +791,10 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer, { HIBLOG("error 0x%x opening hibernation file\n", err); if (vars->fileRef) - kern_close_file_for_direct_io(vars->fileRef); + { + kern_close_file_for_direct_io(vars->fileRef, 0, 0, 0); + gIOHibernateFileRef = vars->fileRef = NULL; + } } if (part) @@ -730,8 +812,6 @@ IOPolledFileClose( IOPolledFileIOVars * vars ) vars->pollers->release(); } - gIOHibernateFileRef = vars->fileRef; - bzero(vars, sizeof(IOPolledFileIOVars)); return (kIOReturnSuccess); @@ -810,29 +890,45 @@ IOPolledFileWrite(IOPolledFileIOVars * vars, - vars->extentPosition + vars->currentExtent->start); uint32_t length = (vars->bufferOffset); - if (cryptvars && vars->encryptStart && (vars->position > vars->encryptStart)) +#if CRYPTO + if (cryptvars && vars->encryptStart + && (vars->position > vars->encryptStart) + && ((vars->position - length) < vars->encryptEnd)) { + AbsoluteTime startTime, endTime; + uint32_t encryptLen, encryptStart; encryptLen = vars->position - vars->encryptStart; if (encryptLen > length) encryptLen = length; encryptStart = length - encryptLen; - + if (vars->position > vars->encryptEnd) + encryptLen -= (vars->position - vars->encryptEnd); + + clock_get_uptime(&startTime); + // encrypt the buffer aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart, &cryptvars->aes_iv[0], encryptLen / AES_BLOCK_SIZE, vars->buffer + vars->bufferHalf + encryptStart, &cryptvars->ctx.encrypt); + + clock_get_uptime(&endTime); + ADD_ABSOLUTETIME(&vars->cryptTime, &endTime); + SUB_ABSOLUTETIME(&vars->cryptTime, &startTime); + vars->cryptBytes += encryptLen; + // save initial vector for following encrypts bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE, &cryptvars->aes_iv[0], AES_BLOCK_SIZE); } +#endif /* CRYPTO */ if (vars->io) { - err = IOHibernatePollerIODone(vars); + err = IOHibernatePollerIODone(vars, true); if (kIOReturnSuccess != err) break; } @@ -898,11 +994,11 @@ IOPolledFileRead(IOPolledFileIOVars * vars, vars->bufferOffset += copy; // vars->position += copy; - if (vars->bufferOffset == vars->bufferLimit) + if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd)) { if (vars->io) { - err = IOHibernatePollerIODone(vars); + err = IOHibernatePollerIODone(vars, false); if (kIOReturnSuccess != err) break; } @@ -911,9 +1007,9 @@ IOPolledFileRead(IOPolledFileIOVars * vars, if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position); - vars->position += vars->lastRead; + vars->position += vars->lastRead; vars->extentRemaining -= vars->lastRead; - vars->bufferLimit = vars->lastRead; + vars->bufferLimit = vars->lastRead; if (!vars->extentRemaining) { @@ -927,39 +1023,56 @@ if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", } } - if (vars->extentRemaining <= vars->bufferSize) - vars->lastRead = vars->extentRemaining; - else - vars->lastRead = vars->bufferSize; - + uint64_t length; + uint64_t lastReadLength = vars->lastRead; uint64_t offset = (vars->position - vars->extentPosition + vars->currentExtent->start); - uint64_t length = (vars->lastRead); + if (vars->extentRemaining <= vars->bufferSize) + length = vars->extentRemaining; + else + length = vars->bufferSize; + if ((length + vars->position) > vars->readEnd) + length = vars->readEnd - vars->position; + vars->lastRead = length; + if (length) + { //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length); - - err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length); - if (kIOReturnSuccess != err) - break; - vars->io = true; + err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length); + if (kIOReturnSuccess != err) + break; + vars->io = true; + } vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize; vars->bufferOffset = 0; +#if CRYPTO if (cryptvars) { uint8_t thisVector[AES_BLOCK_SIZE]; + AbsoluteTime startTime, endTime; + // save initial vector for following decrypts bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE); - bcopy(vars->buffer + vars->bufferHalf + vars->lastRead - AES_BLOCK_SIZE, + bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE, &cryptvars->aes_iv[0], AES_BLOCK_SIZE); + // decrypt the buffer + clock_get_uptime(&startTime); + aes_decrypt_cbc(vars->buffer + vars->bufferHalf, &thisVector[0], - vars->lastRead / AES_BLOCK_SIZE, + lastReadLength / AES_BLOCK_SIZE, vars->buffer + vars->bufferHalf, &cryptvars->ctx.decrypt); + + clock_get_uptime(&endTime); + ADD_ABSOLUTETIME(&vars->cryptTime, &endTime); + SUB_ABSOLUTETIME(&vars->cryptTime, &startTime); + vars->cryptBytes += lastReadLength; } +#endif /* CRYPTO */ } } while (size); @@ -976,7 +1089,7 @@ IOHibernateSystemSleep(void) OSData * data; OSObject * obj; OSString * str; - OSNumber * num; + bool dsSSD; IOHibernateVars * vars = &gIOHibernateVars; @@ -986,33 +1099,24 @@ IOHibernateSystemSleep(void) gIOHibernateState = kIOHibernateStateInactive; - if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateModeKey))) + gIOHibernateDebugFlags = 0; + if (kIOLogHibernate & gIOKitDebug) + gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs; + + if (IOService::getPMRootDomain()->getHibernateSettings( + &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime)) { - if ((num = OSDynamicCast(OSNumber, obj))) - gIOHibernateMode = num->unsigned32BitValue(); if (kIOHibernateModeSleep & gIOHibernateMode) // default to discard clean for safe sleep gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive | kIOHibernateModeDiscardCleanActive); - - obj->release(); - } - if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeRatioKey))) - { - if ((num = OSDynamicCast(OSNumber, obj))) - gIOHibernateFreeRatio = num->unsigned32BitValue(); - obj->release(); - } - if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeTimeKey))) - { - if ((num = OSDynamicCast(OSNumber, obj))) - gIOHibernateFreeTime = num->unsigned32BitValue(); - obj->release(); } + if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey))) { if ((str = OSDynamicCast(OSString, obj))) - strcpy(gIOHibernateFilename, str->getCStringNoCopy()); + strlcpy(gIOHibernateFilename, str->getCStringNoCopy(), + sizeof(gIOHibernateFilename)); obj->release(); } @@ -1021,6 +1125,7 @@ IOHibernateSystemSleep(void) HIBLOG("hibernate image path: %s\n", gIOHibernateFilename); + do { vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, @@ -1028,47 +1133,80 @@ IOHibernateSystemSleep(void) vars->ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, 2 * kDefaultIOSize, page_size); - if (!vars->srcBuffer || !vars->ioBuffer) + vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, + ptoa_64(gIOHibernateHandoffPageCount), page_size); + + if (!vars->srcBuffer || !vars->ioBuffer || !vars->handoffBuffer) { err = kIOReturnNoMemory; break; } + // open & invalidate the image file + gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature; err = IOPolledFileOpen(gIOHibernateFilename, vars->ioBuffer, - &vars->fileVars, &vars->fileExtents, &data); + &vars->fileVars, &vars->fileExtents, &data, + &vars->volumeCryptKey[0]); if (KERN_SUCCESS != err) { HIBLOG("IOPolledFileOpen(%x)\n", err); break; } - if (vars->fileVars->fileRef) - { - // invalidate the image file - gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature; - int err = kern_write_file(vars->fileVars->fileRef, 0, - (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader)); - if (KERN_SUCCESS != err) - HIBLOG("kern_write_file(%d)\n", err); - } bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader)); + gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags; + dsSSD = ((0 != (kIOHibernateOptionSSD & vars->fileVars->flags)) + && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey))); + if (dsSSD) + { + gIOHibernateCurrentHeader->options |= + kIOHibernateOptionSSD + | kIOHibernateOptionColor; + +#if defined(__i386__) || defined(__x86_64__) + if (!uuid_is_null(vars->volumeCryptKey) && + (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey))) + { + uintptr_t smcVars[2]; + smcVars[0] = sizeof(vars->volumeCryptKey); + smcVars[1] = (uintptr_t)(void *) &vars->volumeCryptKey[0]; + + IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars)); + bzero(smcVars, sizeof(smcVars)); + } +#endif + } + else + { + gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress; + } boolean_t encryptedswap; + AbsoluteTime startTime, endTime; + uint64_t nsec; + + clock_get_uptime(&startTime); err = hibernate_setup(gIOHibernateCurrentHeader, gIOHibernateFreeRatio, gIOHibernateFreeTime, - &vars->page_list, &vars->page_list_wired, &encryptedswap); + dsSSD, + &vars->page_list, &vars->page_list_wired, &vars->page_list_pal, &encryptedswap); + clock_get_uptime(&endTime); + SUB_ABSOLUTETIME(&endTime, &startTime); + absolutetime_to_nanoseconds(endTime, &nsec); + HIBLOG("hibernate_setup(%d) took %qd ms\n", err, nsec / 1000000ULL); + if (KERN_SUCCESS != err) - { - HIBLOG("hibernate_setup(%d)\n", err); break; - } - if (encryptedswap) + if (encryptedswap || !uuid_is_null(vars->volumeCryptKey)) gIOHibernateMode ^= kIOHibernateModeEncrypt; - vars->videoAllocSize = kVideoMapSize; - if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize)) - vars->videoMapping = 0; + if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options) + { + vars->videoAllocSize = kVideoMapSize; + if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize)) + vars->videoMapping = 0; + } // generate crypt keys for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++) @@ -1101,46 +1239,7 @@ IOHibernateSystemSleep(void) } data->release(); -#ifdef __ppc__ - size_t len; - char valueString[16]; - - vars->saveBootDevice = gIOOptionsEntry->copyProperty(kIOSelectedBootDeviceKey); - if (gIOChosenEntry) - { - OSData * bootDevice = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOBootPathKey)); - if (bootDevice) - { - sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey); - OSString * str2 = OSString::withCStringNoCopy((const char *) bootDevice->getBytesNoCopy()); - if (sym && str2) - gIOOptionsEntry->setProperty(sym, str2); - if (sym) - sym->release(); - if (str2) - str2->release(); - } - data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMemorySignatureKey)); - if (data) - { - vars->haveFastBoot = true; - - len = sprintf(valueString, "0x%lx", *((UInt32 *)data->getBytesNoCopy())); - data = OSData::withBytes(valueString, len + 1); - sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey); - if (sym && data) - gIOOptionsEntry->setProperty(sym, data); - if (sym) - sym->release(); - if (data) - data->release(); - } - data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey)); - if (data) - gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy()); - } -#endif /* __ppc__ */ -#ifdef __i386__ +#if defined(__i386__) || defined(__x86_64__) struct AppleRTCHibernateVars { uint8_t signature[4]; @@ -1179,8 +1278,26 @@ IOHibernateSystemSleep(void) } data = OSData::withBytes(&rtcVars, sizeof(rtcVars)); if (data) - { - IOService::getPMRootDomain()->setProperty(kIOHibernateRTCVariablesKey, data); + { + if (!gIOHibernateRTCVariablesKey) + gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey); + if (gIOHibernateRTCVariablesKey) + IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data); + + if( gIOOptionsEntry ) + { + if( gIOHibernateMode & kIOHibernateModeSwitch ) + { + const OSSymbol *sym; + sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSwitchVarsKey); + if( sym ) + { + gIOOptionsEntry->setProperty(sym, data); /* intentional insecure backup of rtc boot vars */ + sym->release(); + } + } + } + data->release(); } if (gIOChosenEntry) @@ -1188,8 +1305,49 @@ IOHibernateSystemSleep(void) data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey)); if (data) gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy()); + { + // set BootNext + + if (!gIOHibernateBoot0082Data) + { + data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path")); + if (data) + { + // AppleNVRAM_EFI_LOAD_OPTION + struct { + uint32_t Attributes; + uint16_t FilePathLength; + uint16_t Desc; + } loadOptionHeader; + loadOptionHeader.Attributes = 1; + loadOptionHeader.FilePathLength = data->getLength(); + loadOptionHeader.Desc = 0; + gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength); + if (gIOHibernateBoot0082Data) + { + gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader)); + gIOHibernateBoot0082Data->appendBytes(data); + } + } + } + if (!gIOHibernateBoot0082Key) + gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082"); + if (!gIOHibernateBootNextKey) + gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext"); + if (!gIOHibernateBootNextData) + { + uint16_t bits = 0x0082; + gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits)); + } + if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData) + { + gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey); + gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data); + gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData); + } + } } -#else /* !__i386__ */ +#else /* !i386 && !x86_64 */ if (kIOHibernateModeEncrypt & gIOHibernateMode) { data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey)); @@ -1200,7 +1358,7 @@ IOHibernateSystemSleep(void) sym->release(); if (data) data->release(); - if (gIOHibernateBootSignature[0]) + if (false && gIOHibernateBootSignature[0]) { data = OSData::withCapacity(16); sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey); @@ -1243,7 +1401,7 @@ IOHibernateSystemSleep(void) &newVolume, sizeof(newVolume)); } } -#endif /* !__i386__ */ +#endif /* !i386 && !x86_64 */ } // -- @@ -1257,27 +1415,175 @@ IOHibernateSystemSleep(void) /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +DECLARE_IOHIBERNATEPROGRESSALPHA + +static void +ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen) +{ + uint32_t rowBytes, pixelShift; + uint32_t x, y; + int32_t blob; + uint32_t alpha, in, color, result; + uint8_t * out; + uint32_t saveindex[kIOHibernateProgressCount] = { 0 }; + + rowBytes = display->rowBytes; + pixelShift = display->depth >> 4; + if (pixelShift < 1) return; + + screen += ((display->width + - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1)) + + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes; + + for (y = 0; y < kIOHibernateProgressHeight; y++) + { + out = screen + y * rowBytes; + for (blob = 0; blob < kIOHibernateProgressCount; blob++) + { + color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray; + for (x = 0; x < kIOHibernateProgressWidth; x++) + { + alpha = gIOHibernateProgressAlpha[y][x]; + result = color; + if (alpha) + { + if (0xff != alpha) + { + if (1 == pixelShift) + { + in = *((uint16_t *)out) & 0x1f; // 16 + in = (in << 3) | (in >> 2); + } + else + in = *((uint32_t *)out) & 0xff; // 32 + saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in; + result = ((255 - alpha) * in + alpha * result + 0xff) >> 8; + } + if (1 == pixelShift) + { + result >>= 3; + *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16 + } + else + *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32 + } + out += (1 << pixelShift); + } + out += (kIOHibernateProgressSpacing << pixelShift); + } + } +} + + +static void +ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select) +{ + uint32_t rowBytes, pixelShift; + uint32_t x, y; + int32_t blob, lastBlob; + uint32_t alpha, in, color, result; + uint8_t * out; + uint32_t saveindex[kIOHibernateProgressCount] = { 0 }; + + pixelShift = display->depth >> 4; + if (pixelShift < 1) + return; + + rowBytes = display->rowBytes; + + screen += ((display->width + - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1)) + + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes; + + lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1); + + screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift; + + for (y = 0; y < kIOHibernateProgressHeight; y++) + { + out = screen + y * rowBytes; + for (blob = firstBlob; blob <= lastBlob; blob++) + { + color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray; + for (x = 0; x < kIOHibernateProgressWidth; x++) + { + alpha = gIOHibernateProgressAlpha[y][x]; + result = color; + if (alpha) + { + if (0xff != alpha) + { + in = display->progressSaveUnder[blob][saveindex[blob]++]; + result = ((255 - alpha) * in + alpha * result + 0xff) / 255; + } + if (1 == pixelShift) + { + result >>= 3; + *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16 + } + else + *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32 + } + out += (1 << pixelShift); + } + out += (kIOHibernateProgressSpacing << pixelShift); + } + } +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + IOReturn IOHibernateSystemHasSlept(void) { IOHibernateVars * vars = &gIOHibernateVars; + OSObject * obj; + OSData * data; + + obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey); + vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj); + if (obj && !vars->previewBuffer) + obj->release(); + + vars->consoleMapping = NULL; + if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare())) + { + vars->previewBuffer->release(); + vars->previewBuffer = 0; + } - if ((vars->previewData = OSDynamicCast(OSData, - IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewBufferKey)))) + if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options) + && vars->previewBuffer + && (data = OSDynamicCast(OSData, + IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey)))) { - vars->previewBuffer = IOMemoryDescriptor::withAddress( - (void *) vars->previewData->getBytesNoCopy(), - vars->previewData->getLength(), - kIODirectionInOut); + UInt32 flags = *((UInt32 *)data->getBytesNoCopy()); + HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags); - if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare())) - { - vars->previewBuffer->release(); - vars->previewBuffer = 0; - } - if (!vars->previewBuffer) - vars->previewData = 0; + IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey); + + if (kIOHibernatePreviewUpdates & flags) + { + PE_Video consoleInfo; + hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo; + + IOService::getPlatform()->getConsoleInfo(&consoleInfo); + + graphicsInfo->width = consoleInfo.v_width; + graphicsInfo->height = consoleInfo.v_height; + graphicsInfo->rowBytes = consoleInfo.v_rowBytes; + graphicsInfo->depth = consoleInfo.v_depth; + vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr; + + HIBPRINT("video %p %d %d %d\n", + vars->consoleMapping, graphicsInfo->depth, + graphicsInfo->width, graphicsInfo->height); + if (vars->consoleMapping) + ProgressInit(graphicsInfo, vars->consoleMapping, + &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder)); + } } + if (gIOOptionsEntry) gIOOptionsEntry->sync(); @@ -1286,6 +1592,38 @@ IOHibernateSystemHasSlept(void) /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +static DeviceTreeNode * +MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry) +{ + DeviceTreeNodeProperty * prop; + DeviceTreeNode * child; + IORegistryEntry * childRegEntry; + const char * nameProp; + unsigned int propLen, idx; + + prop = (DeviceTreeNodeProperty *) (entry + 1); + for (idx = 0; idx < entry->nProperties; idx++) + { + if (regEntry && (0 != strcmp("name", prop->name))) + { + regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length); +// HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length); + } + prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3)); + } + + child = (DeviceTreeNode *) prop; + for (idx = 0; idx < entry->nChildren; idx++) + { + if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen)) + panic("no name"); + childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL; +// HIBPRINT("%s == %p\n", nameProp, childRegEntry); + child = MergeDeviceTree(child, childRegEntry); + } + return (child); +} + IOReturn IOHibernateSystemWake(void) { @@ -1300,7 +1638,7 @@ IOHibernateSystemWake(void) IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize); if (vars->videoAllocSize) // dealloc range - kmem_free(kernel_map, trunc_page_32(vars->videoMapping), vars->videoAllocSize); + kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize); } if (vars->previewBuffer) @@ -1309,77 +1647,112 @@ IOHibernateSystemWake(void) vars->previewBuffer = 0; } - if (vars->fileVars) + if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) { - IOPolledFileClose(vars->fileVars); + IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey, + gIOHibernateCurrentHeader->options, 32); } - - // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched - -#ifdef __ppc__ - OSData * data = OSData::withCapacity(4); - if (gIOOptionsEntry && data) + else { - const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey); - if (sym) - { - gIOOptionsEntry->setProperty(sym, data); - sym->release(); - } - sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey); - if (sym) - { - if (vars->saveBootDevice) - { - gIOOptionsEntry->setProperty(sym, vars->saveBootDevice); - vars->saveBootDevice->release(); - } - sym->release(); - } - sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey); - if (sym) - { - gIOOptionsEntry->setProperty(sym, data); - sym->release(); - } - sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey); - if (sym) - { - gIOOptionsEntry->removeProperty(sym); - sym->release(); - } + IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey); } - if (data) - data->release(); - if (gIOOptionsEntry) + if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) + && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus)) { - if (!vars->haveFastBoot) - { - // reset boot audio volume - IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform()); - if (platform) - platform->writeXPRAM(kXPRamAudioVolume, - &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume)); - } + IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey, + &gIOHibernateGraphicsInfo->gfxStatus, + sizeof(gIOHibernateGraphicsInfo->gfxStatus)); + } + else + { + IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey); + } - // sync now to hardware if the booter has not - if (kIOHibernateStateInactive == gIOHibernateState) - gIOOptionsEntry->sync(); - else - // just sync the variables in case a later panic syncs nvram (it won't sync variables) - gIOOptionsEntry->syncOFVariables(); + + if (vars->fileVars) + { + IOPolledFileClose(vars->fileVars); } -#endif -#ifdef __i386__ - IOService::getPMRootDomain()->removeProperty(kIOHibernateRTCVariablesKey); + // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched + +#if defined(__i386__) || defined(__x86_64__) + IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey); + IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey); + + /* + * Hibernate variable is written to NVRAM on platforms in which RtcRam + * is not backed by coin cell. Remove Hibernate data from NVRAM. + */ + if (gIOOptionsEntry) { + + if (gIOHibernateRTCVariablesKey) { + if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) { + gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey); + } + } + + if (gIOHibernateBootNextKey) + { + if (gIOHibernateBootNextSave) + { + gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave); + gIOHibernateBootNextSave->release(); + gIOHibernateBootNextSave = NULL; + } + else + gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey); + } + gIOOptionsEntry->sync(); + } #endif if (vars->srcBuffer) vars->srcBuffer->release(); if (vars->ioBuffer) vars->ioBuffer->release(); + bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0])); + if (vars->handoffBuffer) + { + IOHibernateHandoff * handoff; + bool done = false; + for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy(); + !done; + handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount]) + { +// HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount); + uint8_t * data = &handoff->data[0]; + switch (handoff->type) + { + case kIOHibernateHandoffTypeEnd: + done = true; + break; + + case kIOHibernateHandoffTypeDeviceTree: + MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot()); + break; + + case kIOHibernateHandoffTypeKeyStore: +#if defined(__i386__) || defined(__x86_64__) + { + IOBufferMemoryDescriptor * + md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn); + if (md) + { + IOSetKeyStoreData(md); + } + } +#endif + break; + + default: + done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000)); + break; + } + } + vars->handoffBuffer->release(); + } if (vars->fileExtents) vars->fileExtents->release(); @@ -1395,14 +1768,11 @@ IOHibernateSystemPostWake(void) { if (gIOHibernateFileRef) { - // invalidate the image file + // invalidate & close the image file gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature; - int err = kern_write_file(gIOHibernateFileRef, 0, - (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader)); - if (KERN_SUCCESS != err) - HIBLOG("kern_write_file(%d)\n", err); - - kern_close_file_for_direct_io(gIOHibernateFileRef); + kern_close_file_for_direct_io(gIOHibernateFileRef, + 0, (caddr_t) gIOHibernateCurrentHeader, + sizeof(IOHibernateImageHeader)); gIOHibernateFileRef = 0; } return (kIOReturnSuccess); @@ -1410,6 +1780,16 @@ IOHibernateSystemPostWake(void) /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +SYSCTL_STRING(_kern, OID_AUTO, hibernatefile, + CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED, + gIOHibernateFilename, sizeof(gIOHibernateFilename), ""); +SYSCTL_STRING(_kern, OID_AUTO, bootsignature, + CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED, + gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), ""); +SYSCTL_UINT(_kern, OID_AUTO, hibernatemode, + CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED, + &gIOHibernateMode, 0, ""); + void IOHibernateSystemInit(IOPMrootDomain * rootDomain) { @@ -1420,43 +1800,71 @@ IOHibernateSystemInit(IOPMrootDomain * rootDomain) data->release(); } - if (PE_parse_boot_arg("hfile", gIOHibernateFilename)) + if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename))) gIOHibernateMode = kIOHibernateModeOn; else gIOHibernateFilename[0] = 0; - static SYSCTL_STRING(_kern, OID_AUTO, hibernatefile, - CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, - gIOHibernateFilename, sizeof(gIOHibernateFilename), ""); sysctl_register_oid(&sysctl__kern_hibernatefile); - - static SYSCTL_STRING(_kern, OID_AUTO, bootsignature, - CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, - gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), ""); sysctl_register_oid(&sysctl__kern_bootsignature); - - static SYSCTL_UINT(_kern, OID_AUTO, hibernatemode, - CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, - &gIOHibernateMode, 0, ""); sysctl_register_oid(&sysctl__kern_hibernatemode); } + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ static void hibernate_setup_for_wake(void) { -#if __ppc__ - // go slow (state needed for wake) - ml_set_processor_speed(1); -#endif } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #define C_ASSERT(e) typedef char __C_ASSERT__[(e) ? 1 : -1] -extern "C" boolean_t +static bool +no_encrypt_page(vm_offset_t ppnum) +{ + if (pmap_is_noencrypt((ppnum_t)ppnum) == TRUE) + { + return true; + } + return false; +} + +uint32_t wired_pages_encrypted = 0; +uint32_t dirty_pages_encrypted = 0; +uint32_t wired_pages_clear = 0; + +static void +hibernate_pal_callback(void *vars_arg, vm_offset_t addr) +{ + IOHibernateVars *vars = (IOHibernateVars *)vars_arg; + /* Make sure it's not in either of the save lists */ + hibernate_set_page_state(vars->page_list, vars->page_list_wired, atop_64(addr), 1, kIOHibernatePageStateFree); + + /* Set it in the bitmap of pages owned by the PAL */ + hibernate_page_bitset(vars->page_list_pal, TRUE, atop_64(addr)); +} + +static struct hibernate_cryptvars_t *local_cryptvars; + +extern "C" int +hibernate_pal_write(void *buffer, size_t size) +{ + IOHibernateVars * vars = &gIOHibernateVars; + + IOReturn err = IOPolledFileWrite(vars->fileVars, (const uint8_t *)buffer, size, local_cryptvars); + if (kIOReturnSuccess != err) { + kprintf("epic hibernate fail! %d\n", err); + return err; + } + + return 0; +} + + +extern "C" uint32_t hibernate_write_image(void) { IOHibernateImageHeader * header = gIOHibernateCurrentHeader; @@ -1467,34 +1875,51 @@ hibernate_write_image(void) uint32_t pageCount, pagesDone; IOReturn err; - vm_offset_t ppnum; - IOItemCount page, count; + vm_offset_t ppnum, page; + IOItemCount count; uint8_t * src; uint8_t * data; IOByteCount pageCompressedSize; uint64_t compressedSize, uncompressedSize; uint64_t image1Size = 0; uint32_t bitmap_size; - bool iterDone, pollerOpen, needEncryptStart; + bool iterDone, pollerOpen, needEncrypt; uint32_t restore1Sum, sum, sum1, sum2; uint32_t tag; uint32_t pageType; uint32_t pageAndCount[2]; + addr64_t phys64; + IOByteCount segLen; AbsoluteTime startTime, endTime; - AbsoluteTime allTime, compTime, decoTime; + AbsoluteTime allTime, compTime; + uint64_t compBytes; uint64_t nsec; uint32_t lastProgressStamp = 0; uint32_t progressStamp; + uint32_t blob, lastBlob = (uint32_t) -1L; hibernate_cryptvars_t _cryptvars; hibernate_cryptvars_t * cryptvars = 0; + wired_pages_encrypted = 0; + dirty_pages_encrypted = 0; + wired_pages_clear = 0; + if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents) return (false /* sleep */ ); + if (kIOHibernateModeSleep & gIOHibernateMode) + kdebug_enable = save_kdebug_enable; + + KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START, 0, 0, 0, 0, 0); + IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate); + restore1Sum = sum1 = sum2 = 0; + hibernate_pal_prepare(); + +#if CRYPTO // encryption data. "iv" is the "initial vector". if (kIOHibernateModeEncrypt & gIOHibernateMode) { @@ -1513,6 +1938,9 @@ hibernate_write_image(void) cryptvars = &_cryptvars; bzero(cryptvars, sizeof(hibernate_cryptvars_t)); + for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++) + vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount]; + bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey)); aes_encrypt_key(vars->wiredCryptKey, kIOHibernateAESKeySize, &cryptvars->ctx.encrypt); @@ -1520,13 +1948,16 @@ hibernate_write_image(void) bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE); bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey)); bzero(&vars->cryptKey[0], sizeof(vars->cryptKey)); - bzero(gIOHibernateCryptWakeVars, sizeof(hibernate_cryptwakevars_t)); + + local_cryptvars = cryptvars; } +#endif /* CRYPTO */ hibernate_setup_for_wake(); hibernate_page_list_setall(vars->page_list, vars->page_list_wired, + vars->page_list_pal, &pageCount); HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount); @@ -1543,19 +1974,18 @@ hibernate_write_image(void) } #endif - needEncryptStart = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode)); - + needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode)); AbsoluteTime_to_scalar(&compTime) = 0; - AbsoluteTime_to_scalar(&decoTime) = 0; + compBytes = 0; clock_get_uptime(&allTime); + IOService::getPMRootDomain()->pmStatsRecordEvent( + kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime); do { compressedSize = 0; uncompressedSize = 0; - iterDone = false; - pageType = 0; // wired pages first IOPolledFileSeek(vars->fileVars, sizeof(IOHibernateImageHeader)); @@ -1579,31 +2009,49 @@ hibernate_write_image(void) break; } + uintptr_t hibernateBase; + uintptr_t hibernateEnd; + + hibernateBase = HIB_BASE; /* Defined in PAL headers */ + + hibernateEnd = (sectHIBB + sectSizeHIB); + // copy out restore1 code - - page = atop_32(sectHIBB); - count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page; - header->restore1CodePage = page; + + for (count = 0; + (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone)); + count += segLen) + { + for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++) + { + gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone; + } + } + + page = atop_32(kvtophys(hibernateBase)); + count = atop_32(round_page(hibernateEnd) - hibernateBase); + header->restore1CodePhysPage = page; + header->restore1CodeVirt = hibernateBase; header->restore1PageCount = count; - header->restore1CodeOffset = ((uint32_t) &hibernate_machine_entrypoint) - sectHIBB; - header->restore1StackOffset = ((uint32_t) &gIOHibernateRestoreStackEnd[0]) - 64 - sectHIBB; + header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase; + header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase; // sum __HIB sect, with zeros for the stack - src = (uint8_t *) trunc_page(sectHIBB); + src = (uint8_t *) trunc_page(hibernateBase); for (page = 0; page < count; page++) { if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0])) - restore1Sum += hibernate_sum(src, page_size); + restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page); else - restore1Sum += 0x10000001; + restore1Sum += 0x00000000; src += page_size; } sum1 = restore1Sum; // write the __HIB sect, with zeros for the stack - src = (uint8_t *) trunc_page(sectHIBB); - count = ((uint32_t) &gIOHibernateRestoreStack[0]) - trunc_page(sectHIBB); + src = (uint8_t *) trunc_page(hibernateBase); + count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase); if (count) { err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars); @@ -1617,7 +2065,7 @@ hibernate_write_image(void) if (kIOReturnSuccess != err) break; src = &gIOHibernateRestoreStackEnd[0]; - count = round_page(sectHIBB + sectSizeHIB) - ((uint32_t) src); + count = round_page(hibernateEnd) - ((uintptr_t) src); if (count) { err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars); @@ -1627,16 +2075,13 @@ hibernate_write_image(void) // write the preview buffer - addr64_t phys64; - IOByteCount segLen; - - if (vars->previewData) + if (vars->previewBuffer) { ppnum = 0; count = 0; do { - phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen); + phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone); pageAndCount[0] = atop_64(phys64); pageAndCount[1] = atop_32(segLen); err = IOPolledFileWrite(vars->fileVars, @@ -1651,15 +2096,17 @@ hibernate_write_image(void) if (kIOReturnSuccess != err) break; - src = (uint8_t *) vars->previewData->getBytesNoCopy(); - count = vars->previewData->getLength(); + src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment); + count = vars->previewBuffer->getLength(); header->previewPageListSize = ppnum; header->previewSize = count + ppnum; for (page = 0; page < count; page += page_size) - sum1 += hibernate_sum(src + page, page_size); - + { + phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone); + sum1 += hibernate_sum_page(src + page, atop_64(phys64)); + } err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars); if (kIOReturnSuccess != err) break; @@ -1668,7 +2115,7 @@ hibernate_write_image(void) // mark areas for no save for (count = 0; - (phys64 = vars->ioBuffer->getPhysicalSegment64(count, &segLen)); + (phys64 = vars->ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone)); count += segLen) { hibernate_set_page_state(vars->page_list, vars->page_list_wired, @@ -1678,7 +2125,7 @@ hibernate_write_image(void) } for (count = 0; - (phys64 = vars->srcBuffer->getPhysicalSegment64(count, &segLen)); + (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone)); count += segLen) { hibernate_set_page_state(vars->page_list, vars->page_list_wired, @@ -1700,15 +2147,16 @@ hibernate_write_image(void) hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount); - page = atop_32(sectHIBB); - count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page; + + page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase)); + count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page; hibernate_set_page_state(vars->page_list, vars->page_list_wired, page, count, kIOHibernatePageStateFree); pageCount -= count; if (vars->previewBuffer) for (count = 0; - (phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen)); + (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone)); count += segLen) { hibernate_set_page_state(vars->page_list, vars->page_list_wired, @@ -1717,130 +2165,201 @@ hibernate_write_image(void) pageCount -= atop_32(segLen); } + for (count = 0; + (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone)); + count += segLen) + { + hibernate_set_page_state(vars->page_list, vars->page_list_wired, + atop_64(phys64), atop_32(segLen), + kIOHibernatePageStateFree); + pageCount -= atop_32(segLen); + } + + (void)hibernate_pal_callback; + src = (uint8_t *) vars->srcBuffer->getBytesNoCopy(); - ppnum = 0; - pagesDone = 0; + pagesDone = 0; + lastBlob = 0; HIBLOG("writing %d pages\n", pageCount); - do + enum + // pageType + { + kWired = 0x02, + kEncrypt = 0x01, + kWiredEncrypt = kWired | kEncrypt, + kWiredClear = kWired, + kUnwiredEncrypt = kEncrypt + }; + + for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--) { - count = hibernate_page_list_iterate(pageType ? vars->page_list : vars->page_list_wired, - &ppnum); -// kprintf("[%d](%x : %x)\n", pageType, ppnum, count); - - iterDone = !count; - - pageAndCount[0] = ppnum; - pageAndCount[1] = count; - err = IOPolledFileWrite(vars->fileVars, - (const uint8_t *) &pageAndCount, sizeof(pageAndCount), - cryptvars); - if (kIOReturnSuccess != err) - break; - - for (page = 0; page < count; page++) + if (needEncrypt && (kEncrypt & pageType)) { - err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(ppnum), page_size); - if (err) + vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1)); + vars->fileVars->encryptEnd = UINT64_MAX; + HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart); + + if (kUnwiredEncrypt == pageType) { - HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%d] %x\n", __LINE__, ppnum, err); - break; + // start unwired image + bcopy(&cryptvars->aes_iv[0], + &gIOHibernateCryptWakeContext.aes_iv[0], + sizeof(cryptvars->aes_iv)); + cryptvars = &gIOHibernateCryptWakeContext; } + } + for (iterDone = false, ppnum = 0; !iterDone; ) + { + count = hibernate_page_list_iterate((kWired & pageType) + ? vars->page_list_wired : vars->page_list, + &ppnum); +// kprintf("[%d](%x : %x)\n", pageType, ppnum, count); + iterDone = !count; - sum = hibernate_sum(src, page_size); - - clock_get_uptime(&startTime); - - pageCompressedSize = WKdm_compress ((WK_word*) src, (WK_word*) (src + page_size), PAGE_SIZE_IN_WORDS); - - clock_get_uptime(&endTime); - ADD_ABSOLUTETIME(&compTime, &endTime); - SUB_ABSOLUTETIME(&compTime, &startTime); - - if (kIOHibernateModeEncrypt & gIOHibernateMode) - pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1); - - if (pageCompressedSize > page_size) + if (count && (kWired & pageType) && needEncrypt) { -// HIBLOG("------------lose: %d\n", pageCompressedSize); - pageCompressedSize = page_size; + uint32_t checkIndex; + for (checkIndex = 0; + (checkIndex < count) + && (((kEncrypt & pageType) == 0) == no_encrypt_page(ppnum + checkIndex)); + checkIndex++) + {} + if (!checkIndex) + { + ppnum++; + continue; + } + count = checkIndex; } - if (pageCompressedSize != page_size) - data = (src + page_size); - else - data = src; - - tag = pageCompressedSize | kIOHibernateTagSignature; - - if (pageType) - sum2 += sum; + switch (pageType) + { + case kWiredEncrypt: wired_pages_encrypted += count; break; + case kWiredClear: wired_pages_clear += count; break; + case kUnwiredEncrypt: dirty_pages_encrypted += count; break; + } + + if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */} else - sum1 += sum; - - if (needEncryptStart && (ppnum >= atop_32(sectDATAB))) { - // start encrypting partway into the data about to be written - vars->fileVars->encryptStart = (vars->fileVars->position + AES_BLOCK_SIZE - 1) - & ~(AES_BLOCK_SIZE - 1); - needEncryptStart = false; + pageAndCount[0] = ppnum; + pageAndCount[1] = count; + err = IOPolledFileWrite(vars->fileVars, + (const uint8_t *) &pageAndCount, sizeof(pageAndCount), + cryptvars); + if (kIOReturnSuccess != err) + break; } - - err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars); - if (kIOReturnSuccess != err) - break; - - err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars); - if (kIOReturnSuccess != err) - break; - - compressedSize += pageCompressedSize; - if (pageCompressedSize) - uncompressedSize += page_size; - ppnum++; - pagesDone++; - if (0 == (8191 & pagesDone)) + for (page = ppnum; page < (ppnum + count); page++) { + err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size); + if (err) + { + HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err); + break; + } + + sum = hibernate_sum_page(src, page); + if (kWired & pageType) + sum1 += sum; + else + sum2 += sum; + + clock_get_uptime(&startTime); + + pageCompressedSize = WKdm_compress ((WK_word*) src, (WK_word*) (src + page_size), PAGE_SIZE_IN_WORDS); + clock_get_uptime(&endTime); - SUB_ABSOLUTETIME(&endTime, &allTime); - absolutetime_to_nanoseconds(endTime, &nsec); - progressStamp = nsec / 750000000ULL; - if (progressStamp != lastProgressStamp) + ADD_ABSOLUTETIME(&compTime, &endTime); + SUB_ABSOLUTETIME(&compTime, &startTime); + compBytes += page_size; + + if (kIOHibernateModeEncrypt & gIOHibernateMode) + pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1); + + if (pageCompressedSize > page_size) { - lastProgressStamp = progressStamp; - HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount); +// HIBLOG("------------lose: %d\n", pageCompressedSize); + pageCompressedSize = page_size; + } + + if (pageCompressedSize != page_size) + data = (src + page_size); + else + data = src; + + tag = pageCompressedSize | kIOHibernateTagSignature; + err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars); + if (kIOReturnSuccess != err) + break; + + err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars); + if (kIOReturnSuccess != err) + break; + + compressedSize += pageCompressedSize; + if (pageCompressedSize) + uncompressedSize += page_size; + pagesDone++; + + if (vars->consoleMapping && (0 == (1023 & pagesDone))) + { + blob = ((pagesDone * kIOHibernateProgressCount) / pageCount); + if (blob != lastBlob) + { + ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob); + lastBlob = blob; + } + } + if (0 == (8191 & pagesDone)) + { + clock_get_uptime(&endTime); + SUB_ABSOLUTETIME(&endTime, &allTime); + absolutetime_to_nanoseconds(endTime, &nsec); + progressStamp = nsec / 750000000ULL; + if (progressStamp != lastProgressStamp) + { + lastProgressStamp = progressStamp; + HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount); + } } } + if (kIOReturnSuccess != err) + break; + ppnum = page; } + if (kIOReturnSuccess != err) break; - if (iterDone && !pageType) + + if ((kEncrypt & pageType)) + { + vars->fileVars->encryptEnd = (vars->fileVars->position + AES_BLOCK_SIZE - 1) + & ~(AES_BLOCK_SIZE - 1); + HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd); + } + + if (kWiredEncrypt != pageType) { + // end of image1/2 - fill to next block err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars); if (kIOReturnSuccess != err) break; - - iterDone = false; - pageType = 1; - ppnum = 0; + } + if (kWiredClear == pageType) + { + // end wired image + header->encryptStart = vars->fileVars->encryptStart; + header->encryptEnd = vars->fileVars->encryptEnd; image1Size = vars->fileVars->position; - if (cryptvars) - { - bcopy(&cryptvars->aes_iv[0], - &gIOHibernateCryptWakeContext.aes_iv[0], - sizeof(cryptvars->aes_iv)); - cryptvars = &gIOHibernateCryptWakeContext; - } - HIBLOG("image1Size %qd\n", image1Size); + HIBLOG("image1Size %qd, encryptStart1 %qx, End1 %qx\n", + image1Size, header->encryptStart, header->encryptEnd); } } - while (!iterDone); - if (kIOReturnSuccess != err) - break; - err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars); if (kIOReturnSuccess != err) break; @@ -1850,7 +2369,6 @@ hibernate_write_image(void) header->image1Size = image1Size; header->bitmapSize = bitmap_size; header->pageCount = pageCount; - header->encryptStart = vars->fileVars->encryptStart; header->restore1Sum = restore1Sum; header->image1Sum = sum1; @@ -1865,6 +2383,8 @@ hibernate_write_image(void) else header->fileExtentMapSize = sizeof(header->fileExtentMap); bcopy(&fileExtents[0], &header->fileExtentMap[0], count); + + header->deviceBase = vars->fileVars->block0; IOPolledFileSeek(vars->fileVars, 0); err = IOPolledFileWrite(vars->fileVars, @@ -1875,25 +2395,33 @@ hibernate_write_image(void) err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars); if (kIOReturnSuccess != err) break; - err = IOHibernatePollerIODone(vars->fileVars); + err = IOHibernatePollerIODone(vars->fileVars, true); if (kIOReturnSuccess != err) break; } while (false); clock_get_uptime(&endTime); + + IOService::getPMRootDomain()->pmStatsRecordEvent( + kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime); + SUB_ABSOLUTETIME(&endTime, &allTime); absolutetime_to_nanoseconds(endTime, &nsec); HIBLOG("all time: %qd ms, ", nsec / 1000000ULL); absolutetime_to_nanoseconds(compTime, &nsec); - HIBLOG("comp time: %qd ms, ", - nsec / 1000000ULL); + HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ", + compBytes, + nsec / 1000000ULL, + nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0); - absolutetime_to_nanoseconds(decoTime, &nsec); - HIBLOG("deco time: %qd ms, ", - nsec / 1000000ULL); + absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec); + HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ", + vars->fileVars->cryptBytes, + nsec / 1000000ULL, + nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0); HIBLOG("\nimage %qd, uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n", header->imageSize, @@ -1901,77 +2429,51 @@ hibernate_write_image(void) uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0, sum1, sum2); + HIBLOG("wired_pages_encrypted %d, wired_pages_clear %d, dirty_pages_encrypted %d\n", + wired_pages_encrypted, wired_pages_clear, dirty_pages_encrypted); + + if (vars->fileVars->io) + (void) IOHibernatePollerIODone(vars->fileVars, false); + if (pollerOpen) IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState); + if (vars->consoleMapping) + ProgressUpdate(gIOHibernateGraphicsInfo, + vars->consoleMapping, 0, kIOHibernateProgressCount); + HIBLOG("hibernate_write_image done(%x)\n", err); // should we come back via regular wake, set the state in memory. gIOHibernateState = kIOHibernateStateInactive; - if ((kIOReturnSuccess == err) && !(kIOHibernateModeSleep & gIOHibernateMode)) - return (true /* power down */ ); - else - return (false /* sleep */ ); -} - -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, + wired_pages_encrypted, wired_pages_clear, dirty_pages_encrypted, 0, 0); -DECLARE_IOHIBERNATEPROGRESSALPHA - -static void -ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select) -{ - uint32_t rowBytes, pixelShift; - uint32_t x, y; - int32_t blob, lastBlob; - uint32_t alpha, in, color, result; - uint8_t * out; - uint32_t saveindex[kIOHibernateProgressCount] = { 0 }; - - pixelShift = display->depth >> 4; - if (pixelShift < 1) - return; - - rowBytes = display->rowBytes; - - screen += ((display->width - - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1)) - + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes; - - lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1); - - screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift; - - for (y = 0; y < kIOHibernateProgressHeight; y++) + if (kIOReturnSuccess == err) { - out = screen + y * rowBytes; - for (blob = firstBlob; blob <= lastBlob; blob++) - { - color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray; - for (x = 0; x < kIOHibernateProgressWidth; x++) - { - alpha = gIOHibernateProgressAlpha[y][x]; - result = color; - if (alpha) - { - if (0xff != alpha) - { - in = display->progressSaveUnder[blob][saveindex[blob]++]; - result = ((255 - alpha) * in + alpha * result + 0xff) / 255; - } - if (1 == pixelShift) - { - result >>= 3; - *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16 - } - else - *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32 - } - out += (1 << pixelShift); - } - out += (kIOHibernateProgressSpacing << pixelShift); - } + if (kIOHibernateModeSleep & gIOHibernateMode) + { + return (kIOHibernatePostWriteSleep); + } + else if(kIOHibernateModeRestart & gIOHibernateMode) + { + return (kIOHibernatePostWriteRestart); + } + else + { + /* by default, power down */ + return (kIOHibernatePostWriteHalt); + } + } + else if (kIOReturnAborted == err) + { + return (kIOHibernatePostWriteWake); + } + else + { + /* on error, sleep */ + return (kIOHibernatePostWriteSleep); } } @@ -1983,7 +2485,10 @@ hibernate_machine_init(void) IOReturn err; uint32_t sum; uint32_t pagesDone; + uint32_t pagesRead = 0; + AbsoluteTime startTime, compTime; AbsoluteTime allTime, endTime; + uint64_t compBytes; uint64_t nsec; uint32_t lastProgressStamp = 0; uint32_t progressStamp; @@ -2011,17 +2516,73 @@ hibernate_machine_init(void) HIBPRINT("diag %x %x %x %x\n", gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1], - gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]); + gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]); - HIBPRINT("video %lx %ld %ld %ld\n", + HIBPRINT("video %x %d %d %d status %x\n", gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth, - gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height); + gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus); if ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode) hibernate_page_list_discard(vars->page_list); boot_args *args = (boot_args *) PE_state.bootArgs; + cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0; + + if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount) + panic("handoff overflow"); + + IOHibernateHandoff * handoff; + bool done = false; + bool foundCryptData = false; + + for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy(); + !done; + handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount]) + { +// HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount); + uint8_t * data = &handoff->data[0]; + switch (handoff->type) + { + case kIOHibernateHandoffTypeEnd: + done = true; + break; + + case kIOHibernateHandoffTypeGraphicsInfo: + bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo)); + break; + + case kIOHibernateHandoffTypeCryptVars: + if (cryptvars) + { + hibernate_cryptwakevars_t * + wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0]; + bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv)); + } + foundCryptData = true; + bzero(data, handoff->bytecount); + break; + + case kIOHibernateHandoffTypeMemoryMap: + hibernate_newruntime_map(data, handoff->bytecount, + gIOHibernateCurrentHeader->systemTableOffset); + break; + + case kIOHibernateHandoffTypeDeviceTree: + { +// DTEntry chosen = NULL; +// HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen)); + } + break; + + default: + done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000)); + break; + } + } + if (cryptvars && !foundCryptData) + panic("hibernate handoff"); + if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress && (args->Video.v_baseAddr == gIOHibernateGraphicsInfo->physicalAddress)) @@ -2034,21 +2595,11 @@ hibernate_machine_init(void) } uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy(); - - if (gIOHibernateWakeMapSize) - { - err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(gIOHibernateWakeMap), - gIOHibernateWakeMapSize); - if (kIOReturnSuccess == err) - hibernate_newruntime_map(src, gIOHibernateWakeMapSize, - gIOHibernateCurrentHeader->systemTableOffset); - gIOHibernateWakeMap = 0; - gIOHibernateWakeMapSize = 0; - } - uint32_t decoOffset; clock_get_uptime(&allTime); + AbsoluteTime_to_scalar(&compTime) = 0; + compBytes = 0; HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled()); err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0); @@ -2062,30 +2613,24 @@ hibernate_machine_init(void) IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size); - if (vars->videoMapping) + if (vars->videoMapSize) { lastBlob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount) / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition); ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, 0, lastBlob); } - cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0; - if (kIOHibernateModeEncrypt & gIOHibernateMode) - { - cryptvars = &gIOHibernateCryptWakeContext; - bcopy(&gIOHibernateCryptWakeVars->aes_iv[0], - &cryptvars->aes_iv[0], - sizeof(cryptvars->aes_iv)); - } - // kick off the read ahead vars->fileVars->io = false; vars->fileVars->bufferHalf = 0; vars->fileVars->bufferLimit = 0; vars->fileVars->lastRead = 0; + vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize; vars->fileVars->bufferOffset = vars->fileVars->bufferLimit; + vars->fileVars->cryptBytes = 0; + AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0; - IOPolledFileRead(vars->fileVars, 0, 0, cryptvars); + err = IOPolledFileRead(vars->fileVars, 0, 0, cryptvars); vars->fileVars->bufferOffset = vars->fileVars->bufferLimit; // -- @@ -2094,14 +2639,16 @@ hibernate_machine_init(void) uint32_t * header = (uint32_t *) src; sum = 0; - do + while (kIOReturnSuccess == err) { unsigned int count; unsigned int page; uint32_t tag; vm_offset_t ppnum, compressedSize; - IOPolledFileRead(vars->fileVars, src, 8, cryptvars); + err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars); + if (kIOReturnSuccess != err) + break; ppnum = header[0]; count = header[1]; @@ -2113,9 +2660,17 @@ hibernate_machine_init(void) for (page = 0; page < count; page++) { - IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars); + err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars); + if (kIOReturnSuccess != err) + break; compressedSize = kIOHibernateTagLength & tag; + if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength)) + { + err = kIOReturnIPCError; + break; + } + if (!compressedSize) { ppnum++; @@ -2123,26 +2678,39 @@ hibernate_machine_init(void) continue; } - IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars); - - if (compressedSize != page_size) + err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars); + if (kIOReturnSuccess != err) + break; + + if (compressedSize < page_size) { decoOffset = page_size; + + clock_get_uptime(&startTime); WKdm_decompress((WK_word*) src, (WK_word*) (src + decoOffset), PAGE_SIZE_IN_WORDS); + clock_get_uptime(&endTime); + ADD_ABSOLUTETIME(&compTime, &endTime); + SUB_ABSOLUTETIME(&compTime, &startTime); + + compBytes += page_size; } else decoOffset = 0; - sum += hibernate_sum((src + decoOffset), page_size); + sum += hibernate_sum_page((src + decoOffset), ppnum); err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size); if (err) - HIBLOG("IOMemoryDescriptorReadToPhysical [%d] %x\n", ppnum, err); + { + HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err); + break; + } ppnum++; pagesDone++; + pagesRead++; - if (vars->videoMapping && (0 == (255 & pagesDone))) + if (vars->videoMapSize && (0 == (1023 & pagesDone))) { blob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount) / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition); @@ -2168,25 +2736,49 @@ hibernate_machine_init(void) } } } - while (true); + if (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages) + err = kIOReturnLockedRead; + + if (kIOReturnSuccess != err) + panic("Hibernate restore error %x", err); gIOHibernateCurrentHeader->actualImage2Sum = sum; if (vars->fileVars->io) - (void) IOHibernatePollerIODone(vars->fileVars); + (void) IOHibernatePollerIODone(vars->fileVars, false); err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState); - if (vars->videoMapping) + if (vars->videoMapSize) ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount); clock_get_uptime(&endTime); + + IOService::getPMRootDomain()->pmStatsRecordEvent( + kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime); + IOService::getPMRootDomain()->pmStatsRecordEvent( + kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime); + SUB_ABSOLUTETIME(&endTime, &allTime); absolutetime_to_nanoseconds(endTime, &nsec); - HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %qd ms\n", + HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %qd ms, ", pagesDone, sum, nsec / 1000000ULL); + + absolutetime_to_nanoseconds(compTime, &nsec); + HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ", + compBytes, + nsec / 1000000ULL, + nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0); + + absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec); + HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n", + vars->fileVars->cryptBytes, + nsec / 1000000ULL, + nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0); + + KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_NONE, pagesRead, pagesDone, 0, 0, 0); } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */