X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/89b3af67bb32e691275bf6fa803d1834b2284115..8a3053a07cee346dca737a5670e546fd26a7c9d6:/iokit/Kernel/IOHibernateIO.cpp diff --git a/iokit/Kernel/IOHibernateIO.cpp b/iokit/Kernel/IOHibernateIO.cpp index a7621a2ce..851adac60 100644 --- a/iokit/Kernel/IOHibernateIO.cpp +++ b/iokit/Kernel/IOHibernateIO.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2004-2008 Apple Computer, Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -58,12 +58,12 @@ Sleep: by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits. The image header and block list are written. The header includes the second file extent so only the header block is needed to read the file, regardless of filesystem. - The kernel section "__HIB" is written uncompressed to the image. This section of code and data + The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data (only) is used to decompress the image during wake/boot. Some additional pages are removed from the bitmaps - the buffers used for hibernation. The bitmaps are written to the image. More areas are removed from the bitmaps (after they have been written to the image) - the - section "__HIB" pages and interrupt stack. + segment "__HIB" pages and interrupt stack. Each wired page is compressed and written and then each non-wired page. Compression and disk writes are in parallel. The image header is written to the start of the file and the polling driver closed. @@ -151,67 +151,83 @@ to restrict I/O ops. #include #include "IOPMPowerStateQueue.h" #include -#include +#include +#include #include #include #include #include // (FWRITE, ...) #include +#include #include #include #include #include "IOHibernateInternal.h" -#include "WKdm.h" +#include #include "IOKitKernelInternal.h" +#include -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +#include +#include +#include -OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject); - -OSMetaClassDefineReservedUnused(IOPolledInterface, 0); -OSMetaClassDefineReservedUnused(IOPolledInterface, 1); -OSMetaClassDefineReservedUnused(IOPolledInterface, 2); -OSMetaClassDefineReservedUnused(IOPolledInterface, 3); -OSMetaClassDefineReservedUnused(IOPolledInterface, 4); -OSMetaClassDefineReservedUnused(IOPolledInterface, 5); -OSMetaClassDefineReservedUnused(IOPolledInterface, 6); -OSMetaClassDefineReservedUnused(IOPolledInterface, 7); -OSMetaClassDefineReservedUnused(IOPolledInterface, 8); -OSMetaClassDefineReservedUnused(IOPolledInterface, 9); -OSMetaClassDefineReservedUnused(IOPolledInterface, 10); -OSMetaClassDefineReservedUnused(IOPolledInterface, 11); -OSMetaClassDefineReservedUnused(IOPolledInterface, 12); -OSMetaClassDefineReservedUnused(IOPolledInterface, 13); -OSMetaClassDefineReservedUnused(IOPolledInterface, 14); -OSMetaClassDefineReservedUnused(IOPolledInterface, 15); +extern "C" addr64_t kvtophys(vm_offset_t va); +extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va); /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +#define DISABLE_TRIM 0 +#define TRIM_DELAY 5000 + +extern unsigned int save_kdebug_enable; extern uint32_t gIOHibernateState; uint32_t gIOHibernateMode; static char gIOHibernateBootSignature[256+1]; static char gIOHibernateFilename[MAXPATHLEN+1]; -static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent) -uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms) +static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent) +uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms) +static uint64_t gIOHibernateCompression = 0x80; // default compression 50% static IODTNVRAM * gIOOptionsEntry; static IORegistryEntry * gIOChosenEntry; -#ifdef __i386__ +#if defined(__i386__) || defined(__x86_64__) static const OSSymbol * gIOCreateEFIDevicePathSymbol; +static const OSSymbol * gIOHibernateRTCVariablesKey; +static const OSSymbol * gIOHibernateBoot0082Key; +static const OSSymbol * gIOHibernateBootNextKey; +static OSData * gIOHibernateBoot0082Data; +static OSData * gIOHibernateBootNextData; +static OSObject * gIOHibernateBootNextSave; #endif +static IOLock * gFSLock; +static uint32_t gFSState; static IOPolledFileIOVars gFileVars; static IOHibernateVars gIOHibernateVars; static struct kern_direct_file_io_ref_t * gIOHibernateFileRef; static hibernate_cryptvars_t gIOHibernateCryptWakeContext; +static hibernate_graphics_t _hibernateGraphics; +static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics; +static hibernate_statistics_t _hibernateStats; +static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats; + +enum +{ + kFSIdle = 0, + kFSOpening = 2, + kFSOpened = 3, + kFSTimedOut = 4, +}; + +static IOReturn IOHibernateDone(IOHibernateVars * vars); /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ enum { kXPRamAudioVolume = 8 }; enum { kDefaultIOSize = 128 * 1024 }; -enum { kVideoMapSize = 32 * 1024 * 1024 }; +enum { kVideoMapSize = 80 * 1024 * 1024 }; #ifndef kIOMediaPreferredBlockSizeKey #define kIOMediaPreferredBlockSizeKey "Preferred Block Size" @@ -224,9 +240,6 @@ enum { kVideoMapSize = 32 * 1024 * 1024 }; #define kIOSelectedBootDeviceKey "boot-device" #endif - -enum { kIOHibernateMinPollersNeeded = 2 }; - /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ // copy from phys addr to MD @@ -243,7 +256,7 @@ IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md, addr64_t dstAddr64; IOByteCount dstLen; - dstAddr64 = md->getPhysicalSegment64(offset, &dstLen); + dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone); if (!dstAddr64) break; @@ -281,7 +294,7 @@ IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md, addr64_t srcAddr64; IOByteCount dstLen; - srcAddr64 = md->getPhysicalSegment64(offset, &dstLen); + srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone); if (!srcAddr64) break; @@ -468,28 +481,45 @@ IOHibernatePollerIO(IOPolledFileIOVars * vars, } static IOReturn -IOHibernatePollerIODone(IOPolledFileIOVars * vars) +IOHibernatePollerIODone(IOPolledFileIOVars * vars, bool abortable) { - IOReturn err = kIOReturnError; - int32_t idx; + IOReturn err = kIOReturnSuccess; + int32_t idx = 0; IOPolledInterface * poller; while (-1 == vars->ioStatus) { - for (idx = 0; - (poller = (IOPolledInterface *) vars->pollers->getObject(idx)); + for (idx = 0; + (poller = (IOPolledInterface *) vars->pollers->getObject(idx)); idx++) { - err = poller->checkForWork(); - if (err) - HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err); + IOReturn newErr; + newErr = poller->checkForWork(); + if ((newErr == kIOReturnAborted) && !abortable) + newErr = kIOReturnSuccess; + if (kIOReturnSuccess == err) + err = newErr; } } - if (kIOReturnSuccess != vars->ioStatus) - HIBLOG("IOPolledInterface::ioStatus 0x%x\n", vars->ioStatus); + if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort()) + { + err = kIOReturnAborted; + HIBLOG("IOPolledInterface::checkForWork sw abort\n"); + } + + if (err) + { + HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err); + } + else + { + err = vars->ioStatus; + if (kIOReturnSuccess != err) + HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err); + } - return (vars->ioStatus); + return (err); } IOReturn @@ -533,33 +563,77 @@ file_extent_callback(void * ref, uint64_t start, uint64_t length) extent.start = start; extent.length = length; + HIBLOG("[0x%qx, 0x%qx]\n", start, length); + ctx->extents->appendBytes(&extent, sizeof(extent)); ctx->size += length; } +static IOService * +IOCopyMediaForDev(dev_t device) +{ + OSDictionary * matching; + OSNumber * num; + OSIterator * iter; + IOService * result = 0; + + matching = IOService::serviceMatching("IOMedia"); + if (!matching) + return (0); + do + { + num = OSNumber::withNumber(major(device), 32); + if (!num) + break; + matching->setObject(kIOBSDMajorKey, num); + num->release(); + num = OSNumber::withNumber(minor(device), 32); + if (!num) + break; + matching->setObject(kIOBSDMinorKey, num); + num->release(); + if (!num) + break; + iter = IOService::getMatchingServices(matching); + if (iter) + { + result = (IOService *) iter->getNextObject(); + result->retain(); + iter->release(); + } + } + while (false); + matching->release(); + + return (result); +} + IOReturn -IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer, - IOPolledFileIOVars ** fileVars, OSData ** fileExtents, - OSData ** imagePath) +IOPolledFileOpen( const char * filename, uint64_t setFileSize, + IOBufferMemoryDescriptor * ioBuffer, + IOPolledFileIOVars ** fileVars, OSData ** fileExtents, + OSData ** imagePath, uint8_t * volumeCryptKey) { - IOReturn err = kIOReturnError; + IOReturn err = kIOReturnSuccess; IOPolledFileIOVars * vars; _OpenFileContext ctx; OSData * extentsData; OSNumber * num; - IORegistryEntry * part = 0; - OSDictionary * matching; - OSIterator * iter; + IOService * part = 0; + OSString * keyUUID = 0; + OSString * keyStoreUUID = 0; + dev_t block_dev; dev_t hibernate_image_dev; uint64_t maxiobytes; + AbsoluteTime startTime, endTime; + uint64_t nsec; + + vars = IONew(IOPolledFileIOVars, 1); + if (!vars) return (kIOReturnNoMemory); + bzero(vars, sizeof(*vars)); - vars = &gFileVars; do { - HIBLOG("sizeof(IOHibernateImageHeader) == %ld\n", sizeof(IOHibernateImageHeader)); - if (sizeof(IOHibernateImageHeader) != 512) - continue; - vars->io = false; vars->buffer = (uint8_t *) ioBuffer->getBytesNoCopy(); vars->bufferHalf = 0; @@ -567,67 +641,112 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer, vars->bufferSize = ioBuffer->getLength() >> 1; extentsData = OSData::withCapacity(32); - - ctx.extents = extentsData; + ctx.extents = extentsData; ctx.size = 0; + clock_get_uptime(&startTime); vars->fileRef = kern_open_file_for_direct_io(filename, &file_extent_callback, &ctx, + setFileSize, + // write file: + 0, (caddr_t) gIOHibernateCurrentHeader, + sizeof(IOHibernateImageHeader), + // results + &block_dev, &hibernate_image_dev, &vars->block0, - &maxiobytes); - if (!vars->fileRef) - { - err = kIOReturnNoSpace; - break; - } - HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx\n", filename, ctx.size, - vars->block0, maxiobytes); + &maxiobytes, + &vars->flags); +#if 0 + uint32_t msDelay = (131071 & random()); + HIBLOG("sleep %d\n", msDelay); + IOSleep(msDelay); +#endif + clock_get_uptime(&endTime); + SUB_ABSOLUTETIME(&endTime, &startTime); + absolutetime_to_nanoseconds(endTime, &nsec); + + if (!vars->fileRef) err = kIOReturnNoSpace; + + IOLockLock(gFSLock); + if (kFSOpening != gFSState) err = kIOReturnTimeout; + IOLockUnlock(gFSLock); + + HIBLOG("kern_open_file_for_direct_io(%d) took %qd ms\n", err, nsec / 1000000ULL); + if (kIOReturnSuccess != err) break; + + if (kIOHibernateModeSSDInvert & gIOHibernateMode) + vars->flags ^= kIOHibernateOptionSSD; + + HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx ssd %d\n", filename, ctx.size, + vars->block0, maxiobytes, kIOHibernateOptionSSD & vars->flags); if (ctx.size < 1*1024*1024) // check against image size estimate! { err = kIOReturnNoSpace; break; } - if (maxiobytes < vars->bufferSize) - vars->bufferSize = maxiobytes; + vars->fileSize = ctx.size; + if (maxiobytes < vars->bufferSize) vars->bufferSize = maxiobytes; vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy(); - - matching = IOService::serviceMatching("IOMedia"); - num = OSNumber::withNumber(major(hibernate_image_dev), 32); - matching->setObject(kIOBSDMajorKey, num); - num->release(); - num = OSNumber::withNumber(minor(hibernate_image_dev), 32); - matching->setObject(kIOBSDMinorKey, num); - num->release(); - iter = IOService::getMatchingServices(matching); - matching->release(); - if (iter) - { - part = (IORegistryEntry *) iter->getNextObject(); - part->retain(); - iter->release(); + + part = IOCopyMediaForDev(block_dev); + if (!part) + { + err = kIOReturnNotFound; + break; } - - int minor, major; - IORegistryEntry * next; - IORegistryEntry * child; - OSData * data; + err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false, + (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL); + if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID) + { +// IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy()); + uuid_t volumeKeyUUID; + aks_volume_key_t vek; + static IOService * sKeyStore; + static const OSSymbol * sAKSGetKey; + + if (!sAKSGetKey) + sAKSGetKey = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY); + if (!sKeyStore) + sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane); + if (sKeyStore) + err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID); + else + err = kIOReturnNoResources; + if (kIOReturnSuccess == err) + err = sKeyStore->callPlatformFunction(sAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL); + if (kIOReturnSuccess != err) + IOLog("volume key err 0x%x\n", err); + else + { + size_t bytes = (kIOHibernateAESKeySize / 8); + if (vek.key.keybytecount < bytes) + bytes = vek.key.keybytecount; + bcopy(&vek.key.keybytes[0], volumeCryptKey, bytes); + } + bzero(&vek, sizeof(vek)); + } + part->release(); - num = (OSNumber *) part->getProperty(kIOBSDMajorKey); - if (!num) - break; - major = num->unsigned32BitValue(); - num = (OSNumber *) part->getProperty(kIOBSDMinorKey); - if (!num) - break; - minor = num->unsigned32BitValue(); + part = IOCopyMediaForDev(hibernate_image_dev); + if (!part) + { + err = kIOReturnNotFound; + break; + } - hibernate_image_dev = makedev(major, minor); + IORegistryEntry * next; + IORegistryEntry * child; + IOService * service; + OSData * data; vars->pollers = OSArray::withCapacity(4); if (!vars->pollers) - break; + { + err = kIOReturnNoMemory; + break; + } vars->blockSize = 512; next = part; @@ -644,6 +763,11 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer, } else if ((poller = OSDynamicCast(IOPolledInterface, obj))) vars->pollers->setObject(poller); + + if ((service = OSDynamicCast(IOService, next)) + && service->getDeviceMemory() + && !vars->pollers->getCount()) break; + if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey)))) vars->blockSize = num->unsigned32BitValue(); child = next; @@ -651,18 +775,36 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer, while ((next = child->getParentEntry(gIOServicePlane)) && child->isParent(next, gIOServicePlane, true)); + if (vars->blockSize < 4096) vars->blockSize = 4096; + HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n", - major, minor, vars->blockSize, vars->pollers->getCount()); - if (vars->pollers->getCount() < kIOHibernateMinPollersNeeded) + major(hibernate_image_dev), minor(hibernate_image_dev), (long)vars->blockSize, + vars->pollers->getCount()); + + if (!vars->pollers->getCount()) + { + err = kIOReturnUnsupported; + continue; + } + if (vars->blockSize < sizeof(IOHibernateImageHeader)) + { + err = kIOReturnError; continue; + } err = IOHibernatePollerProbe(vars, (IOService *) part); - if (kIOReturnSuccess != err) - break; + if (kIOReturnSuccess != err) break; err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer); - if (kIOReturnSuccess != err) - break; + if (kIOReturnSuccess != err) break; + + vars->media = part; + next = part; + while (next) + { + next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue); + next = next->getParentEntry(gIOServicePlane); + } *fileVars = vars; *fileExtents = extentsData; @@ -671,18 +813,22 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer, if ((extentsData->getLength() >= sizeof(IOPolledFileExtent))) { - char str2[24]; + char str2[24 + sizeof(uuid_string_t) + 2]; -#if __i386__ +#if defined(__i386__) || defined(__x86_64__) if (!gIOCreateEFIDevicePathSymbol) gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath"); - sprintf(str2, "%qx", vars->extentMap[0]); + if (keyUUID) + snprintf(str2, sizeof(str2), "%qx:%s", + vars->extentMap[0].start, keyUUID->getCStringNoCopy()); + else + snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start); err = IOService::getPlatform()->callPlatformFunction( gIOCreateEFIDevicePathSymbol, false, - (void *) part, (void *) str2, (void *) true, - (void *) &data); + (void *) part, (void *) str2, + (void *) (uintptr_t) true, (void *) &data); #else char str1[256]; int len = sizeof(str1); @@ -691,7 +837,7 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer, err = kIOReturnNotFound; else { - sprintf(str2, ",%qx", vars->extentMap[0]); + snprintf(str2, sizeof(str2), ",%qx", vars->extentMap[0].start); // (strip the plane name) char * tail = strchr(str1, ':'); if (!tail) @@ -712,7 +858,10 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer, { HIBLOG("error 0x%x opening hibernation file\n", err); if (vars->fileRef) - kern_close_file_for_direct_io(vars->fileRef); + { + kern_close_file_for_direct_io(vars->fileRef, 0, 0, 0, 0, 0); + vars->fileRef = NULL; + } } if (part) @@ -730,8 +879,6 @@ IOPolledFileClose( IOPolledFileIOVars * vars ) vars->pollers->release(); } - gIOHibernateFileRef = vars->fileRef; - bzero(vars, sizeof(IOPolledFileIOVars)); return (kIOReturnSuccess); @@ -810,29 +957,45 @@ IOPolledFileWrite(IOPolledFileIOVars * vars, - vars->extentPosition + vars->currentExtent->start); uint32_t length = (vars->bufferOffset); - if (cryptvars && vars->encryptStart && (vars->position > vars->encryptStart)) +#if CRYPTO + if (cryptvars && vars->encryptStart + && (vars->position > vars->encryptStart) + && ((vars->position - length) < vars->encryptEnd)) { - uint32_t encryptLen, encryptStart; + AbsoluteTime startTime, endTime; + + uint64_t encryptLen, encryptStart; encryptLen = vars->position - vars->encryptStart; if (encryptLen > length) encryptLen = length; encryptStart = length - encryptLen; - + if (vars->position > vars->encryptEnd) + encryptLen -= (vars->position - vars->encryptEnd); + + clock_get_uptime(&startTime); + // encrypt the buffer aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart, &cryptvars->aes_iv[0], encryptLen / AES_BLOCK_SIZE, vars->buffer + vars->bufferHalf + encryptStart, &cryptvars->ctx.encrypt); + + clock_get_uptime(&endTime); + ADD_ABSOLUTETIME(&vars->cryptTime, &endTime); + SUB_ABSOLUTETIME(&vars->cryptTime, &startTime); + vars->cryptBytes += encryptLen; + // save initial vector for following encrypts bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE, &cryptvars->aes_iv[0], AES_BLOCK_SIZE); } +#endif /* CRYPTO */ if (vars->io) { - err = IOHibernatePollerIODone(vars); + err = IOHibernatePollerIODone(vars, true); if (kIOReturnSuccess != err) break; } @@ -898,11 +1061,11 @@ IOPolledFileRead(IOPolledFileIOVars * vars, vars->bufferOffset += copy; // vars->position += copy; - if (vars->bufferOffset == vars->bufferLimit) + if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd)) { if (vars->io) { - err = IOHibernatePollerIODone(vars); + err = IOHibernatePollerIODone(vars, false); if (kIOReturnSuccess != err) break; } @@ -911,9 +1074,9 @@ IOPolledFileRead(IOPolledFileIOVars * vars, if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position); - vars->position += vars->lastRead; + vars->position += vars->lastRead; vars->extentRemaining -= vars->lastRead; - vars->bufferLimit = vars->lastRead; + vars->bufferLimit = vars->lastRead; if (!vars->extentRemaining) { @@ -927,39 +1090,56 @@ if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", } } - if (vars->extentRemaining <= vars->bufferSize) - vars->lastRead = vars->extentRemaining; - else - vars->lastRead = vars->bufferSize; - + uint64_t length; + uint64_t lastReadLength = vars->lastRead; uint64_t offset = (vars->position - vars->extentPosition + vars->currentExtent->start); - uint64_t length = (vars->lastRead); + if (vars->extentRemaining <= vars->bufferSize) + length = vars->extentRemaining; + else + length = vars->bufferSize; + if ((length + vars->position) > vars->readEnd) + length = vars->readEnd - vars->position; + vars->lastRead = length; + if (length) + { //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length); - - err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length); - if (kIOReturnSuccess != err) - break; - vars->io = true; + err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length); + if (kIOReturnSuccess != err) + break; + vars->io = true; + } vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize; vars->bufferOffset = 0; +#if CRYPTO if (cryptvars) { uint8_t thisVector[AES_BLOCK_SIZE]; + AbsoluteTime startTime, endTime; + // save initial vector for following decrypts bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE); - bcopy(vars->buffer + vars->bufferHalf + vars->lastRead - AES_BLOCK_SIZE, + bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE, &cryptvars->aes_iv[0], AES_BLOCK_SIZE); + // decrypt the buffer + clock_get_uptime(&startTime); + aes_decrypt_cbc(vars->buffer + vars->bufferHalf, &thisVector[0], - vars->lastRead / AES_BLOCK_SIZE, + lastReadLength / AES_BLOCK_SIZE, vars->buffer + vars->bufferHalf, &cryptvars->ctx.decrypt); + + clock_get_uptime(&endTime); + ADD_ABSOLUTETIME(&vars->cryptTime, &endTime); + SUB_ABSOLUTETIME(&vars->cryptTime, &startTime); + vars->cryptBytes += lastReadLength; } +#endif /* CRYPTO */ } } while (size); @@ -968,7 +1148,7 @@ if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - + IOReturn IOHibernateSystemSleep(void) { @@ -977,42 +1157,32 @@ IOHibernateSystemSleep(void) OSObject * obj; OSString * str; OSNumber * num; + bool dsSSD, vmflush; + IOHibernateVars * vars; - IOHibernateVars * vars = &gIOHibernateVars; + gIOHibernateState = kIOHibernateStateInactive; - if (vars->fileVars && vars->fileVars->fileRef) - // already on the way down - return (kIOReturnSuccess); + if (!gIOChosenEntry) + gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane); - gIOHibernateState = kIOHibernateStateInactive; + gIOHibernateDebugFlags = 0; + if (kIOLogHibernate & gIOKitDebug) + gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs; - if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateModeKey))) + if (IOService::getPMRootDomain()->getHibernateSettings( + &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime)) { - if ((num = OSDynamicCast(OSNumber, obj))) - gIOHibernateMode = num->unsigned32BitValue(); if (kIOHibernateModeSleep & gIOHibernateMode) // default to discard clean for safe sleep gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive | kIOHibernateModeDiscardCleanActive); - - obj->release(); - } - if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeRatioKey))) - { - if ((num = OSDynamicCast(OSNumber, obj))) - gIOHibernateFreeRatio = num->unsigned32BitValue(); - obj->release(); - } - if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeTimeKey))) - { - if ((num = OSDynamicCast(OSNumber, obj))) - gIOHibernateFreeTime = num->unsigned32BitValue(); - obj->release(); } + if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey))) { if ((str = OSDynamicCast(OSString, obj))) - strcpy(gIOHibernateFilename, str->getCStringNoCopy()); + strlcpy(gIOHibernateFilename, str->getCStringNoCopy(), + sizeof(gIOHibernateFilename)); obj->release(); } @@ -1021,54 +1191,157 @@ IOHibernateSystemSleep(void) HIBLOG("hibernate image path: %s\n", gIOHibernateFilename); + vars = IONew(IOHibernateVars, 1); + if (!vars) return (kIOReturnNoMemory); + bzero(vars, sizeof(*vars)); + + IOLockLock(gFSLock); + if (kFSIdle != gFSState) + { + HIBLOG("hibernate file busy\n"); + IOLockUnlock(gFSLock); + IODelete(vars, IOHibernateVars, 1); + return (kIOReturnBusy); + } + gFSState = kFSOpening; + IOLockUnlock(gFSLock); + do { vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, - 4 * page_size, page_size); + 2 * page_size + WKdm_SCRATCH_BUF_SIZE, page_size); vars->ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, 2 * kDefaultIOSize, page_size); - if (!vars->srcBuffer || !vars->ioBuffer) + vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, + ptoa_64(gIOHibernateHandoffPageCount), page_size); + + if (!vars->srcBuffer || !vars->ioBuffer || !vars->handoffBuffer) { err = kIOReturnNoMemory; break; } - err = IOPolledFileOpen(gIOHibernateFilename, vars->ioBuffer, - &vars->fileVars, &vars->fileExtents, &data); + if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey))) + { + if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMinSize = num->unsigned64BitValue(); + obj->release(); + } + if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey))) + { + if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMaxSize = num->unsigned64BitValue(); + obj->release(); + } + + boolean_t encryptedswap = true; + uint32_t pageCount; + AbsoluteTime startTime, endTime; + uint64_t nsec; + + bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader)); + gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags; + gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature; + + vmflush = (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)); + uint64_t setFileSize = 0; + err = hibernate_alloc_page_lists(&vars->page_list, + &vars->page_list_wired, + &vars->page_list_pal); if (KERN_SUCCESS != err) - { - HIBLOG("IOPolledFileOpen(%x)\n", err); break; - } - if (vars->fileVars->fileRef) + + if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode)) { - // invalidate the image file - gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature; - int err = kern_write_file(vars->fileVars->fileRef, 0, - (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader)); - if (KERN_SUCCESS != err) - HIBLOG("kern_write_file(%d)\n", err); + hibernate_page_list_setall(vars->page_list, + vars->page_list_wired, + vars->page_list_pal, + true /* preflight */, + vmflush /* discard */, + &pageCount); + PE_Video consoleInfo; + bzero(&consoleInfo, sizeof(consoleInfo)); + IOService::getPlatform()->getConsoleInfo(&consoleInfo); + + // estimate: 6% increase in pages compressed + // screen preview 2 images compressed 50% + setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8) + + vars->page_list->list_size + + (consoleInfo.v_width * consoleInfo.v_height * 8); + enum { setFileRound = 1024*1024ULL }; + setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1)); + + HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n", + pageCount, (100ULL * gIOHibernateCompression) >> 8, + setFileSize, vars->fileMinSize); + + if (!(kIOHibernateModeFileResize & gIOHibernateMode) + && (setFileSize < vars->fileMinSize)) + { + setFileSize = vars->fileMinSize; + } } + + // open & invalidate the image file - bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader)); + err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, vars->ioBuffer, + &vars->fileVars, &vars->fileExtents, &data, + &vars->volumeCryptKey[0]); - boolean_t encryptedswap; - err = hibernate_setup(gIOHibernateCurrentHeader, - gIOHibernateFreeRatio, gIOHibernateFreeTime, - &vars->page_list, &vars->page_list_wired, &encryptedswap); if (KERN_SUCCESS != err) { - HIBLOG("hibernate_setup(%d)\n", err); + HIBLOG("IOPolledFileOpen(%x)\n", err); break; } - if (encryptedswap) + clock_get_uptime(&startTime); + err = hibernate_setup(gIOHibernateCurrentHeader, + gIOHibernateFreeRatio, gIOHibernateFreeTime, + vmflush, + vars->page_list, vars->page_list_wired, vars->page_list_pal); + clock_get_uptime(&endTime); + SUB_ABSOLUTETIME(&endTime, &startTime); + absolutetime_to_nanoseconds(endTime, &nsec); + HIBLOG("hibernate_setup(%d) took %qd ms\n", err, nsec / 1000000ULL); + + dsSSD = ((0 != (kIOHibernateOptionSSD & vars->fileVars->flags)) + && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey))); + if (dsSSD) + { + gIOHibernateCurrentHeader->options |= + kIOHibernateOptionSSD + | kIOHibernateOptionColor; + +#if defined(__i386__) || defined(__x86_64__) + if (!uuid_is_null(vars->volumeCryptKey) && + (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey))) + { + uintptr_t smcVars[2]; + smcVars[0] = sizeof(vars->volumeCryptKey); + smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0]; + + IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars)); + bzero(smcVars, sizeof(smcVars)); + } +#endif + } + else + { + gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress; + } + + + if (KERN_SUCCESS != err) + break; + + if (encryptedswap || !uuid_is_null(vars->volumeCryptKey)) gIOHibernateMode ^= kIOHibernateModeEncrypt; - vars->videoAllocSize = kVideoMapSize; - if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize)) - vars->videoMapping = 0; + if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options) + { + vars->videoAllocSize = kVideoMapSize; + if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize)) + vars->videoMapping = 0; + } // generate crypt keys for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++) @@ -1086,8 +1359,6 @@ IOHibernateSystemSleep(void) if (regEntry && !gIOOptionsEntry) regEntry->release(); } - if (!gIOChosenEntry) - gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane); if (gIOOptionsEntry) { @@ -1101,46 +1372,7 @@ IOHibernateSystemSleep(void) } data->release(); -#ifdef __ppc__ - size_t len; - char valueString[16]; - - vars->saveBootDevice = gIOOptionsEntry->copyProperty(kIOSelectedBootDeviceKey); - if (gIOChosenEntry) - { - OSData * bootDevice = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOBootPathKey)); - if (bootDevice) - { - sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey); - OSString * str2 = OSString::withCStringNoCopy((const char *) bootDevice->getBytesNoCopy()); - if (sym && str2) - gIOOptionsEntry->setProperty(sym, str2); - if (sym) - sym->release(); - if (str2) - str2->release(); - } - data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMemorySignatureKey)); - if (data) - { - vars->haveFastBoot = true; - - len = sprintf(valueString, "0x%lx", *((UInt32 *)data->getBytesNoCopy())); - data = OSData::withBytes(valueString, len + 1); - sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey); - if (sym && data) - gIOOptionsEntry->setProperty(sym, data); - if (sym) - sym->release(); - if (data) - data->release(); - } - data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey)); - if (data) - gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy()); - } -#endif /* __ppc__ */ -#ifdef __i386__ +#if defined(__i386__) || defined(__x86_64__) struct AppleRTCHibernateVars { uint8_t signature[4]; @@ -1179,8 +1411,26 @@ IOHibernateSystemSleep(void) } data = OSData::withBytes(&rtcVars, sizeof(rtcVars)); if (data) - { - IOService::getPMRootDomain()->setProperty(kIOHibernateRTCVariablesKey, data); + { + if (!gIOHibernateRTCVariablesKey) + gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey); + if (gIOHibernateRTCVariablesKey) + IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data); + + if( gIOOptionsEntry ) + { + if( gIOHibernateMode & kIOHibernateModeSwitch ) + { + const OSSymbol *sym; + sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSwitchVarsKey); + if( sym ) + { + gIOOptionsEntry->setProperty(sym, data); /* intentional insecure backup of rtc boot vars */ + sym->release(); + } + } + } + data->release(); } if (gIOChosenEntry) @@ -1188,8 +1438,49 @@ IOHibernateSystemSleep(void) data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey)); if (data) gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy()); + { + // set BootNext + + if (!gIOHibernateBoot0082Data) + { + data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path")); + if (data) + { + // AppleNVRAM_EFI_LOAD_OPTION + struct { + uint32_t Attributes; + uint16_t FilePathLength; + uint16_t Desc; + } loadOptionHeader; + loadOptionHeader.Attributes = 1; + loadOptionHeader.FilePathLength = data->getLength(); + loadOptionHeader.Desc = 0; + gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength); + if (gIOHibernateBoot0082Data) + { + gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader)); + gIOHibernateBoot0082Data->appendBytes(data); + } + } + } + if (!gIOHibernateBoot0082Key) + gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082"); + if (!gIOHibernateBootNextKey) + gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext"); + if (!gIOHibernateBootNextData) + { + uint16_t bits = 0x0082; + gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits)); + } + if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData) + { + gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey); + gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data); + gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData); + } + } } -#else /* !__i386__ */ +#else /* !i386 && !x86_64 */ if (kIOHibernateModeEncrypt & gIOHibernateMode) { data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey)); @@ -1200,7 +1491,7 @@ IOHibernateSystemSleep(void) sym->release(); if (data) data->release(); - if (gIOHibernateBootSignature[0]) + if (false && gIOHibernateBootSignature[0]) { data = OSData::withCapacity(16); sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey); @@ -1243,55 +1534,297 @@ IOHibernateSystemSleep(void) &newVolume, sizeof(newVolume)); } } -#endif /* !__i386__ */ +#endif /* !i386 && !x86_64 */ } // -- + } + while (false); + + IOLockLock(gFSLock); + if ((kIOReturnSuccess == err) && (kFSOpening == gFSState)) + { + gFSState = kFSOpened; + gIOHibernateVars = *vars; + gFileVars = *vars->fileVars; + gIOHibernateVars.fileVars = &gFileVars; + gIOHibernateFileRef = gFileVars.fileRef; gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature; gIOHibernateState = kIOHibernateStateHibernating; } - while (false); + else + { + HIBLOG("hibernate file close due timeout\n"); + if (vars->fileVars && vars->fileVars->fileRef) kern_close_file_for_direct_io(vars->fileVars->fileRef, 0, 0, 0, 0, 0); + IOHibernateDone(vars); + gFSState = kFSIdle; + } + IOLockUnlock(gFSLock); + + if (vars->fileVars) IODelete(vars->fileVars, IOPolledFileIOVars, 1); + IODelete(vars, IOHibernateVars, 1); return (err); } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -IOReturn -IOHibernateSystemHasSlept(void) +DECLARE_IOHIBERNATEPROGRESSALPHA + +static void +ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen) { - IOHibernateVars * vars = &gIOHibernateVars; + uint32_t rowBytes, pixelShift; + uint32_t x, y; + int32_t blob; + uint32_t alpha, in, color, result; + uint8_t * out; + uint32_t saveindex[kIOHibernateProgressCount] = { 0 }; - if ((vars->previewData = OSDynamicCast(OSData, - IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewBufferKey)))) + rowBytes = display->rowBytes; + pixelShift = display->depth >> 4; + if (pixelShift < 1) return; + + screen += ((display->width + - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1)) + + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes; + + for (y = 0; y < kIOHibernateProgressHeight; y++) { - vars->previewBuffer = IOMemoryDescriptor::withAddress( - (void *) vars->previewData->getBytesNoCopy(), - vars->previewData->getLength(), - kIODirectionInOut); - - if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare())) + out = screen + y * rowBytes; + for (blob = 0; blob < kIOHibernateProgressCount; blob++) { - vars->previewBuffer->release(); - vars->previewBuffer = 0; + color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray; + for (x = 0; x < kIOHibernateProgressWidth; x++) + { + alpha = gIOHibernateProgressAlpha[y][x]; + result = color; + if (alpha) + { + if (0xff != alpha) + { + if (1 == pixelShift) + { + in = *((uint16_t *)out) & 0x1f; // 16 + in = (in << 3) | (in >> 2); + } + else + in = *((uint32_t *)out) & 0xff; // 32 + saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in; + result = ((255 - alpha) * in + alpha * result + 0xff) >> 8; + } + if (1 == pixelShift) + { + result >>= 3; + *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16 + } + else + *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32 + } + out += (1 << pixelShift); + } + out += (kIOHibernateProgressSpacing << pixelShift); } - if (!vars->previewBuffer) - vars->previewData = 0; } - if (gIOOptionsEntry) - gIOOptionsEntry->sync(); - - return (kIOReturnSuccess); } -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -IOReturn -IOHibernateSystemWake(void) +static void +ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select) { - IOHibernateVars * vars = &gIOHibernateVars; - - hibernate_teardown(vars->page_list, vars->page_list_wired); + uint32_t rowBytes, pixelShift; + uint32_t x, y; + int32_t blob, lastBlob; + uint32_t alpha, in, color, result; + uint8_t * out; + uint32_t saveindex[kIOHibernateProgressCount] = { 0 }; + + pixelShift = display->depth >> 4; + if (pixelShift < 1) + return; + + rowBytes = display->rowBytes; + + screen += ((display->width + - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1)) + + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes; + + lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1); + + screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift; + + for (y = 0; y < kIOHibernateProgressHeight; y++) + { + out = screen + y * rowBytes; + for (blob = firstBlob; blob <= lastBlob; blob++) + { + color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray; + for (x = 0; x < kIOHibernateProgressWidth; x++) + { + alpha = gIOHibernateProgressAlpha[y][x]; + result = color; + if (alpha) + { + if (0xff != alpha) + { + in = display->progressSaveUnder[blob][saveindex[blob]++]; + result = ((255 - alpha) * in + alpha * result + 0xff) / 255; + } + if (1 == pixelShift) + { + result >>= 3; + *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16 + } + else + *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32 + } + out += (1 << pixelShift); + } + out += (kIOHibernateProgressSpacing << pixelShift); + } + } +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +IOReturn +IOHibernateIOKitSleep(void) +{ + IOReturn ret = kIOReturnSuccess; + IOLockLock(gFSLock); + if (kFSOpening == gFSState) + { + gFSState = kFSTimedOut; + HIBLOG("hibernate file open timed out\n"); + ret = kIOReturnTimeout; + } + IOLockUnlock(gFSLock); + return (ret); +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +IOReturn +IOHibernateSystemHasSlept(void) +{ + IOReturn ret = kIOReturnSuccess; + IOHibernateVars * vars = &gIOHibernateVars; + OSObject * obj = 0; + OSData * data; + + IOLockLock(gFSLock); + if ((kFSOpened != gFSState) && gIOHibernateMode) + { + ret = kIOReturnTimeout; + } + IOLockUnlock(gFSLock); + if (kIOReturnSuccess != ret) return (ret); + + if (gIOHibernateMode) obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey); + vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj); + if (obj && !vars->previewBuffer) + obj->release(); + + vars->consoleMapping = NULL; + if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare())) + { + vars->previewBuffer->release(); + vars->previewBuffer = 0; + } + + if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options) + && vars->previewBuffer + && (data = OSDynamicCast(OSData, + IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey)))) + { + UInt32 flags = *((UInt32 *)data->getBytesNoCopy()); + HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags); + + IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey); + + if (kIOHibernatePreviewUpdates & flags) + { + PE_Video consoleInfo; + hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo; + + IOService::getPlatform()->getConsoleInfo(&consoleInfo); + + graphicsInfo->width = consoleInfo.v_width; + graphicsInfo->height = consoleInfo.v_height; + graphicsInfo->rowBytes = consoleInfo.v_rowBytes; + graphicsInfo->depth = consoleInfo.v_depth; + vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr; + + HIBPRINT("video %p %d %d %d\n", + vars->consoleMapping, graphicsInfo->depth, + graphicsInfo->width, graphicsInfo->height); + if (vars->consoleMapping) + ProgressInit(graphicsInfo, vars->consoleMapping, + &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder)); + } + } + + if (gIOOptionsEntry) + gIOOptionsEntry->sync(); + + return (ret); +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +static DeviceTreeNode * +MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry) +{ + DeviceTreeNodeProperty * prop; + DeviceTreeNode * child; + IORegistryEntry * childRegEntry; + const char * nameProp; + unsigned int propLen, idx; + + prop = (DeviceTreeNodeProperty *) (entry + 1); + for (idx = 0; idx < entry->nProperties; idx++) + { + if (regEntry && (0 != strcmp("name", prop->name))) + { + regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length); +// HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length); + } + prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3)); + } + + child = (DeviceTreeNode *) prop; + for (idx = 0; idx < entry->nChildren; idx++) + { + if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen)) + panic("no name"); + childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL; +// HIBPRINT("%s == %p\n", nameProp, childRegEntry); + child = MergeDeviceTree(child, childRegEntry); + } + return (child); +} + +IOReturn +IOHibernateSystemWake(void) +{ + if (kFSOpened == gFSState) + { + IOHibernateDone(&gIOHibernateVars); + } + else + { + IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey); + IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey); + } + return (kIOReturnSuccess); +} + +static IOReturn +IOHibernateDone(IOHibernateVars * vars) +{ + IORegistryEntry * next; + + hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal); if (vars->videoMapping) { @@ -1300,7 +1833,7 @@ IOHibernateSystemWake(void) IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize); if (vars->videoAllocSize) // dealloc range - kmem_free(kernel_map, trunc_page_32(vars->videoMapping), vars->videoAllocSize); + kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize); } if (vars->previewBuffer) @@ -1309,77 +1842,120 @@ IOHibernateSystemWake(void) vars->previewBuffer = 0; } - if (vars->fileVars) + if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) { - IOPolledFileClose(vars->fileVars); + IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey, + gIOHibernateCurrentHeader->options, 32); + } + else + { + IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey); } - // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched - -#ifdef __ppc__ - OSData * data = OSData::withCapacity(4); - if (gIOOptionsEntry && data) + if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) + && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus)) { - const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey); - if (sym) - { - gIOOptionsEntry->setProperty(sym, data); - sym->release(); - } - sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey); - if (sym) - { - if (vars->saveBootDevice) - { - gIOOptionsEntry->setProperty(sym, vars->saveBootDevice); - vars->saveBootDevice->release(); - } - sym->release(); - } - sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey); - if (sym) - { - gIOOptionsEntry->setProperty(sym, data); - sym->release(); - } - sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey); - if (sym) - { - gIOOptionsEntry->removeProperty(sym); - sym->release(); - } + IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey, + &gIOHibernateGraphicsInfo->gfxStatus, + sizeof(gIOHibernateGraphicsInfo->gfxStatus)); + } + else + { + IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey); } - if (data) - data->release(); - if (gIOOptionsEntry) + if (vars->fileVars) { - if (!vars->haveFastBoot) + if ((next = vars->fileVars->media)) do { - // reset boot audio volume - IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform()); - if (platform) - platform->writeXPRAM(kXPRamAudioVolume, - &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume)); + next->removeProperty(kIOPolledInterfaceActiveKey); + next = next->getParentEntry(gIOServicePlane); } - - // sync now to hardware if the booter has not - if (kIOHibernateStateInactive == gIOHibernateState) - gIOOptionsEntry->sync(); - else - // just sync the variables in case a later panic syncs nvram (it won't sync variables) - gIOOptionsEntry->syncOFVariables(); + while (next); + IOPolledFileClose(vars->fileVars); } -#endif -#ifdef __i386__ - IOService::getPMRootDomain()->removeProperty(kIOHibernateRTCVariablesKey); + // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched + +#if defined(__i386__) || defined(__x86_64__) + IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey); + IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey); + + /* + * Hibernate variable is written to NVRAM on platforms in which RtcRam + * is not backed by coin cell. Remove Hibernate data from NVRAM. + */ + if (gIOOptionsEntry) { + + if (gIOHibernateRTCVariablesKey) { + if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) { + gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey); + } + } + + if (gIOHibernateBootNextKey) + { + if (gIOHibernateBootNextSave) + { + gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave); + gIOHibernateBootNextSave->release(); + gIOHibernateBootNextSave = NULL; + } + else + gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey); + } + if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) gIOOptionsEntry->sync(); + } #endif if (vars->srcBuffer) vars->srcBuffer->release(); if (vars->ioBuffer) vars->ioBuffer->release(); + bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0])); + if (vars->handoffBuffer) + { + if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) + { + IOHibernateHandoff * handoff; + bool done = false; + for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy(); + !done; + handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount]) + { + HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount); + uint8_t * data = &handoff->data[0]; + switch (handoff->type) + { + case kIOHibernateHandoffTypeEnd: + done = true; + break; + + case kIOHibernateHandoffTypeDeviceTree: + MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot()); + break; + + case kIOHibernateHandoffTypeKeyStore: +#if defined(__i386__) || defined(__x86_64__) + { + IOBufferMemoryDescriptor * + md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn); + if (md) + { + IOSetKeyStoreData(md); + } + } +#endif + break; + + default: + done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000)); + break; + } + } + } + vars->handoffBuffer->release(); + } if (vars->fileExtents) vars->fileExtents->release(); @@ -1393,23 +1969,83 @@ IOHibernateSystemWake(void) IOReturn IOHibernateSystemPostWake(void) { - if (gIOHibernateFileRef) + struct kern_direct_file_io_ref_t * fileRef; + + if (kFSOpened == gFSState) { - // invalidate the image file + // invalidate & close the image file gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature; - int err = kern_write_file(gIOHibernateFileRef, 0, - (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader)); - if (KERN_SUCCESS != err) - HIBLOG("kern_write_file(%d)\n", err); - - kern_close_file_for_direct_io(gIOHibernateFileRef); - gIOHibernateFileRef = 0; + if ((fileRef = gIOHibernateFileRef)) + { + gIOHibernateFileRef = 0; + IOSleep(TRIM_DELAY); + kern_close_file_for_direct_io(fileRef, +#if DISABLE_TRIM + 0, 0, 0, 0, 0); +#else + 0, (caddr_t) gIOHibernateCurrentHeader, + sizeof(IOHibernateImageHeader), + 0, + gIOHibernateCurrentHeader->imageSize); +#endif + } + gFSState = kFSIdle; } return (kIOReturnSuccess); } +bool IOHibernateWasScreenLocked(void) +{ + bool ret = false; + if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry) + { + OSData * + data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey)); + if (data) switch (*((uint32_t *)data->getBytesNoCopy())) + { + case kIOScreenLockLocked: + case kIOScreenLockFileVaultDialog: + ret = true; + break; + case kIOScreenLockNoLock: + case kIOScreenLockUnlocked: + default: + ret = false; + break; + } + } + return (ret); +} + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +SYSCTL_STRING(_kern, OID_AUTO, hibernatefile, + CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED, + gIOHibernateFilename, sizeof(gIOHibernateFilename), ""); +SYSCTL_STRING(_kern, OID_AUTO, bootsignature, + CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED, + gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), ""); +SYSCTL_UINT(_kern, OID_AUTO, hibernatemode, + CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED, + &gIOHibernateMode, 0, ""); +SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics, + CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED, + gIOHibernateStats, hibernate_statistics_t, ""); + +SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready, + CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY, + &gIOHibernateStats->graphicsReadyTime, 0, ""); +SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification, + CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY, + &gIOHibernateStats->wakeNotificationTime, 0, ""); +SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready, + CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY, + &gIOHibernateStats->lockScreenReadyTime, 0, ""); +SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready, + CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY, + &gIOHibernateStats->hidReadyTime, 0, ""); + + void IOHibernateSystemInit(IOPMrootDomain * rootDomain) { @@ -1420,43 +2056,74 @@ IOHibernateSystemInit(IOPMrootDomain * rootDomain) data->release(); } - if (PE_parse_boot_arg("hfile", gIOHibernateFilename)) + if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename))) gIOHibernateMode = kIOHibernateModeOn; else gIOHibernateFilename[0] = 0; - static SYSCTL_STRING(_kern, OID_AUTO, hibernatefile, - CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, - gIOHibernateFilename, sizeof(gIOHibernateFilename), ""); sysctl_register_oid(&sysctl__kern_hibernatefile); - - static SYSCTL_STRING(_kern, OID_AUTO, bootsignature, - CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, - gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), ""); sysctl_register_oid(&sysctl__kern_bootsignature); - - static SYSCTL_UINT(_kern, OID_AUTO, hibernatemode, - CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, - &gIOHibernateMode, 0, ""); sysctl_register_oid(&sysctl__kern_hibernatemode); + sysctl_register_oid(&sysctl__kern_hibernatestatistics); + sysctl_register_oid(&sysctl__kern_hibernategraphicsready); + sysctl_register_oid(&sysctl__kern_hibernatewakenotification); + sysctl_register_oid(&sysctl__kern_hibernatelockscreenready); + sysctl_register_oid(&sysctl__kern_hibernatehidready); + + gFSLock = IOLockAlloc(); } + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ static void hibernate_setup_for_wake(void) { -#if __ppc__ - // go slow (state needed for wake) - ml_set_processor_speed(1); -#endif } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #define C_ASSERT(e) typedef char __C_ASSERT__[(e) ? 1 : -1] -extern "C" boolean_t +static bool +no_encrypt_page(vm_offset_t ppnum) +{ + if (pmap_is_noencrypt((ppnum_t)ppnum) == TRUE) + { + return true; + } + return false; +} + +static void +hibernate_pal_callback(void *vars_arg, vm_offset_t addr) +{ + IOHibernateVars *vars = (IOHibernateVars *)vars_arg; + /* Make sure it's not in either of the save lists */ + hibernate_set_page_state(vars->page_list, vars->page_list_wired, atop_64(addr), 1, kIOHibernatePageStateFree); + + /* Set it in the bitmap of pages owned by the PAL */ + hibernate_page_bitset(vars->page_list_pal, TRUE, atop_64(addr)); +} + +static struct hibernate_cryptvars_t *local_cryptvars; + +extern "C" int +hibernate_pal_write(void *buffer, size_t size) +{ + IOHibernateVars * vars = &gIOHibernateVars; + + IOReturn err = IOPolledFileWrite(vars->fileVars, (const uint8_t *)buffer, size, local_cryptvars); + if (kIOReturnSuccess != err) { + kprintf("epic hibernate fail! %d\n", err); + return err; + } + + return 0; +} + + +extern "C" uint32_t hibernate_write_image(void) { IOHibernateImageHeader * header = gIOHibernateCurrentHeader; @@ -1467,34 +2134,61 @@ hibernate_write_image(void) uint32_t pageCount, pagesDone; IOReturn err; - vm_offset_t ppnum; - IOItemCount page, count; + vm_offset_t ppnum, page; + IOItemCount count; uint8_t * src; uint8_t * data; - IOByteCount pageCompressedSize; + uint8_t * compressed; + uint8_t * scratch; + void * zerosCompressed; + IOByteCount pageCompressedSize, zerosCompressedLen; uint64_t compressedSize, uncompressedSize; uint64_t image1Size = 0; uint32_t bitmap_size; - bool iterDone, pollerOpen, needEncryptStart; + bool iterDone, pollerOpen, needEncrypt; uint32_t restore1Sum, sum, sum1, sum2; + int wkresult; uint32_t tag; uint32_t pageType; uint32_t pageAndCount[2]; + addr64_t phys64; + IOByteCount segLen; AbsoluteTime startTime, endTime; - AbsoluteTime allTime, compTime, decoTime; + AbsoluteTime allTime, compTime; + uint64_t compBytes; uint64_t nsec; uint32_t lastProgressStamp = 0; uint32_t progressStamp; + uint32_t blob, lastBlob = (uint32_t) -1L; + + uint32_t wiredPagesEncrypted; + uint32_t dirtyPagesEncrypted; + uint32_t wiredPagesClear; + uint32_t zeroPageCount; hibernate_cryptvars_t _cryptvars; hibernate_cryptvars_t * cryptvars = 0; + wiredPagesEncrypted = 0; + dirtyPagesEncrypted = 0; + wiredPagesClear = 0; + zeroPageCount = 0; + if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents) return (false /* sleep */ ); + if (kIOHibernateModeSleep & gIOHibernateMode) + kdebug_enable = save_kdebug_enable; + + KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START, 0, 0, 0, 0, 0); + IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate); + restore1Sum = sum1 = sum2 = 0; + hibernate_pal_prepare(); + +#if CRYPTO // encryption data. "iv" is the "initial vector". if (kIOHibernateModeEncrypt & gIOHibernateMode) { @@ -1513,6 +2207,9 @@ hibernate_write_image(void) cryptvars = &_cryptvars; bzero(cryptvars, sizeof(hibernate_cryptvars_t)); + for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++) + vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount]; + bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey)); aes_encrypt_key(vars->wiredCryptKey, kIOHibernateAESKeySize, &cryptvars->ctx.encrypt); @@ -1520,13 +2217,20 @@ hibernate_write_image(void) bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE); bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey)); bzero(&vars->cryptKey[0], sizeof(vars->cryptKey)); - bzero(gIOHibernateCryptWakeVars, sizeof(hibernate_cryptwakevars_t)); + + local_cryptvars = cryptvars; } +#endif /* CRYPTO */ hibernate_setup_for_wake(); hibernate_page_list_setall(vars->page_list, vars->page_list_wired, + vars->page_list_pal, + false /* !preflight */, + /* discard_all */ + ((0 == (kIOHibernateModeSleep & gIOHibernateMode)) + && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))), &pageCount); HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount); @@ -1543,21 +2247,20 @@ hibernate_write_image(void) } #endif - needEncryptStart = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode)); - + needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode)); AbsoluteTime_to_scalar(&compTime) = 0; - AbsoluteTime_to_scalar(&decoTime) = 0; + compBytes = 0; clock_get_uptime(&allTime); - + IOService::getPMRootDomain()->pmStatsRecordEvent( + kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime); do { compressedSize = 0; uncompressedSize = 0; - iterDone = false; - pageType = 0; // wired pages first + zeroPageCount = 0; - IOPolledFileSeek(vars->fileVars, sizeof(IOHibernateImageHeader)); + IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize); HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled()); @@ -1579,31 +2282,49 @@ hibernate_write_image(void) break; } + uintptr_t hibernateBase; + uintptr_t hibernateEnd; + + hibernateBase = HIB_BASE; /* Defined in PAL headers */ + + hibernateEnd = (segHIBB + segSizeHIB); + // copy out restore1 code - - page = atop_32(sectHIBB); - count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page; - header->restore1CodePage = page; + + for (count = 0; + (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone)); + count += segLen) + { + for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++) + { + gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone; + } + } + + page = atop_32(kvtophys(hibernateBase)); + count = atop_32(round_page(hibernateEnd) - hibernateBase); + header->restore1CodePhysPage = page; + header->restore1CodeVirt = hibernateBase; header->restore1PageCount = count; - header->restore1CodeOffset = ((uint32_t) &hibernate_machine_entrypoint) - sectHIBB; - header->restore1StackOffset = ((uint32_t) &gIOHibernateRestoreStackEnd[0]) - 64 - sectHIBB; + header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase; + header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase; - // sum __HIB sect, with zeros for the stack - src = (uint8_t *) trunc_page(sectHIBB); + // sum __HIB seg, with zeros for the stack + src = (uint8_t *) trunc_page(hibernateBase); for (page = 0; page < count; page++) { if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0])) - restore1Sum += hibernate_sum(src, page_size); + restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page); else - restore1Sum += 0x10000001; + restore1Sum += 0x00000000; src += page_size; } sum1 = restore1Sum; - // write the __HIB sect, with zeros for the stack + // write the __HIB seg, with zeros for the stack - src = (uint8_t *) trunc_page(sectHIBB); - count = ((uint32_t) &gIOHibernateRestoreStack[0]) - trunc_page(sectHIBB); + src = (uint8_t *) trunc_page(hibernateBase); + count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase); if (count) { err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars); @@ -1617,7 +2338,7 @@ hibernate_write_image(void) if (kIOReturnSuccess != err) break; src = &gIOHibernateRestoreStackEnd[0]; - count = round_page(sectHIBB + sectSizeHIB) - ((uint32_t) src); + count = round_page(hibernateEnd) - ((uintptr_t) src); if (count) { err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars); @@ -1625,18 +2346,22 @@ hibernate_write_image(void) break; } - // write the preview buffer + if (kIOHibernateModeEncrypt & gIOHibernateMode) + { + vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1)); + vars->fileVars->encryptEnd = UINT64_MAX; + HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart); + } - addr64_t phys64; - IOByteCount segLen; + // write the preview buffer - if (vars->previewData) + if (vars->previewBuffer) { ppnum = 0; count = 0; do { - phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen); + phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone); pageAndCount[0] = atop_64(phys64); pageAndCount[1] = atop_32(segLen); err = IOPolledFileWrite(vars->fileVars, @@ -1651,15 +2376,20 @@ hibernate_write_image(void) if (kIOReturnSuccess != err) break; - src = (uint8_t *) vars->previewData->getBytesNoCopy(); - count = vars->previewData->getLength(); + src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment); + + ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime; + + count = vars->previewBuffer->getLength(); header->previewPageListSize = ppnum; header->previewSize = count + ppnum; for (page = 0; page < count; page += page_size) - sum1 += hibernate_sum(src + page, page_size); - + { + phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone); + sum1 += hibernate_sum_page(src + page, atop_64(phys64)); + } err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars); if (kIOReturnSuccess != err) break; @@ -1668,7 +2398,7 @@ hibernate_write_image(void) // mark areas for no save for (count = 0; - (phys64 = vars->ioBuffer->getPhysicalSegment64(count, &segLen)); + (phys64 = vars->ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone)); count += segLen) { hibernate_set_page_state(vars->page_list, vars->page_list_wired, @@ -1678,7 +2408,7 @@ hibernate_write_image(void) } for (count = 0; - (phys64 = vars->srcBuffer->getPhysicalSegment64(count, &segLen)); + (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone)); count += segLen) { hibernate_set_page_state(vars->page_list, vars->page_list_wired, @@ -1700,15 +2430,16 @@ hibernate_write_image(void) hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount); - page = atop_32(sectHIBB); - count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page; + + page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase)); + count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page; hibernate_set_page_state(vars->page_list, vars->page_list_wired, page, count, kIOHibernatePageStateFree); pageCount -= count; if (vars->previewBuffer) for (count = 0; - (phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen)); + (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone)); count += segLen) { hibernate_set_page_state(vars->page_list, vars->page_list_wired, @@ -1717,132 +2448,227 @@ hibernate_write_image(void) pageCount -= atop_32(segLen); } - src = (uint8_t *) vars->srcBuffer->getBytesNoCopy(); - - ppnum = 0; - pagesDone = 0; - - HIBLOG("writing %d pages\n", pageCount); - - do + for (count = 0; + (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone)); + count += segLen) { - count = hibernate_page_list_iterate(pageType ? vars->page_list : vars->page_list_wired, - &ppnum); -// kprintf("[%d](%x : %x)\n", pageType, ppnum, count); - - iterDone = !count; + hibernate_set_page_state(vars->page_list, vars->page_list_wired, + atop_64(phys64), atop_32(segLen), + kIOHibernatePageStateFree); + pageCount -= atop_32(segLen); + } - pageAndCount[0] = ppnum; - pageAndCount[1] = count; - err = IOPolledFileWrite(vars->fileVars, - (const uint8_t *) &pageAndCount, sizeof(pageAndCount), - cryptvars); - if (kIOReturnSuccess != err) - break; + (void)hibernate_pal_callback; - for (page = 0; page < count; page++) + src = (uint8_t *) vars->srcBuffer->getBytesNoCopy(); + compressed = src + page_size; + scratch = compressed + page_size; + + // compress a zero page + bzero(src, page_size); + zerosCompressed = vars->handoffBuffer->getBytesNoCopy(); + zerosCompressedLen = WKdm_compress_new((WK_word*) src, + (WK_word*) zerosCompressed, + (WK_word*) scratch, + page_size - 4); + + pagesDone = 0; + lastBlob = 0; + + HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n", + bitmap_size, header->previewSize, + pageCount, vars->fileVars->position); + + enum + // pageType + { + kWired = 0x02, + kEncrypt = 0x01, + kWiredEncrypt = kWired | kEncrypt, + kWiredClear = kWired, + kUnwiredEncrypt = kEncrypt + }; + + for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--) + { + if (kUnwiredEncrypt == pageType) + { + // start unwired image + if (kIOHibernateModeEncrypt & gIOHibernateMode) + { + vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1)); + vars->fileVars->encryptEnd = UINT64_MAX; + HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart); + } + bcopy(&cryptvars->aes_iv[0], + &gIOHibernateCryptWakeContext.aes_iv[0], + sizeof(cryptvars->aes_iv)); + cryptvars = &gIOHibernateCryptWakeContext; + } + for (iterDone = false, ppnum = 0; !iterDone; ) { - err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(ppnum), page_size); - if (err) + count = hibernate_page_list_iterate((kWired & pageType) + ? vars->page_list_wired : vars->page_list, + &ppnum); +// kprintf("[%d](%x : %x)\n", pageType, ppnum, count); + iterDone = !count; + + if (count && (kWired & pageType) && needEncrypt) { - HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%d] %x\n", __LINE__, ppnum, err); - break; + uint32_t checkIndex; + for (checkIndex = 0; + (checkIndex < count) + && (((kEncrypt & pageType) == 0) == no_encrypt_page(ppnum + checkIndex)); + checkIndex++) + {} + if (!checkIndex) + { + ppnum++; + continue; + } + count = checkIndex; } - - sum = hibernate_sum(src, page_size); - - clock_get_uptime(&startTime); - - pageCompressedSize = WKdm_compress ((WK_word*) src, (WK_word*) (src + page_size), PAGE_SIZE_IN_WORDS); - - clock_get_uptime(&endTime); - ADD_ABSOLUTETIME(&compTime, &endTime); - SUB_ABSOLUTETIME(&compTime, &startTime); - - if (kIOHibernateModeEncrypt & gIOHibernateMode) - pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1); - if (pageCompressedSize > page_size) + switch (pageType) { -// HIBLOG("------------lose: %d\n", pageCompressedSize); - pageCompressedSize = page_size; + case kWiredEncrypt: wiredPagesEncrypted += count; break; + case kWiredClear: wiredPagesClear += count; break; + case kUnwiredEncrypt: dirtyPagesEncrypted += count; break; } - - if (pageCompressedSize != page_size) - data = (src + page_size); - else - data = src; - - tag = pageCompressedSize | kIOHibernateTagSignature; - - if (pageType) - sum2 += sum; + + if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */} else - sum1 += sum; - - if (needEncryptStart && (ppnum >= atop_32(sectDATAB))) { - // start encrypting partway into the data about to be written - vars->fileVars->encryptStart = (vars->fileVars->position + AES_BLOCK_SIZE - 1) - & ~(AES_BLOCK_SIZE - 1); - needEncryptStart = false; + pageAndCount[0] = ppnum; + pageAndCount[1] = count; + err = IOPolledFileWrite(vars->fileVars, + (const uint8_t *) &pageAndCount, sizeof(pageAndCount), + cryptvars); + if (kIOReturnSuccess != err) + break; } + + for (page = ppnum; page < (ppnum + count); page++) + { + err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size); + if (err) + { + HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err); + break; + } + + sum = hibernate_sum_page(src, page); + if (kWired & pageType) + sum1 += sum; + else + sum2 += sum; + + clock_get_uptime(&startTime); + wkresult = WKdm_compress_new((WK_word*) src, + (WK_word*) compressed, + (WK_word*) scratch, + page_size - 4); - err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars); - if (kIOReturnSuccess != err) - break; + clock_get_uptime(&endTime); + ADD_ABSOLUTETIME(&compTime, &endTime); + SUB_ABSOLUTETIME(&compTime, &startTime); - err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars); - if (kIOReturnSuccess != err) - break; + compBytes += page_size; + pageCompressedSize = (-1 == wkresult) ? page_size : wkresult; - compressedSize += pageCompressedSize; - if (pageCompressedSize) + if ((pageCompressedSize == zerosCompressedLen) + && !bcmp(compressed, zerosCompressed, zerosCompressedLen)) + { + pageCompressedSize = 0; + zeroPageCount++; + } + + if (kIOHibernateModeEncrypt & gIOHibernateMode) + pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1); + + if (pageCompressedSize != page_size) + data = compressed; + else + data = src; + + tag = pageCompressedSize | kIOHibernateTagSignature; + err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars); + if (kIOReturnSuccess != err) + break; + + err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars); + if (kIOReturnSuccess != err) + break; + + compressedSize += pageCompressedSize; uncompressedSize += page_size; - ppnum++; - pagesDone++; + pagesDone++; - if (0 == (8191 & pagesDone)) - { - clock_get_uptime(&endTime); - SUB_ABSOLUTETIME(&endTime, &allTime); - absolutetime_to_nanoseconds(endTime, &nsec); - progressStamp = nsec / 750000000ULL; - if (progressStamp != lastProgressStamp) + if (vars->consoleMapping && (0 == (1023 & pagesDone))) + { + blob = ((pagesDone * kIOHibernateProgressCount) / pageCount); + if (blob != lastBlob) + { + ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob); + lastBlob = blob; + } + } + if (0 == (8191 & pagesDone)) { - lastProgressStamp = progressStamp; - HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount); + clock_get_uptime(&endTime); + SUB_ABSOLUTETIME(&endTime, &allTime); + absolutetime_to_nanoseconds(endTime, &nsec); + progressStamp = nsec / 750000000ULL; + if (progressStamp != lastProgressStamp) + { + lastProgressStamp = progressStamp; + HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount); + } } } + if (kIOReturnSuccess != err) + break; + ppnum = page; } + if (kIOReturnSuccess != err) break; - if (iterDone && !pageType) + + if ((kEncrypt & pageType) && vars->fileVars->encryptStart) + { + vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL); + HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd); + } + + if (kWiredEncrypt != pageType) { + // end of image1/2 - fill to next block err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars); if (kIOReturnSuccess != err) break; + } + if (kWiredClear == pageType) + { + // enlarge wired image for test +// err = IOPolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars); - iterDone = false; - pageType = 1; - ppnum = 0; + // end wired image + header->encryptStart = vars->fileVars->encryptStart; + header->encryptEnd = vars->fileVars->encryptEnd; image1Size = vars->fileVars->position; - if (cryptvars) - { - bcopy(&cryptvars->aes_iv[0], - &gIOHibernateCryptWakeContext.aes_iv[0], - sizeof(cryptvars->aes_iv)); - cryptvars = &gIOHibernateCryptWakeContext; - } - HIBLOG("image1Size %qd\n", image1Size); + HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n", + image1Size, header->encryptStart, header->encryptEnd); } } - while (!iterDone); - if (kIOReturnSuccess != err) - break; - err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars); if (kIOReturnSuccess != err) + { + if (kIOReturnOverrun == err) + { + // update actual compression ratio on not enough space + gIOHibernateCompression = (compressedSize << 8) / uncompressedSize; + } break; + } // Header: @@ -1850,11 +2676,14 @@ hibernate_write_image(void) header->image1Size = image1Size; header->bitmapSize = bitmap_size; header->pageCount = pageCount; - header->encryptStart = vars->fileVars->encryptStart; header->restore1Sum = restore1Sum; header->image1Sum = sum1; header->image2Sum = sum2; + header->sleepTime = gIOLastSleepTime.tv_sec; + + header->compression = (compressedSize << 8) / uncompressedSize; + gIOHibernateCompression = header->compression; count = vars->fileExtents->getLength(); if (count > sizeof(header->fileExtentMap)) @@ -1865,6 +2694,9 @@ hibernate_write_image(void) else header->fileExtentMapSize = sizeof(header->fileExtentMap); bcopy(&fileExtents[0], &header->fileExtentMap[0], count); + + header->deviceBase = vars->fileVars->block0; + header->deviceBlockSize = vars->fileVars->blockSize; IOPolledFileSeek(vars->fileVars, 0); err = IOPolledFileWrite(vars->fileVars, @@ -1875,103 +2707,84 @@ hibernate_write_image(void) err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars); if (kIOReturnSuccess != err) break; - err = IOHibernatePollerIODone(vars->fileVars); + err = IOHibernatePollerIODone(vars->fileVars, true); if (kIOReturnSuccess != err) break; } while (false); clock_get_uptime(&endTime); + + IOService::getPMRootDomain()->pmStatsRecordEvent( + kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime); + SUB_ABSOLUTETIME(&endTime, &allTime); absolutetime_to_nanoseconds(endTime, &nsec); - HIBLOG("all time: %qd ms, ", - nsec / 1000000ULL); + HIBLOG("all time: %qd ms, ", nsec / 1000000ULL); absolutetime_to_nanoseconds(compTime, &nsec); - HIBLOG("comp time: %qd ms, ", - nsec / 1000000ULL); - - absolutetime_to_nanoseconds(decoTime, &nsec); - HIBLOG("deco time: %qd ms, ", - nsec / 1000000ULL); - - HIBLOG("\nimage %qd, uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n", - header->imageSize, + HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ", + compBytes, + nsec / 1000000ULL, + nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0); + + absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec); + HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ", + vars->fileVars->cryptBytes, + nsec / 1000000ULL, + nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0); + + HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n", + header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize, uncompressedSize, atop_32(uncompressedSize), compressedSize, uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0, sum1, sum2); + HIBLOG("zeroPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n", + zeroPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted); + + if (vars->fileVars->io) + (void) IOHibernatePollerIODone(vars->fileVars, false); + if (pollerOpen) IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState); + if (vars->consoleMapping) + ProgressUpdate(gIOHibernateGraphicsInfo, + vars->consoleMapping, 0, kIOHibernateProgressCount); + HIBLOG("hibernate_write_image done(%x)\n", err); // should we come back via regular wake, set the state in memory. gIOHibernateState = kIOHibernateStateInactive; - if ((kIOReturnSuccess == err) && !(kIOHibernateModeSleep & gIOHibernateMode)) - return (true /* power down */ ); - else - return (false /* sleep */ ); -} - -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -DECLARE_IOHIBERNATEPROGRESSALPHA - -static void -ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select) -{ - uint32_t rowBytes, pixelShift; - uint32_t x, y; - int32_t blob, lastBlob; - uint32_t alpha, in, color, result; - uint8_t * out; - uint32_t saveindex[kIOHibernateProgressCount] = { 0 }; - - pixelShift = display->depth >> 4; - if (pixelShift < 1) - return; - - rowBytes = display->rowBytes; + KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, + wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted, 0, 0); - screen += ((display->width - - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1)) - + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes; - - lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1); - - screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift; - - for (y = 0; y < kIOHibernateProgressHeight; y++) + if (kIOReturnSuccess == err) { - out = screen + y * rowBytes; - for (blob = firstBlob; blob <= lastBlob; blob++) - { - color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray; - for (x = 0; x < kIOHibernateProgressWidth; x++) - { - alpha = gIOHibernateProgressAlpha[y][x]; - result = color; - if (alpha) - { - if (0xff != alpha) - { - in = display->progressSaveUnder[blob][saveindex[blob]++]; - result = ((255 - alpha) * in + alpha * result + 0xff) / 255; - } - if (1 == pixelShift) - { - result >>= 3; - *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16 - } - else - *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32 - } - out += (1 << pixelShift); - } - out += (kIOHibernateProgressSpacing << pixelShift); - } + if (kIOHibernateModeSleep & gIOHibernateMode) + { + return (kIOHibernatePostWriteSleep); + } + else if(kIOHibernateModeRestart & gIOHibernateMode) + { + return (kIOHibernatePostWriteRestart); + } + else + { + /* by default, power down */ + return (kIOHibernatePostWriteHalt); + } + } + else if (kIOReturnAborted == err) + { + return (kIOHibernatePostWriteWake); + } + else + { + /* on error, sleep */ + return (kIOHibernatePostWriteSleep); } } @@ -1983,15 +2796,18 @@ hibernate_machine_init(void) IOReturn err; uint32_t sum; uint32_t pagesDone; + uint32_t pagesRead = 0; + AbsoluteTime startTime, compTime; AbsoluteTime allTime, endTime; - uint64_t nsec; + AbsoluteTime startIOTime, endIOTime; + uint64_t nsec, nsecIO; + uint64_t compBytes; uint32_t lastProgressStamp = 0; uint32_t progressStamp; - uint64_t progressZeroPosition = 0; - uint32_t blob, lastBlob = (uint32_t) -1L; hibernate_cryptvars_t * cryptvars = 0; IOHibernateVars * vars = &gIOHibernateVars; + bzero(gIOHibernateStats, sizeof(hibernate_statistics_t)); if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents) return; @@ -1999,10 +2815,6 @@ hibernate_machine_init(void) sum = gIOHibernateCurrentHeader->actualImage1Sum; pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages; - HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, image1Size %qx, conflictCount %d, nextFree %x\n", - gIOHibernateState, pagesDone, sum, gIOHibernateCurrentHeader->image1Size, - gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree); - if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) { HIBLOG("regular wake\n"); @@ -2011,81 +2823,163 @@ hibernate_machine_init(void) HIBPRINT("diag %x %x %x %x\n", gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1], - gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]); - - HIBPRINT("video %lx %ld %ld %ld\n", - gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth, - gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height); + gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]); + +#define t40ms(x) (tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000) +#define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y); + tStat(booterStart, booterStart); + gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart, + tStat(booterDuration0, booterTime0); + tStat(booterDuration1, booterTime1); + tStat(booterDuration2, booterTime2); + tStat(booterDuration, booterTime); + tStat(booterConnectDisplayDuration, connectDisplayTime); + tStat(booterSplashDuration, splashTime); + tStat(trampolineDuration, trampolineTime); + + gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size; + gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize; + gIOHibernateStats->image1Pages = pagesDone; + + HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n", + gIOHibernateStats->booterStart, + gIOHibernateStats->smcStart, + gIOHibernateStats->booterDuration0, + gIOHibernateStats->booterDuration1, + gIOHibernateStats->booterDuration2, + gIOHibernateStats->booterDuration, + gIOHibernateStats->booterConnectDisplayDuration, + gIOHibernateStats->booterSplashDuration, + gIOHibernateStats->trampolineDuration); + + HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n", + gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size, + gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree); - if ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode) + if ((0 != (kIOHibernateModeSleep & gIOHibernateMode)) + && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))) + { hibernate_page_list_discard(vars->page_list); + } - boot_args *args = (boot_args *) PE_state.bootArgs; + cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0; + + if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount) + panic("handoff overflow"); - if (vars->videoMapping - && gIOHibernateGraphicsInfo->physicalAddress - && (args->Video.v_baseAddr == gIOHibernateGraphicsInfo->physicalAddress)) + IOHibernateHandoff * handoff; + bool done = false; + bool foundCryptData = false; + + for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy(); + !done; + handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount]) { - vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height - * gIOHibernateGraphicsInfo->rowBytes); - IOMapPages(kernel_map, - vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress, - vars->videoMapSize, kIOMapInhibitCache ); +// HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount); + uint8_t * data = &handoff->data[0]; + switch (handoff->type) + { + case kIOHibernateHandoffTypeEnd: + done = true; + break; + + case kIOHibernateHandoffTypeGraphicsInfo: + bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo)); + break; + + case kIOHibernateHandoffTypeCryptVars: + if (cryptvars) + { + hibernate_cryptwakevars_t * + wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0]; + bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv)); + } + foundCryptData = true; + bzero(data, handoff->bytecount); + break; + + case kIOHibernateHandoffTypeMemoryMap: + + clock_get_uptime(&allTime); + + hibernate_newruntime_map(data, handoff->bytecount, + gIOHibernateCurrentHeader->systemTableOffset); + + clock_get_uptime(&endTime); + + SUB_ABSOLUTETIME(&endTime, &allTime); + absolutetime_to_nanoseconds(endTime, &nsec); + + HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL); + + break; + + case kIOHibernateHandoffTypeDeviceTree: + { +// DTEntry chosen = NULL; +// HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen)); + } + break; + + default: + done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000)); + break; + } } + if (cryptvars && !foundCryptData) + panic("hibernate handoff"); - uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy(); + HIBPRINT("video %x %d %d %d status %x\n", + gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth, + gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus); - if (gIOHibernateWakeMapSize) + if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress) { - err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(gIOHibernateWakeMap), - gIOHibernateWakeMapSize); - if (kIOReturnSuccess == err) - hibernate_newruntime_map(src, gIOHibernateWakeMapSize, - gIOHibernateCurrentHeader->systemTableOffset); - gIOHibernateWakeMap = 0; - gIOHibernateWakeMapSize = 0; + vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height + * gIOHibernateGraphicsInfo->rowBytes); + if (vars->videoMapSize > vars->videoAllocSize) vars->videoMapSize = 0; + else + { + IOMapPages(kernel_map, + vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress, + vars->videoMapSize, kIOMapInhibitCache ); + } } - uint32_t decoOffset; + if (vars->videoMapSize) + ProgressUpdate(gIOHibernateGraphicsInfo, + (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount); + + uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy(); + uint8_t * compressed = src + page_size; + uint8_t * scratch = compressed + page_size; + uint32_t decoOffset; clock_get_uptime(&allTime); + AbsoluteTime_to_scalar(&compTime) = 0; + compBytes = 0; HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled()); err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0); - HIBLOG("IOHibernatePollerOpen(%x)\n", err); - - if (gIOHibernateCurrentHeader->previewSize) - progressZeroPosition = gIOHibernateCurrentHeader->previewSize - + gIOHibernateCurrentHeader->fileExtentMapSize - - sizeof(gIOHibernateCurrentHeader->fileExtentMap) - + ptoa_64(gIOHibernateCurrentHeader->restore1PageCount); + clock_get_uptime(&startIOTime); + endTime = startIOTime; + SUB_ABSOLUTETIME(&endTime, &allTime); + absolutetime_to_nanoseconds(endTime, &nsec); + HIBLOG("IOHibernatePollerOpen(%x) %qd ms\n", err, nsec / 1000000ULL); IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size); - if (vars->videoMapping) - { - lastBlob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount) - / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition); - ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, 0, lastBlob); - } - - cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0; - if (kIOHibernateModeEncrypt & gIOHibernateMode) - { - cryptvars = &gIOHibernateCryptWakeContext; - bcopy(&gIOHibernateCryptWakeVars->aes_iv[0], - &cryptvars->aes_iv[0], - sizeof(cryptvars->aes_iv)); - } - // kick off the read ahead vars->fileVars->io = false; vars->fileVars->bufferHalf = 0; vars->fileVars->bufferLimit = 0; vars->fileVars->lastRead = 0; + vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize; vars->fileVars->bufferOffset = vars->fileVars->bufferLimit; + vars->fileVars->cryptBytes = 0; + AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0; - IOPolledFileRead(vars->fileVars, 0, 0, cryptvars); + err = IOPolledFileRead(vars->fileVars, 0, 0, cryptvars); vars->fileVars->bufferOffset = vars->fileVars->bufferLimit; // -- @@ -2094,14 +2988,16 @@ hibernate_machine_init(void) uint32_t * header = (uint32_t *) src; sum = 0; - do + while (kIOReturnSuccess == err) { unsigned int count; unsigned int page; uint32_t tag; vm_offset_t ppnum, compressedSize; - IOPolledFileRead(vars->fileVars, src, 8, cryptvars); + err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars); + if (kIOReturnSuccess != err) + break; ppnum = header[0]; count = header[1]; @@ -2113,45 +3009,46 @@ hibernate_machine_init(void) for (page = 0; page < count; page++) { - IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars); + err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars); + if (kIOReturnSuccess != err) + break; compressedSize = kIOHibernateTagLength & tag; - if (!compressedSize) + if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength)) { - ppnum++; - pagesDone++; - continue; + err = kIOReturnIPCError; + break; } - IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars); - - if (compressedSize != page_size) - { - decoOffset = page_size; - WKdm_decompress((WK_word*) src, (WK_word*) (src + decoOffset), PAGE_SIZE_IN_WORDS); - } + if (!compressedSize) bzero_phys(ptoa_64(ppnum), page_size); else - decoOffset = 0; - - sum += hibernate_sum((src + decoOffset), page_size); + { + err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars); + if (kIOReturnSuccess != err) break; + if (compressedSize < page_size) + { + decoOffset = page_size; + clock_get_uptime(&startTime); + WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, page_size); + clock_get_uptime(&endTime); + ADD_ABSOLUTETIME(&compTime, &endTime); + SUB_ABSOLUTETIME(&compTime, &startTime); + compBytes += page_size; + } + else decoOffset = 0; - err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size); - if (err) - HIBLOG("IOMemoryDescriptorReadToPhysical [%d] %x\n", ppnum, err); + sum += hibernate_sum_page((src + decoOffset), ppnum); + err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size); + if (err) + { + HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err); + break; + } + } ppnum++; pagesDone++; - - if (vars->videoMapping && (0 == (255 & pagesDone))) - { - blob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount) - / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition); - if (blob != lastBlob) - { - ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, lastBlob, blob); - lastBlob = blob; - } - } + pagesRead++; if (0 == (8191 & pagesDone)) { @@ -2168,25 +3065,119 @@ hibernate_machine_init(void) } } } - while (true); + if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages)) + err = kIOReturnLockedRead; + + if (kIOReturnSuccess != err) + panic("Hibernate restore error %x", err); gIOHibernateCurrentHeader->actualImage2Sum = sum; + gIOHibernateCompression = gIOHibernateCurrentHeader->compression; if (vars->fileVars->io) - (void) IOHibernatePollerIODone(vars->fileVars); + (void) IOHibernatePollerIODone(vars->fileVars, false); - err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState); + clock_get_uptime(&endIOTime); - if (vars->videoMapping) - ProgressUpdate(gIOHibernateGraphicsInfo, - (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount); + err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState); clock_get_uptime(&endTime); + + IOService::getPMRootDomain()->pmStatsRecordEvent( + kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime); + IOService::getPMRootDomain()->pmStatsRecordEvent( + kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime); + SUB_ABSOLUTETIME(&endTime, &allTime); absolutetime_to_nanoseconds(endTime, &nsec); - HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %qd ms\n", - pagesDone, sum, nsec / 1000000ULL); + SUB_ABSOLUTETIME(&endIOTime, &startIOTime); + absolutetime_to_nanoseconds(endIOTime, &nsecIO); + + gIOHibernateStats->kernelImageReadDuration = nsec / 1000000ULL; + gIOHibernateStats->imagePages = pagesDone; + + HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ", + pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize, + nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0); + + absolutetime_to_nanoseconds(compTime, &nsec); + HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ", + compBytes, + nsec / 1000000ULL, + nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0); + + absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec); + HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n", + vars->fileVars->cryptBytes, + nsec / 1000000ULL, + nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0); + + KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_NONE, pagesRead, pagesDone, 0, 0, 0); } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +void IOHibernateSetWakeCapabilities(uint32_t capability) +{ + if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) + { + gIOHibernateStats->wakeCapability = capability; + + if (kIOPMSystemCapabilityGraphics & capability) + { + vm_compressor_do_warmup(); + } + + } +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +void IOHibernateSystemRestart(void) +{ + static uint8_t noteStore[32] __attribute__((aligned(32))); + IORegistryEntry * regEntry; + const OSSymbol * sym; + OSData * noteProp; + OSData * data; + uintptr_t * smcVars; + uint8_t * smcBytes; + size_t len; + addr64_t element; + + data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey)); + if (!data) return; + + smcVars = (typeof(smcVars)) data->getBytesNoCopy(); + smcBytes = (typeof(smcBytes)) smcVars[1]; + len = smcVars[0]; + if (len > sizeof(noteStore)) len = sizeof(noteStore); + noteProp = OSData::withCapacity(3 * sizeof(element)); + if (!noteProp) return; + element = len; + noteProp->appendBytes(&element, sizeof(element)); + element = crc32(0, smcBytes, len); + noteProp->appendBytes(&element, sizeof(element)); + + bcopy(smcBytes, noteStore, len); + element = (addr64_t) ¬eStore[0]; + element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element)); + noteProp->appendBytes(&element, sizeof(element)); + + if (!gIOOptionsEntry) + { + regEntry = IORegistryEntry::fromPath("/options", gIODTPlane); + gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry); + if (regEntry && !gIOOptionsEntry) + regEntry->release(); + } + + sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey); + if (gIOOptionsEntry && sym) gIOOptionsEntry->setProperty(sym, noteProp); + if (noteProp) noteProp->release(); + if (sym) sym->release(); +} + + +