X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/0c530ab8987f0ae6a1a3d9284f40182b88852816..7e4a7d3939db04e70062ae6c7bf24b8c8b2f5a7c:/iokit/Kernel/IOHibernateIO.cpp diff --git a/iokit/Kernel/IOHibernateIO.cpp b/iokit/Kernel/IOHibernateIO.cpp index e98692e32..9abb3f6ca 100644 --- a/iokit/Kernel/IOHibernateIO.cpp +++ b/iokit/Kernel/IOHibernateIO.cpp @@ -1,23 +1,29 @@ /* - * Copyright (c) 2004 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2004-2008 Apple Computer, Inc. All rights reserved. * - * @APPLE_LICENSE_HEADER_START@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * - * The contents of this file constitute Original Code as defined in and - * are subject to the Apple Public Source License Version 1.1 (the - * "License"). You may not use this file except in compliance with the - * License. Please obtain a copy of the License at - * http://www.apple.com/publicsource and read it before using this file. + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. * - * This Original Code and all software distributed under the License are - * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the - * License for the specific language governing rights and limitations - * under the License. + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. * - * @APPLE_LICENSE_HEADER_END@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ @@ -192,7 +198,7 @@ uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (m static IODTNVRAM * gIOOptionsEntry; static IORegistryEntry * gIOChosenEntry; -#ifdef __i386__ +#if defined(__i386__) || defined(__x86_64__) static const OSSymbol * gIOCreateEFIDevicePathSymbol; #endif @@ -237,7 +243,7 @@ IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md, addr64_t dstAddr64; IOByteCount dstLen; - dstAddr64 = md->getPhysicalSegment64(offset, &dstLen); + dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone); if (!dstAddr64) break; @@ -275,7 +281,7 @@ IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md, addr64_t srcAddr64; IOByteCount dstLen; - srcAddr64 = md->getPhysicalSegment64(offset, &dstLen); + srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone); if (!srcAddr64) break; @@ -462,28 +468,39 @@ IOHibernatePollerIO(IOPolledFileIOVars * vars, } static IOReturn -IOHibernatePollerIODone(IOPolledFileIOVars * vars) +IOHibernatePollerIODone(IOPolledFileIOVars * vars, bool abortable) { - IOReturn err = kIOReturnError; - int32_t idx; + IOReturn err = kIOReturnSuccess; + int32_t idx = 0; IOPolledInterface * poller; while (-1 == vars->ioStatus) { - for (idx = 0; - (poller = (IOPolledInterface *) vars->pollers->getObject(idx)); + for (idx = 0; + (poller = (IOPolledInterface *) vars->pollers->getObject(idx)); idx++) { - err = poller->checkForWork(); - if (err) - HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err); + IOReturn newErr; + newErr = poller->checkForWork(); + if ((newErr == kIOReturnAborted) && !abortable) + newErr = kIOReturnSuccess; + if (kIOReturnSuccess == err) + err = newErr; } } - if (kIOReturnSuccess != vars->ioStatus) - HIBLOG("IOPolledInterface::ioStatus 0x%x\n", vars->ioStatus); + if (err) + { + HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err); + } + else + { + err = vars->ioStatus; + if (kIOReturnSuccess != err) + HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err); + } - return (vars->ioStatus); + return (err); } IOReturn @@ -574,6 +591,7 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer, err = kIOReturnNoSpace; break; } + gIOHibernateFileRef = vars->fileRef; HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx\n", filename, ctx.size, vars->block0, maxiobytes); if (ctx.size < 1*1024*1024) // check against image size estimate! @@ -602,7 +620,9 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer, part->retain(); iter->release(); } - + if (!part) + break; + int minor, major; IORegistryEntry * next; IORegistryEntry * child; @@ -646,7 +666,7 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer, && child->isParent(next, gIOServicePlane, true)); HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n", - major, minor, vars->blockSize, vars->pollers->getCount()); + major, minor, (long)vars->blockSize, vars->pollers->getCount()); if (vars->pollers->getCount() < kIOHibernateMinPollersNeeded) continue; @@ -667,11 +687,11 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer, { char str2[24]; -#if __i386__ +#if defined(__i386__) || defined(__x86_64__) if (!gIOCreateEFIDevicePathSymbol) gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath"); - sprintf(str2, "%qx", vars->extentMap[0]); + snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start); err = IOService::getPlatform()->callPlatformFunction( gIOCreateEFIDevicePathSymbol, false, @@ -685,7 +705,7 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer, err = kIOReturnNotFound; else { - sprintf(str2, ",%qx", vars->extentMap[0]); + snprintf(str2, sizeof(str2), ",%qx", vars->extentMap[0].start); // (strip the plane name) char * tail = strchr(str1, ':'); if (!tail) @@ -706,7 +726,10 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer, { HIBLOG("error 0x%x opening hibernation file\n", err); if (vars->fileRef) + { kern_close_file_for_direct_io(vars->fileRef); + gIOHibernateFileRef = vars->fileRef = NULL; + } } if (part) @@ -724,8 +747,6 @@ IOPolledFileClose( IOPolledFileIOVars * vars ) vars->pollers->release(); } - gIOHibernateFileRef = vars->fileRef; - bzero(vars, sizeof(IOPolledFileIOVars)); return (kIOReturnSuccess); @@ -804,6 +825,7 @@ IOPolledFileWrite(IOPolledFileIOVars * vars, - vars->extentPosition + vars->currentExtent->start); uint32_t length = (vars->bufferOffset); +#if CRYPTO if (cryptvars && vars->encryptStart && (vars->position > vars->encryptStart)) { uint32_t encryptLen, encryptStart; @@ -823,10 +845,11 @@ IOPolledFileWrite(IOPolledFileIOVars * vars, &cryptvars->aes_iv[0], AES_BLOCK_SIZE); } +#endif /* CRYPTO */ if (vars->io) { - err = IOHibernatePollerIODone(vars); + err = IOHibernatePollerIODone(vars, true); if (kIOReturnSuccess != err) break; } @@ -896,7 +919,7 @@ IOPolledFileRead(IOPolledFileIOVars * vars, { if (vars->io) { - err = IOHibernatePollerIODone(vars); + err = IOHibernatePollerIODone(vars, false); if (kIOReturnSuccess != err) break; } @@ -921,14 +944,15 @@ if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", } } - if (vars->extentRemaining <= vars->bufferSize) - vars->lastRead = vars->extentRemaining; - else - vars->lastRead = vars->bufferSize; - + uint64_t length; + uint64_t lastReadLength = vars->lastRead; uint64_t offset = (vars->position - vars->extentPosition + vars->currentExtent->start); - uint64_t length = (vars->lastRead); + if (vars->extentRemaining <= vars->bufferSize) + length = vars->extentRemaining; + else + length = vars->bufferSize; + vars->lastRead = length; //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length); @@ -940,20 +964,22 @@ if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize; vars->bufferOffset = 0; +#if CRYPTO if (cryptvars) { uint8_t thisVector[AES_BLOCK_SIZE]; // save initial vector for following decrypts bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE); - bcopy(vars->buffer + vars->bufferHalf + vars->lastRead - AES_BLOCK_SIZE, + bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE, &cryptvars->aes_iv[0], AES_BLOCK_SIZE); // decrypt the buffer aes_decrypt_cbc(vars->buffer + vars->bufferHalf, &thisVector[0], - vars->lastRead / AES_BLOCK_SIZE, + lastReadLength / AES_BLOCK_SIZE, vars->buffer + vars->bufferHalf, &cryptvars->ctx.decrypt); } +#endif /* CRYPTO */ } } while (size); @@ -971,6 +997,7 @@ IOHibernateSystemSleep(void) OSObject * obj; OSString * str; OSNumber * num; + OSDictionary *sleepOverrideOptions; IOHibernateVars * vars = &gIOHibernateVars; @@ -980,23 +1007,57 @@ IOHibernateSystemSleep(void) gIOHibernateState = kIOHibernateStateInactive; - if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateModeKey))) + gIOHibernateDebugFlags = 0; + if (kIOLogHibernate & gIOKitDebug) + gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs; + + /* The invocation of IOPMSleepSystemWithOptions() may override + * existing hibernation settings. + */ + sleepOverrideOptions = (OSDictionary *)OSDynamicCast( OSDictionary, + IOService::getPMRootDomain()->copyProperty(kRootDomainSleepOptionsKey)); + + + /* Hibernate mode overriden by sleep otions ? */ + obj = NULL; + + if (sleepOverrideOptions) { + obj = sleepOverrideOptions->getObject(kIOHibernateModeKey); + if (obj) obj->retain(); + } + + if(!obj) { + obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateModeKey); + } + + if (obj && (num = OSDynamicCast(OSNumber, obj)) ) { - if ((num = OSDynamicCast(OSNumber, obj))) gIOHibernateMode = num->unsigned32BitValue(); if (kIOHibernateModeSleep & gIOHibernateMode) // default to discard clean for safe sleep gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive | kIOHibernateModeDiscardCleanActive); + } - obj->release(); + if (obj) obj->release(); + + /* Hibernate free rotio overriden by sleep options ? */ + obj = NULL; + + if (sleepOverrideOptions) { + obj = sleepOverrideOptions->getObject(kIOHibernateFreeRatioKey); + if (obj) obj->retain(); + } + + if(!obj) { + obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeRatioKey); } - if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeRatioKey))) + if (obj && (num = OSDynamicCast(OSNumber, obj))) { - if ((num = OSDynamicCast(OSNumber, obj))) gIOHibernateFreeRatio = num->unsigned32BitValue(); - obj->release(); } + if (obj) obj->release(); + if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeTimeKey))) { if ((num = OSDynamicCast(OSNumber, obj))) @@ -1006,15 +1067,20 @@ IOHibernateSystemSleep(void) if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey))) { if ((str = OSDynamicCast(OSString, obj))) - strcpy(gIOHibernateFilename, str->getCStringNoCopy()); + strlcpy(gIOHibernateFilename, str->getCStringNoCopy(), + sizeof(gIOHibernateFilename)); obj->release(); } + if (sleepOverrideOptions) + sleepOverrideOptions->release(); + if (!gIOHibernateMode || !gIOHibernateFilename[0]) return (kIOReturnUnsupported); HIBLOG("hibernate image path: %s\n", gIOHibernateFilename); + do { vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, @@ -1046,6 +1112,7 @@ IOHibernateSystemSleep(void) } bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader)); + gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags; boolean_t encryptedswap; err = hibernate_setup(gIOHibernateCurrentHeader, @@ -1095,7 +1162,7 @@ IOHibernateSystemSleep(void) } data->release(); -#ifdef __ppc__ +#if defined(__ppc__) size_t len; char valueString[16]; @@ -1119,7 +1186,7 @@ IOHibernateSystemSleep(void) { vars->haveFastBoot = true; - len = sprintf(valueString, "0x%lx", *((UInt32 *)data->getBytesNoCopy())); + len = snprintf(valueString, sizeof(valueString), "0x%lx", *((UInt32 *)data->getBytesNoCopy())); data = OSData::withBytes(valueString, len + 1); sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey); if (sym && data) @@ -1134,7 +1201,7 @@ IOHibernateSystemSleep(void) gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy()); } #endif /* __ppc__ */ -#ifdef __i386__ +#if defined(__i386__) || defined(__x86_64__) struct AppleRTCHibernateVars { uint8_t signature[4]; @@ -1173,9 +1240,24 @@ IOHibernateSystemSleep(void) } data = OSData::withBytes(&rtcVars, sizeof(rtcVars)); if (data) - { - IOService::getPMRootDomain()->setProperty(kIOHibernateRTCVariablesKey, data); - data->release(); + { + IOService::getPMRootDomain()->setProperty(kIOHibernateRTCVariablesKey, data); + + if( gIOOptionsEntry ) + { + if( gIOHibernateMode & kIOHibernateModeSwitch ) + { + const OSSymbol *sym; + sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSwitchVarsKey); + if( sym ) + { + gIOOptionsEntry->setProperty(sym, data); /* intentional insecure backup of rtc boot vars */ + sym->release(); + } + } + } + + data->release(); } if (gIOChosenEntry) { @@ -1183,7 +1265,7 @@ IOHibernateSystemSleep(void) if (data) gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy()); } -#else /* !__i386__ */ +#else /* !i386 && !x86_64 */ if (kIOHibernateModeEncrypt & gIOHibernateMode) { data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey)); @@ -1194,7 +1276,7 @@ IOHibernateSystemSleep(void) sym->release(); if (data) data->release(); - if (gIOHibernateBootSignature[0]) + if (false && gIOHibernateBootSignature[0]) { data = OSData::withCapacity(16); sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey); @@ -1237,7 +1319,7 @@ IOHibernateSystemSleep(void) &newVolume, sizeof(newVolume)); } } -#endif /* !__i386__ */ +#endif /* !i386 && !x86_64 */ } // -- @@ -1251,27 +1333,173 @@ IOHibernateSystemSleep(void) /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +DECLARE_IOHIBERNATEPROGRESSALPHA + +static void +ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen) +{ + uint32_t rowBytes, pixelShift; + uint32_t x, y; + int32_t blob; + uint32_t alpha, in, color, result; + uint8_t * out; + uint32_t saveindex[kIOHibernateProgressCount] = { 0 }; + + rowBytes = display->rowBytes; + pixelShift = display->depth >> 4; + if (pixelShift < 1) return; + + screen += ((display->width + - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1)) + + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes; + + for (y = 0; y < kIOHibernateProgressHeight; y++) + { + out = screen + y * rowBytes; + for (blob = 0; blob < kIOHibernateProgressCount; blob++) + { + color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray; + for (x = 0; x < kIOHibernateProgressWidth; x++) + { + alpha = gIOHibernateProgressAlpha[y][x]; + result = color; + if (alpha) + { + if (0xff != alpha) + { + if (1 == pixelShift) + { + in = *((uint16_t *)out) & 0x1f; // 16 + in = (in << 3) | (in >> 2); + } + else + in = *((uint32_t *)out) & 0xff; // 32 + saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in; + result = ((255 - alpha) * in + alpha * result + 0xff) >> 8; + } + if (1 == pixelShift) + { + result >>= 3; + *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16 + } + else + *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32 + } + out += (1 << pixelShift); + } + out += (kIOHibernateProgressSpacing << pixelShift); + } + } +} + + +static void +ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select) +{ + uint32_t rowBytes, pixelShift; + uint32_t x, y; + int32_t blob, lastBlob; + uint32_t alpha, in, color, result; + uint8_t * out; + uint32_t saveindex[kIOHibernateProgressCount] = { 0 }; + + pixelShift = display->depth >> 4; + if (pixelShift < 1) + return; + + rowBytes = display->rowBytes; + + screen += ((display->width + - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1)) + + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes; + + lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1); + + screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift; + + for (y = 0; y < kIOHibernateProgressHeight; y++) + { + out = screen + y * rowBytes; + for (blob = firstBlob; blob <= lastBlob; blob++) + { + color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray; + for (x = 0; x < kIOHibernateProgressWidth; x++) + { + alpha = gIOHibernateProgressAlpha[y][x]; + result = color; + if (alpha) + { + if (0xff != alpha) + { + in = display->progressSaveUnder[blob][saveindex[blob]++]; + result = ((255 - alpha) * in + alpha * result + 0xff) / 255; + } + if (1 == pixelShift) + { + result >>= 3; + *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16 + } + else + *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32 + } + out += (1 << pixelShift); + } + out += (kIOHibernateProgressSpacing << pixelShift); + } + } +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + IOReturn IOHibernateSystemHasSlept(void) { IOHibernateVars * vars = &gIOHibernateVars; + OSObject * obj; + OSData * data; + + obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey); + vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj); + if (obj && !vars->previewBuffer) + obj->release(); - if ((vars->previewData = OSDynamicCast(OSData, - IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewBufferKey)))) + vars->consoleMapping = NULL; + if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare())) { - vars->previewBuffer = IOMemoryDescriptor::withAddress( - (void *) vars->previewData->getBytesNoCopy(), - vars->previewData->getLength(), - kIODirectionInOut); + vars->previewBuffer->release(); + vars->previewBuffer = 0; + } - if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare())) - { - vars->previewBuffer->release(); - vars->previewBuffer = 0; - } - if (!vars->previewBuffer) - vars->previewData = 0; + if (vars->previewBuffer && (data = OSDynamicCast(OSData, + IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey)))) + { + UInt32 flags = *((UInt32 *)data->getBytesNoCopy()); + HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags); + + IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey); + + if (kIOHibernatePreviewUpdates & flags) + { + PE_Video consoleInfo; + hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo; + + IOService::getPlatform()->getConsoleInfo(&consoleInfo); + + graphicsInfo->width = consoleInfo.v_width; + graphicsInfo->height = consoleInfo.v_height; + graphicsInfo->rowBytes = consoleInfo.v_rowBytes; + graphicsInfo->depth = consoleInfo.v_depth; + vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr; + + HIBPRINT("video %p %d %d %d\n", + vars->consoleMapping, gIOHibernateGraphicsInfo->depth, + gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height); + if (vars->consoleMapping) + ProgressInit(graphicsInfo, vars->consoleMapping, + &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder)); + } } + if (gIOOptionsEntry) gIOOptionsEntry->sync(); @@ -1294,7 +1522,7 @@ IOHibernateSystemWake(void) IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize); if (vars->videoAllocSize) // dealloc range - kmem_free(kernel_map, trunc_page_32(vars->videoMapping), vars->videoAllocSize); + kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize); } if (vars->previewBuffer) @@ -1366,21 +1594,24 @@ IOHibernateSystemWake(void) } #endif -#ifdef __i386__ - IOService::getPMRootDomain()->removeProperty(kIOHibernateRTCVariablesKey); - - /* - * Hibernate variable is written to NVRAM on platforms in which RtcRam - * is not backed by coin cell. Remove Hibernate data from NVRAM. - */ - if (gIOOptionsEntry) { - const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey); - - if (sym) { - gIOOptionsEntry->removeProperty(sym); - sym->release(); - } - } +#if defined(__i386__) || defined(__x86_64__) + IOService::getPMRootDomain()->removeProperty(kIOHibernateRTCVariablesKey); + + /* + * Hibernate variable is written to NVRAM on platforms in which RtcRam + * is not backed by coin cell. Remove Hibernate data from NVRAM. + */ + if (gIOOptionsEntry) { + const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey); + + if (sym) { + if (gIOOptionsEntry->getProperty(sym)) { + gIOOptionsEntry->removeProperty(sym); + gIOOptionsEntry->sync(); + } + sym->release(); + } + } #endif if (vars->srcBuffer) @@ -1417,6 +1648,16 @@ IOHibernateSystemPostWake(void) /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +SYSCTL_STRING(_kern, OID_AUTO, hibernatefile, + CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, + gIOHibernateFilename, sizeof(gIOHibernateFilename), ""); +SYSCTL_STRING(_kern, OID_AUTO, bootsignature, + CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, + gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), ""); +SYSCTL_UINT(_kern, OID_AUTO, hibernatemode, + CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, + &gIOHibernateMode, 0, ""); + void IOHibernateSystemInit(IOPMrootDomain * rootDomain) { @@ -1427,27 +1668,17 @@ IOHibernateSystemInit(IOPMrootDomain * rootDomain) data->release(); } - if (PE_parse_boot_arg("hfile", gIOHibernateFilename)) + if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename))) gIOHibernateMode = kIOHibernateModeOn; else gIOHibernateFilename[0] = 0; - static SYSCTL_STRING(_kern, OID_AUTO, hibernatefile, - CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, - gIOHibernateFilename, sizeof(gIOHibernateFilename), ""); sysctl_register_oid(&sysctl__kern_hibernatefile); - - static SYSCTL_STRING(_kern, OID_AUTO, bootsignature, - CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, - gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), ""); sysctl_register_oid(&sysctl__kern_bootsignature); - - static SYSCTL_UINT(_kern, OID_AUTO, hibernatemode, - CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, - &gIOHibernateMode, 0, ""); sysctl_register_oid(&sysctl__kern_hibernatemode); } + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ static void @@ -1463,7 +1694,7 @@ hibernate_setup_for_wake(void) #define C_ASSERT(e) typedef char __C_ASSERT__[(e) ? 1 : -1] -extern "C" boolean_t +extern "C" uint32_t hibernate_write_image(void) { IOHibernateImageHeader * header = gIOHibernateCurrentHeader; @@ -1493,6 +1724,7 @@ hibernate_write_image(void) uint64_t nsec; uint32_t lastProgressStamp = 0; uint32_t progressStamp; + uint32_t blob, lastBlob = (uint32_t) -1L; hibernate_cryptvars_t _cryptvars; hibernate_cryptvars_t * cryptvars = 0; @@ -1502,6 +1734,7 @@ hibernate_write_image(void) restore1Sum = sum1 = sum2 = 0; +#if CRYPTO // encryption data. "iv" is the "initial vector". if (kIOHibernateModeEncrypt & gIOHibernateMode) { @@ -1529,6 +1762,7 @@ hibernate_write_image(void) bzero(&vars->cryptKey[0], sizeof(vars->cryptKey)); bzero(gIOHibernateCryptWakeVars, sizeof(hibernate_cryptwakevars_t)); } +#endif /* CRYPTO */ hibernate_setup_for_wake(); @@ -1556,6 +1790,8 @@ hibernate_write_image(void) AbsoluteTime_to_scalar(&decoTime) = 0; clock_get_uptime(&allTime); + IOService::getPMRootDomain()->pmStatsRecordEvent( + kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime); do { @@ -1586,17 +1822,27 @@ hibernate_write_image(void) break; } + uintptr_t hibernateBase; + uintptr_t hibernateEnd; + +#if defined(__i386__) || defined(__x86_64__) + hibernateBase = sectINITPTB; +#else + hibernateBase = sectHIBB; +#endif + + hibernateEnd = (sectHIBB + sectSizeHIB); // copy out restore1 code - page = atop_32(sectHIBB); - count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page; + page = atop_32(hibernateBase); + count = atop_32(round_page(hibernateEnd)) - page; header->restore1CodePage = page; header->restore1PageCount = count; - header->restore1CodeOffset = ((uint32_t) &hibernate_machine_entrypoint) - sectHIBB; - header->restore1StackOffset = ((uint32_t) &gIOHibernateRestoreStackEnd[0]) - 64 - sectHIBB; + header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase; + header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase; // sum __HIB sect, with zeros for the stack - src = (uint8_t *) trunc_page(sectHIBB); + src = (uint8_t *) trunc_page(hibernateBase); for (page = 0; page < count; page++) { if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0])) @@ -1609,8 +1855,8 @@ hibernate_write_image(void) // write the __HIB sect, with zeros for the stack - src = (uint8_t *) trunc_page(sectHIBB); - count = ((uint32_t) &gIOHibernateRestoreStack[0]) - trunc_page(sectHIBB); + src = (uint8_t *) trunc_page(hibernateBase); + count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase); if (count) { err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars); @@ -1624,7 +1870,7 @@ hibernate_write_image(void) if (kIOReturnSuccess != err) break; src = &gIOHibernateRestoreStackEnd[0]; - count = round_page(sectHIBB + sectSizeHIB) - ((uint32_t) src); + count = round_page(hibernateEnd) - ((uintptr_t) src); if (count) { err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars); @@ -1637,13 +1883,13 @@ hibernate_write_image(void) addr64_t phys64; IOByteCount segLen; - if (vars->previewData) + if (vars->previewBuffer) { ppnum = 0; count = 0; do { - phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen); + phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone); pageAndCount[0] = atop_64(phys64); pageAndCount[1] = atop_32(segLen); err = IOPolledFileWrite(vars->fileVars, @@ -1658,8 +1904,8 @@ hibernate_write_image(void) if (kIOReturnSuccess != err) break; - src = (uint8_t *) vars->previewData->getBytesNoCopy(); - count = vars->previewData->getLength(); + src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment); + count = vars->previewBuffer->getLength(); header->previewPageListSize = ppnum; header->previewSize = count + ppnum; @@ -1675,7 +1921,7 @@ hibernate_write_image(void) // mark areas for no save for (count = 0; - (phys64 = vars->ioBuffer->getPhysicalSegment64(count, &segLen)); + (phys64 = vars->ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone)); count += segLen) { hibernate_set_page_state(vars->page_list, vars->page_list_wired, @@ -1685,7 +1931,7 @@ hibernate_write_image(void) } for (count = 0; - (phys64 = vars->srcBuffer->getPhysicalSegment64(count, &segLen)); + (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone)); count += segLen) { hibernate_set_page_state(vars->page_list, vars->page_list_wired, @@ -1707,15 +1953,15 @@ hibernate_write_image(void) hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount); - page = atop_32(sectHIBB); - count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page; + page = atop_32(hibernateBase); + count = atop_32(round_page(hibernateEnd)) - page; hibernate_set_page_state(vars->page_list, vars->page_list_wired, page, count, kIOHibernatePageStateFree); pageCount -= count; if (vars->previewBuffer) for (count = 0; - (phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen)); + (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone)); count += segLen) { hibernate_set_page_state(vars->page_list, vars->page_list_wired, @@ -1728,6 +1974,7 @@ hibernate_write_image(void) ppnum = 0; pagesDone = 0; + lastBlob = 0; HIBLOG("writing %d pages\n", pageCount); @@ -1752,7 +1999,7 @@ hibernate_write_image(void) err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(ppnum), page_size); if (err) { - HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%d] %x\n", __LINE__, ppnum, err); + HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)ppnum, err); break; } @@ -1808,7 +2055,16 @@ hibernate_write_image(void) uncompressedSize += page_size; ppnum++; pagesDone++; - + + if (vars->consoleMapping && (0 == (1023 & pagesDone))) + { + blob = ((pagesDone * kIOHibernateProgressCount) / pageCount); + if (blob != lastBlob) + { + ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob); + lastBlob = blob; + } + } if (0 == (8191 & pagesDone)) { clock_get_uptime(&endTime); @@ -1882,13 +2138,17 @@ hibernate_write_image(void) err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars); if (kIOReturnSuccess != err) break; - err = IOHibernatePollerIODone(vars->fileVars); + err = IOHibernatePollerIODone(vars->fileVars, true); if (kIOReturnSuccess != err) break; } while (false); clock_get_uptime(&endTime); + + IOService::getPMRootDomain()->pmStatsRecordEvent( + kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime); + SUB_ABSOLUTETIME(&endTime, &allTime); absolutetime_to_nanoseconds(endTime, &nsec); HIBLOG("all time: %qd ms, ", @@ -1908,77 +2168,45 @@ hibernate_write_image(void) uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0, sum1, sum2); + if (vars->fileVars->io) + (void) IOHibernatePollerIODone(vars->fileVars, false); + if (pollerOpen) IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState); + if (vars->consoleMapping) + ProgressUpdate(gIOHibernateGraphicsInfo, + vars->consoleMapping, 0, kIOHibernateProgressCount); + HIBLOG("hibernate_write_image done(%x)\n", err); // should we come back via regular wake, set the state in memory. gIOHibernateState = kIOHibernateStateInactive; - if ((kIOReturnSuccess == err) && !(kIOHibernateModeSleep & gIOHibernateMode)) - return (true /* power down */ ); + if (kIOReturnSuccess == err) + { + if (kIOHibernateModeSleep & gIOHibernateMode) + { + return (kIOHibernatePostWriteSleep); + } + else if(kIOHibernateModeRestart & gIOHibernateMode) + { + return (kIOHibernatePostWriteRestart); + } + else + { + /* by default, power down */ + return (kIOHibernatePostWriteHalt); + } + } + else if (kIOReturnAborted == err) + { + return (kIOHibernatePostWriteWake); + } else - return (false /* sleep */ ); -} - -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -DECLARE_IOHIBERNATEPROGRESSALPHA - -static void -ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select) -{ - uint32_t rowBytes, pixelShift; - uint32_t x, y; - int32_t blob, lastBlob; - uint32_t alpha, in, color, result; - uint8_t * out; - uint32_t saveindex[kIOHibernateProgressCount] = { 0 }; - - pixelShift = display->depth >> 4; - if (pixelShift < 1) - return; - - rowBytes = display->rowBytes; - - screen += ((display->width - - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1)) - + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes; - - lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1); - - screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift; - - for (y = 0; y < kIOHibernateProgressHeight; y++) { - out = screen + y * rowBytes; - for (blob = firstBlob; blob <= lastBlob; blob++) - { - color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray; - for (x = 0; x < kIOHibernateProgressWidth; x++) - { - alpha = gIOHibernateProgressAlpha[y][x]; - result = color; - if (alpha) - { - if (0xff != alpha) - { - in = display->progressSaveUnder[blob][saveindex[blob]++]; - result = ((255 - alpha) * in + alpha * result + 0xff) / 255; - } - if (1 == pixelShift) - { - result >>= 3; - *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16 - } - else - *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32 - } - out += (1 << pixelShift); - } - out += (kIOHibernateProgressSpacing << pixelShift); - } + /* on error, sleep */ + return (kIOHibernatePostWriteSleep); } } @@ -2069,7 +2297,7 @@ hibernate_machine_init(void) IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size); - if (vars->videoMapping) + if (vars->videoMapSize) { lastBlob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount) / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition); @@ -2108,7 +2336,9 @@ hibernate_machine_init(void) uint32_t tag; vm_offset_t ppnum, compressedSize; - IOPolledFileRead(vars->fileVars, src, 8, cryptvars); + err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars); + if (kIOReturnSuccess != err) + break; ppnum = header[0]; count = header[1]; @@ -2120,9 +2350,17 @@ hibernate_machine_init(void) for (page = 0; page < count; page++) { - IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars); + err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars); + if (kIOReturnSuccess != err) + break; compressedSize = kIOHibernateTagLength & tag; + if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength)) + { + err = kIOReturnIPCError; + break; + } + if (!compressedSize) { ppnum++; @@ -2130,9 +2368,11 @@ hibernate_machine_init(void) continue; } - IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars); - - if (compressedSize != page_size) + err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars); + if (kIOReturnSuccess != err) + break; + + if (compressedSize < page_size) { decoOffset = page_size; WKdm_decompress((WK_word*) src, (WK_word*) (src + decoOffset), PAGE_SIZE_IN_WORDS); @@ -2144,12 +2384,15 @@ hibernate_machine_init(void) err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size); if (err) - HIBLOG("IOMemoryDescriptorReadToPhysical [%d] %x\n", ppnum, err); + { + HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err); + break; + } ppnum++; pagesDone++; - if (vars->videoMapping && (0 == (255 & pagesDone))) + if (vars->videoMapSize && (0 == (1023 & pagesDone))) { blob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount) / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition); @@ -2177,18 +2420,27 @@ hibernate_machine_init(void) } while (true); + if (kIOReturnSuccess != err) + panic("Hibernate restore error %x", err); + gIOHibernateCurrentHeader->actualImage2Sum = sum; if (vars->fileVars->io) - (void) IOHibernatePollerIODone(vars->fileVars); + (void) IOHibernatePollerIODone(vars->fileVars, false); err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState); - if (vars->videoMapping) + if (vars->videoMapSize) ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount); clock_get_uptime(&endTime); + + IOService::getPMRootDomain()->pmStatsRecordEvent( + kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime); + IOService::getPMRootDomain()->pmStatsRecordEvent( + kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime); + SUB_ABSOLUTETIME(&endTime, &allTime); absolutetime_to_nanoseconds(endTime, &nsec);