X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/2d21ac55c334faf3a56e5634905ed6987fc787d4..c331a0bec715536613c8dd5f34a4e115d5b15824:/iokit/Kernel/IOHibernateIO.cpp diff --git a/iokit/Kernel/IOHibernateIO.cpp b/iokit/Kernel/IOHibernateIO.cpp index 58d4836c4..6b906baa9 100644 --- a/iokit/Kernel/IOHibernateIO.cpp +++ b/iokit/Kernel/IOHibernateIO.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2004-2008 Apple Computer, Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -168,27 +168,7 @@ to restrict I/O ops. /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject); - -OSMetaClassDefineReservedUnused(IOPolledInterface, 0); -OSMetaClassDefineReservedUnused(IOPolledInterface, 1); -OSMetaClassDefineReservedUnused(IOPolledInterface, 2); -OSMetaClassDefineReservedUnused(IOPolledInterface, 3); -OSMetaClassDefineReservedUnused(IOPolledInterface, 4); -OSMetaClassDefineReservedUnused(IOPolledInterface, 5); -OSMetaClassDefineReservedUnused(IOPolledInterface, 6); -OSMetaClassDefineReservedUnused(IOPolledInterface, 7); -OSMetaClassDefineReservedUnused(IOPolledInterface, 8); -OSMetaClassDefineReservedUnused(IOPolledInterface, 9); -OSMetaClassDefineReservedUnused(IOPolledInterface, 10); -OSMetaClassDefineReservedUnused(IOPolledInterface, 11); -OSMetaClassDefineReservedUnused(IOPolledInterface, 12); -OSMetaClassDefineReservedUnused(IOPolledInterface, 13); -OSMetaClassDefineReservedUnused(IOPolledInterface, 14); -OSMetaClassDefineReservedUnused(IOPolledInterface, 15); - -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - +extern unsigned int save_kdebug_enable; extern uint32_t gIOHibernateState; uint32_t gIOHibernateMode; static char gIOHibernateBootSignature[256+1]; @@ -198,7 +178,7 @@ uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (m static IODTNVRAM * gIOOptionsEntry; static IORegistryEntry * gIOChosenEntry; -#ifdef __i386__ +#if defined(__i386__) || defined(__x86_64__) static const OSSymbol * gIOCreateEFIDevicePathSymbol; #endif @@ -243,7 +223,7 @@ IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md, addr64_t dstAddr64; IOByteCount dstLen; - dstAddr64 = md->getPhysicalSegment64(offset, &dstLen); + dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone); if (!dstAddr64) break; @@ -281,7 +261,7 @@ IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md, addr64_t srcAddr64; IOByteCount dstLen; - srcAddr64 = md->getPhysicalSegment64(offset, &dstLen); + srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone); if (!srcAddr64) break; @@ -489,6 +469,12 @@ IOHibernatePollerIODone(IOPolledFileIOVars * vars, bool abortable) } } + if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort()) + { + err = kIOReturnAborted; + HIBLOG("IOPolledInterface::checkForWork sw abort\n"); + } + if (err) { HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err); @@ -585,14 +571,20 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer, &file_extent_callback, &ctx, &hibernate_image_dev, &vars->block0, - &maxiobytes); + &maxiobytes, + &vars->solid_state); if (!vars->fileRef) { err = kIOReturnNoSpace; break; } - HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx\n", filename, ctx.size, - vars->block0, maxiobytes); + gIOHibernateFileRef = vars->fileRef; + + if (kIOHibernateModeSSDInvert & gIOHibernateMode) + vars->solid_state = vars->solid_state ? false : true; + + HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx ssd %d\n", filename, ctx.size, + vars->block0, maxiobytes, vars->solid_state); if (ctx.size < 1*1024*1024) // check against image size estimate! { err = kIOReturnNoSpace; @@ -665,7 +657,7 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer, && child->isParent(next, gIOServicePlane, true)); HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n", - major, minor, vars->blockSize, vars->pollers->getCount()); + major, minor, (long)vars->blockSize, vars->pollers->getCount()); if (vars->pollers->getCount() < kIOHibernateMinPollersNeeded) continue; @@ -686,7 +678,7 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer, { char str2[24]; -#if __i386__ +#if defined(__i386__) || defined(__x86_64__) if (!gIOCreateEFIDevicePathSymbol) gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath"); @@ -725,7 +717,10 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer, { HIBLOG("error 0x%x opening hibernation file\n", err); if (vars->fileRef) + { kern_close_file_for_direct_io(vars->fileRef); + gIOHibernateFileRef = vars->fileRef = NULL; + } } if (part) @@ -743,8 +738,6 @@ IOPolledFileClose( IOPolledFileIOVars * vars ) vars->pollers->release(); } - gIOHibernateFileRef = vars->fileRef; - bzero(vars, sizeof(IOPolledFileIOVars)); return (kIOReturnSuccess); @@ -824,14 +817,18 @@ IOPolledFileWrite(IOPolledFileIOVars * vars, uint32_t length = (vars->bufferOffset); #if CRYPTO - if (cryptvars && vars->encryptStart && (vars->position > vars->encryptStart)) + if (cryptvars && vars->encryptStart + && (vars->position > vars->encryptStart) + && ((vars->position - length) < vars->encryptEnd)) { uint32_t encryptLen, encryptStart; encryptLen = vars->position - vars->encryptStart; if (encryptLen > length) encryptLen = length; encryptStart = length - encryptLen; - + if (vars->position > vars->encryptEnd) + encryptLen -= (vars->position - vars->encryptEnd); + // encrypt the buffer aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart, &cryptvars->aes_iv[0], @@ -942,14 +939,15 @@ if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", } } - if (vars->extentRemaining <= vars->bufferSize) - vars->lastRead = vars->extentRemaining; - else - vars->lastRead = vars->bufferSize; - + uint64_t length; + uint64_t lastReadLength = vars->lastRead; uint64_t offset = (vars->position - vars->extentPosition + vars->currentExtent->start); - uint64_t length = (vars->lastRead); + if (vars->extentRemaining <= vars->bufferSize) + length = vars->extentRemaining; + else + length = vars->bufferSize; + vars->lastRead = length; //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length); @@ -967,16 +965,16 @@ if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", uint8_t thisVector[AES_BLOCK_SIZE]; // save initial vector for following decrypts bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE); - bcopy(vars->buffer + vars->bufferHalf + vars->lastRead - AES_BLOCK_SIZE, + bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE, &cryptvars->aes_iv[0], AES_BLOCK_SIZE); // decrypt the buffer aes_decrypt_cbc(vars->buffer + vars->bufferHalf, &thisVector[0], - vars->lastRead / AES_BLOCK_SIZE, + lastReadLength / AES_BLOCK_SIZE, vars->buffer + vars->bufferHalf, &cryptvars->ctx.decrypt); } -#endif CRYPTO +#endif /* CRYPTO */ } } while (size); @@ -993,8 +991,7 @@ IOHibernateSystemSleep(void) OSData * data; OSObject * obj; OSString * str; - OSNumber * num; - OSDictionary *sleepOverrideOptions; + bool dsSSD; IOHibernateVars * vars = &gIOHibernateVars; @@ -1004,59 +1001,17 @@ IOHibernateSystemSleep(void) gIOHibernateState = kIOHibernateStateInactive; - /* The invocation of IOPMSleepSystemWithOptions() may override - * existing hibernation settings. - */ - sleepOverrideOptions = (OSDictionary *)OSDynamicCast( OSDictionary, - IOService::getPMRootDomain()->copyProperty(kRootDomainSleepOptionsKey)); - - - /* Hibernate mode overriden by sleep otions ? */ - obj = NULL; - - if (sleepOverrideOptions) { - obj = sleepOverrideOptions->getObject(kIOHibernateModeKey); - if (obj) obj->retain(); - } - - if(!obj) { - obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateModeKey); - } + gIOHibernateDebugFlags = 0; + if (kIOLogHibernate & gIOKitDebug) + gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs; - if (obj && (num = OSDynamicCast(OSNumber, obj)) ) - { - gIOHibernateMode = num->unsigned32BitValue(); + if (IOService::getPMRootDomain()->getHibernateSettings( + &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime)) if (kIOHibernateModeSleep & gIOHibernateMode) // default to discard clean for safe sleep gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive | kIOHibernateModeDiscardCleanActive); - } - if (obj) obj->release(); - - /* Hibernate free rotio overriden by sleep options ? */ - obj = NULL; - - if (sleepOverrideOptions) { - obj = sleepOverrideOptions->getObject(kIOHibernateFreeRatioKey); - if (obj) obj->retain(); - } - - if(!obj) { - obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeRatioKey); - } - if (obj && (num = OSDynamicCast(OSNumber, obj))) - { - gIOHibernateFreeRatio = num->unsigned32BitValue(); - } - if (obj) obj->release(); - - if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeTimeKey))) - { - if ((num = OSDynamicCast(OSNumber, obj))) - gIOHibernateFreeTime = num->unsigned32BitValue(); - obj->release(); - } if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey))) { if ((str = OSDynamicCast(OSString, obj))) @@ -1065,9 +1020,6 @@ IOHibernateSystemSleep(void) obj->release(); } - if (sleepOverrideOptions) - sleepOverrideOptions->release(); - if (!gIOHibernateMode || !gIOHibernateFilename[0]) return (kIOReturnUnsupported); @@ -1105,23 +1057,48 @@ IOHibernateSystemSleep(void) } bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader)); + gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags; + + dsSSD = (vars->fileVars->solid_state + && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey))); + + if (dsSSD) + { + gIOHibernateCurrentHeader->options |= + kIOHibernateOptionSSD + | kIOHibernateOptionColor; + } + else + { + gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress; + } boolean_t encryptedswap; + AbsoluteTime startTime, endTime; + uint64_t nsec; + + clock_get_uptime(&startTime); err = hibernate_setup(gIOHibernateCurrentHeader, gIOHibernateFreeRatio, gIOHibernateFreeTime, + dsSSD, &vars->page_list, &vars->page_list_wired, &encryptedswap); + clock_get_uptime(&endTime); + SUB_ABSOLUTETIME(&endTime, &startTime); + absolutetime_to_nanoseconds(endTime, &nsec); + HIBLOG("hibernate_setup(%d) took %qd ms\n", err, nsec / 1000000ULL); + if (KERN_SUCCESS != err) - { - HIBLOG("hibernate_setup(%d)\n", err); break; - } if (encryptedswap) gIOHibernateMode ^= kIOHibernateModeEncrypt; - vars->videoAllocSize = kVideoMapSize; - if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize)) - vars->videoMapping = 0; + if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options) + { + vars->videoAllocSize = kVideoMapSize; + if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize)) + vars->videoMapping = 0; + } // generate crypt keys for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++) @@ -1154,7 +1131,7 @@ IOHibernateSystemSleep(void) } data->release(); -#ifdef __ppc__ +#if defined(__ppc__) size_t len; char valueString[16]; @@ -1178,7 +1155,7 @@ IOHibernateSystemSleep(void) { vars->haveFastBoot = true; - len = sprintf(valueString, "0x%lx", *((UInt32 *)data->getBytesNoCopy())); + len = snprintf(valueString, sizeof(valueString), "0x%lx", *((UInt32 *)data->getBytesNoCopy())); data = OSData::withBytes(valueString, len + 1); sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey); if (sym && data) @@ -1193,7 +1170,7 @@ IOHibernateSystemSleep(void) gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy()); } #endif /* __ppc__ */ -#ifdef __i386__ +#if defined(__i386__) || defined(__x86_64__) struct AppleRTCHibernateVars { uint8_t signature[4]; @@ -1257,7 +1234,7 @@ IOHibernateSystemSleep(void) if (data) gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy()); } -#else /* !__i386__ */ +#else /* !i386 && !x86_64 */ if (kIOHibernateModeEncrypt & gIOHibernateMode) { data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey)); @@ -1311,7 +1288,7 @@ IOHibernateSystemSleep(void) &newVolume, sizeof(newVolume)); } } -#endif /* !__i386__ */ +#endif /* !i386 && !x86_64 */ } // -- @@ -1462,11 +1439,13 @@ IOHibernateSystemHasSlept(void) vars->previewBuffer = 0; } - if (vars->previewBuffer && (data = OSDynamicCast(OSData, + if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options) + && vars->previewBuffer + && (data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey)))) { UInt32 flags = *((UInt32 *)data->getBytesNoCopy()); - HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", flags); + HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags); IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey); @@ -1484,8 +1463,8 @@ IOHibernateSystemHasSlept(void) vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr; HIBPRINT("video %p %d %d %d\n", - vars->consoleMapping, gIOHibernateGraphicsInfo->depth, - gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height); + vars->consoleMapping, graphicsInfo->depth, + graphicsInfo->width, graphicsInfo->height); if (vars->consoleMapping) ProgressInit(graphicsInfo, vars->consoleMapping, &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder)); @@ -1514,7 +1493,7 @@ IOHibernateSystemWake(void) IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize); if (vars->videoAllocSize) // dealloc range - kmem_free(kernel_map, trunc_page_32(vars->videoMapping), vars->videoAllocSize); + kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize); } if (vars->previewBuffer) @@ -1523,6 +1502,29 @@ IOHibernateSystemWake(void) vars->previewBuffer = 0; } + if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) + { + IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey, + gIOHibernateCurrentHeader->options, 32); + } + else + { + IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey); + } + + if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) + && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus)) + { + IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey, + &gIOHibernateGraphicsInfo->gfxStatus, + sizeof(gIOHibernateGraphicsInfo->gfxStatus)); + } + else + { + IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey); + } + + if (vars->fileVars) { IOPolledFileClose(vars->fileVars); @@ -1586,7 +1588,7 @@ IOHibernateSystemWake(void) } #endif -#ifdef __i386__ +#if defined(__i386__) || defined(__x86_64__) IOService::getPMRootDomain()->removeProperty(kIOHibernateRTCVariablesKey); /* @@ -1597,7 +1599,10 @@ IOHibernateSystemWake(void) const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey); if (sym) { - gIOOptionsEntry->removeProperty(sym); + if (gIOOptionsEntry->getProperty(sym)) { + gIOOptionsEntry->removeProperty(sym); + gIOOptionsEntry->sync(); + } sym->release(); } } @@ -1683,6 +1688,20 @@ hibernate_setup_for_wake(void) #define C_ASSERT(e) typedef char __C_ASSERT__[(e) ? 1 : -1] +static bool +no_encrypt_page(vm_offset_t ppnum) +{ + if (pmap_is_noencrypt((ppnum_t)ppnum) == TRUE) + { + return true; + } + return false; +} + +uint32_t wired_pages_encrypted = 0; +uint32_t dirty_pages_encrypted = 0; +uint32_t wired_pages_clear = 0; + extern "C" uint32_t hibernate_write_image(void) { @@ -1694,15 +1713,15 @@ hibernate_write_image(void) uint32_t pageCount, pagesDone; IOReturn err; - vm_offset_t ppnum; - IOItemCount page, count; + vm_offset_t ppnum, page; + IOItemCount count; uint8_t * src; uint8_t * data; IOByteCount pageCompressedSize; uint64_t compressedSize, uncompressedSize; uint64_t image1Size = 0; uint32_t bitmap_size; - bool iterDone, pollerOpen, needEncryptStart; + bool iterDone, pollerOpen, needEncrypt; uint32_t restore1Sum, sum, sum1, sum2; uint32_t tag; uint32_t pageType; @@ -1710,6 +1729,7 @@ hibernate_write_image(void) AbsoluteTime startTime, endTime; AbsoluteTime allTime, compTime, decoTime; + uint64_t compBytes; uint64_t nsec; uint32_t lastProgressStamp = 0; uint32_t progressStamp; @@ -1718,9 +1738,18 @@ hibernate_write_image(void) hibernate_cryptvars_t _cryptvars; hibernate_cryptvars_t * cryptvars = 0; + wired_pages_encrypted = 0; + dirty_pages_encrypted = 0; + wired_pages_clear = 0; + if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents) return (false /* sleep */ ); + if (kIOHibernateModeSleep & gIOHibernateMode) + kdebug_enable = save_kdebug_enable; + + KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START, 0, 0, 0, 0, 0); + restore1Sum = sum1 = sum2 = 0; #if CRYPTO @@ -1773,19 +1802,19 @@ hibernate_write_image(void) } #endif - needEncryptStart = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode)); + needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode)); AbsoluteTime_to_scalar(&compTime) = 0; AbsoluteTime_to_scalar(&decoTime) = 0; clock_get_uptime(&allTime); + IOService::getPMRootDomain()->pmStatsRecordEvent( + kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime); do { compressedSize = 0; uncompressedSize = 0; - iterDone = false; - pageType = 0; // wired pages first IOPolledFileSeek(vars->fileVars, sizeof(IOHibernateImageHeader)); @@ -1809,31 +1838,41 @@ hibernate_write_image(void) break; } + uintptr_t hibernateBase; + uintptr_t hibernateEnd; + +#if defined(__i386__) || defined(__x86_64__) + hibernateBase = sectINITPTB; +#else + hibernateBase = sectHIBB; +#endif + + hibernateEnd = (sectHIBB + sectSizeHIB); // copy out restore1 code - page = atop_32(sectHIBB); - count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page; + page = atop_32(hibernateBase); + count = atop_32(round_page(hibernateEnd)) - page; header->restore1CodePage = page; header->restore1PageCount = count; - header->restore1CodeOffset = ((uint32_t) &hibernate_machine_entrypoint) - sectHIBB; - header->restore1StackOffset = ((uint32_t) &gIOHibernateRestoreStackEnd[0]) - 64 - sectHIBB; + header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase; + header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase; // sum __HIB sect, with zeros for the stack - src = (uint8_t *) trunc_page(sectHIBB); + src = (uint8_t *) trunc_page(hibernateBase); for (page = 0; page < count; page++) { if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0])) - restore1Sum += hibernate_sum(src, page_size); + restore1Sum += hibernate_sum_page(src, header->restore1CodePage + page); else - restore1Sum += 0x10000001; + restore1Sum += 0x00000000; src += page_size; } sum1 = restore1Sum; // write the __HIB sect, with zeros for the stack - src = (uint8_t *) trunc_page(sectHIBB); - count = ((uint32_t) &gIOHibernateRestoreStack[0]) - trunc_page(sectHIBB); + src = (uint8_t *) trunc_page(hibernateBase); + count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase); if (count) { err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars); @@ -1847,7 +1886,7 @@ hibernate_write_image(void) if (kIOReturnSuccess != err) break; src = &gIOHibernateRestoreStackEnd[0]; - count = round_page(sectHIBB + sectSizeHIB) - ((uint32_t) src); + count = round_page(hibernateEnd) - ((uintptr_t) src); if (count) { err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars); @@ -1866,7 +1905,7 @@ hibernate_write_image(void) count = 0; do { - phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen); + phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone); pageAndCount[0] = atop_64(phys64); pageAndCount[1] = atop_32(segLen); err = IOPolledFileWrite(vars->fileVars, @@ -1881,15 +1920,17 @@ hibernate_write_image(void) if (kIOReturnSuccess != err) break; - src = (uint8_t *) vars->previewBuffer->getSourceSegment(0, NULL); + src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment); count = vars->previewBuffer->getLength(); header->previewPageListSize = ppnum; header->previewSize = count + ppnum; for (page = 0; page < count; page += page_size) - sum1 += hibernate_sum(src + page, page_size); - + { + phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone); + sum1 += hibernate_sum_page(src + page, atop_64(phys64)); + } err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars); if (kIOReturnSuccess != err) break; @@ -1898,7 +1939,7 @@ hibernate_write_image(void) // mark areas for no save for (count = 0; - (phys64 = vars->ioBuffer->getPhysicalSegment64(count, &segLen)); + (phys64 = vars->ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone)); count += segLen) { hibernate_set_page_state(vars->page_list, vars->page_list_wired, @@ -1908,7 +1949,7 @@ hibernate_write_image(void) } for (count = 0; - (phys64 = vars->srcBuffer->getPhysicalSegment64(count, &segLen)); + (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone)); count += segLen) { hibernate_set_page_state(vars->page_list, vars->page_list_wired, @@ -1930,15 +1971,15 @@ hibernate_write_image(void) hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount); - page = atop_32(sectHIBB); - count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page; + page = atop_32(hibernateBase); + count = atop_32(round_page(hibernateEnd)) - page; hibernate_set_page_state(vars->page_list, vars->page_list_wired, page, count, kIOHibernatePageStateFree); pageCount -= count; if (vars->previewBuffer) for (count = 0; - (phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen)); + (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone)); count += segLen) { hibernate_set_page_state(vars->page_list, vars->page_list_wired, @@ -1949,138 +1990,187 @@ hibernate_write_image(void) src = (uint8_t *) vars->srcBuffer->getBytesNoCopy(); - ppnum = 0; pagesDone = 0; lastBlob = 0; HIBLOG("writing %d pages\n", pageCount); - do + enum + // pageType + { + kWired = 0x02, + kEncrypt = 0x01, + kWiredEncrypt = kWired | kEncrypt, + kWiredClear = kWired, + kUnwiredEncrypt = kEncrypt + }; + + for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--) { - count = hibernate_page_list_iterate(pageType ? vars->page_list : vars->page_list_wired, - &ppnum); -// kprintf("[%d](%x : %x)\n", pageType, ppnum, count); - - iterDone = !count; - - pageAndCount[0] = ppnum; - pageAndCount[1] = count; - err = IOPolledFileWrite(vars->fileVars, - (const uint8_t *) &pageAndCount, sizeof(pageAndCount), - cryptvars); - if (kIOReturnSuccess != err) - break; - - for (page = 0; page < count; page++) + if (needEncrypt && (kEncrypt & pageType)) { - err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(ppnum), page_size); - if (err) + vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1)); + vars->fileVars->encryptEnd = UINT64_MAX; + HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart); + + if (kUnwiredEncrypt == pageType) { - HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%d] %x\n", __LINE__, ppnum, err); - break; + // start unwired image + bcopy(&cryptvars->aes_iv[0], + &gIOHibernateCryptWakeContext.aes_iv[0], + sizeof(cryptvars->aes_iv)); + cryptvars = &gIOHibernateCryptWakeContext; } + } + for (iterDone = false, ppnum = 0; !iterDone; ) + { + count = hibernate_page_list_iterate((kWired & pageType) + ? vars->page_list_wired : vars->page_list, + &ppnum); +// kprintf("[%d](%x : %x)\n", pageType, ppnum, count); + iterDone = !count; - sum = hibernate_sum(src, page_size); - - clock_get_uptime(&startTime); - - pageCompressedSize = WKdm_compress ((WK_word*) src, (WK_word*) (src + page_size), PAGE_SIZE_IN_WORDS); - - clock_get_uptime(&endTime); - ADD_ABSOLUTETIME(&compTime, &endTime); - SUB_ABSOLUTETIME(&compTime, &startTime); - - if (kIOHibernateModeEncrypt & gIOHibernateMode) - pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1); - - if (pageCompressedSize > page_size) + if (count && (kWired & pageType) && needEncrypt) { -// HIBLOG("------------lose: %d\n", pageCompressedSize); - pageCompressedSize = page_size; + uint32_t checkIndex; + for (checkIndex = 0; + (checkIndex < count) + && (((kEncrypt & pageType) == 0) == no_encrypt_page(ppnum + checkIndex)); + checkIndex++) + {} + if (!checkIndex) + { + ppnum++; + continue; + } + count = checkIndex; } - if (pageCompressedSize != page_size) - data = (src + page_size); - else - data = src; - - tag = pageCompressedSize | kIOHibernateTagSignature; - - if (pageType) - sum2 += sum; + switch (pageType) + { + case kWiredEncrypt: wired_pages_encrypted += count; break; + case kWiredClear: wired_pages_clear += count; break; + case kUnwiredEncrypt: dirty_pages_encrypted += count; break; + } + + if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */} else - sum1 += sum; - - if (needEncryptStart && (ppnum >= atop_32(sectDATAB))) { - // start encrypting partway into the data about to be written - vars->fileVars->encryptStart = (vars->fileVars->position + AES_BLOCK_SIZE - 1) - & ~(AES_BLOCK_SIZE - 1); - needEncryptStart = false; + pageAndCount[0] = ppnum; + pageAndCount[1] = count; + err = IOPolledFileWrite(vars->fileVars, + (const uint8_t *) &pageAndCount, sizeof(pageAndCount), + cryptvars); + if (kIOReturnSuccess != err) + break; } - - err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars); - if (kIOReturnSuccess != err) - break; - - err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars); - if (kIOReturnSuccess != err) - break; - - compressedSize += pageCompressedSize; - if (pageCompressedSize) - uncompressedSize += page_size; - ppnum++; - pagesDone++; - - if (vars->consoleMapping && (0 == (1023 & pagesDone))) - { - blob = ((pagesDone * kIOHibernateProgressCount) / pageCount); - if (blob != lastBlob) - { - ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob); - lastBlob = blob; - } - } - if (0 == (8191 & pagesDone)) + + for (page = ppnum; page < (ppnum + count); page++) { + err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size); + if (err) + { + HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err); + break; + } + + sum = hibernate_sum_page(src, page); + if (kWired & pageType) + sum1 += sum; + else + sum2 += sum; + + clock_get_uptime(&startTime); + + pageCompressedSize = WKdm_compress ((WK_word*) src, (WK_word*) (src + page_size), PAGE_SIZE_IN_WORDS); + clock_get_uptime(&endTime); - SUB_ABSOLUTETIME(&endTime, &allTime); - absolutetime_to_nanoseconds(endTime, &nsec); - progressStamp = nsec / 750000000ULL; - if (progressStamp != lastProgressStamp) + ADD_ABSOLUTETIME(&compTime, &endTime); + SUB_ABSOLUTETIME(&compTime, &startTime); + compBytes += page_size; + + if (kIOHibernateModeEncrypt & gIOHibernateMode) + pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1); + + if (pageCompressedSize > page_size) + { +// HIBLOG("------------lose: %d\n", pageCompressedSize); + pageCompressedSize = page_size; + } + + if (pageCompressedSize != page_size) + data = (src + page_size); + else + data = src; + + tag = pageCompressedSize | kIOHibernateTagSignature; + err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars); + if (kIOReturnSuccess != err) + break; + + err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars); + if (kIOReturnSuccess != err) + break; + + compressedSize += pageCompressedSize; + if (pageCompressedSize) + uncompressedSize += page_size; + pagesDone++; + + if (vars->consoleMapping && (0 == (1023 & pagesDone))) + { + blob = ((pagesDone * kIOHibernateProgressCount) / pageCount); + if (blob != lastBlob) + { + ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob); + lastBlob = blob; + } + } + if (0 == (8191 & pagesDone)) { - lastProgressStamp = progressStamp; - HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount); + clock_get_uptime(&endTime); + SUB_ABSOLUTETIME(&endTime, &allTime); + absolutetime_to_nanoseconds(endTime, &nsec); + progressStamp = nsec / 750000000ULL; + if (progressStamp != lastProgressStamp) + { + lastProgressStamp = progressStamp; + HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount); + } } } + if (kIOReturnSuccess != err) + break; + ppnum = page; } + if (kIOReturnSuccess != err) break; - if (iterDone && !pageType) + + if ((kEncrypt & pageType)) + { + vars->fileVars->encryptEnd = (vars->fileVars->position + AES_BLOCK_SIZE - 1) + & ~(AES_BLOCK_SIZE - 1); + HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd); + } + + if (kWiredEncrypt != pageType) { + // end of image1/2 - fill to next block err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars); if (kIOReturnSuccess != err) break; - - iterDone = false; - pageType = 1; - ppnum = 0; + } + if (kWiredClear == pageType) + { + // end wired image + header->encryptStart = vars->fileVars->encryptStart; + header->encryptEnd = vars->fileVars->encryptEnd; image1Size = vars->fileVars->position; - if (cryptvars) - { - bcopy(&cryptvars->aes_iv[0], - &gIOHibernateCryptWakeContext.aes_iv[0], - sizeof(cryptvars->aes_iv)); - cryptvars = &gIOHibernateCryptWakeContext; - } - HIBLOG("image1Size %qd\n", image1Size); + HIBLOG("image1Size %qd, encryptStart1 %qx, End1 %qx\n", + image1Size, header->encryptStart, header->encryptEnd); } } - while (!iterDone); - if (kIOReturnSuccess != err) - break; - err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars); if (kIOReturnSuccess != err) break; @@ -2090,7 +2180,6 @@ hibernate_write_image(void) header->image1Size = image1Size; header->bitmapSize = bitmap_size; header->pageCount = pageCount; - header->encryptStart = vars->fileVars->encryptStart; header->restore1Sum = restore1Sum; header->image1Sum = sum1; @@ -2106,6 +2195,8 @@ hibernate_write_image(void) header->fileExtentMapSize = sizeof(header->fileExtentMap); bcopy(&fileExtents[0], &header->fileExtentMap[0], count); + header->deviceBase = vars->fileVars->block0; + IOPolledFileSeek(vars->fileVars, 0); err = IOPolledFileWrite(vars->fileVars, (uint8_t *) header, sizeof(IOHibernateImageHeader), @@ -2122,6 +2213,10 @@ hibernate_write_image(void) while (false); clock_get_uptime(&endTime); + + IOService::getPMRootDomain()->pmStatsRecordEvent( + kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime); + SUB_ABSOLUTETIME(&endTime, &allTime); absolutetime_to_nanoseconds(endTime, &nsec); HIBLOG("all time: %qd ms, ", @@ -2141,6 +2236,12 @@ hibernate_write_image(void) uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0, sum1, sum2); + HIBLOG("wired_pages_encrypted %d, wired_pages_clear %d, dirty_pages_encrypted %d\n", + wired_pages_encrypted, wired_pages_clear, dirty_pages_encrypted); + + if (vars->fileVars->io) + (void) IOHibernatePollerIODone(vars->fileVars, false); + if (pollerOpen) IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState); @@ -2153,6 +2254,9 @@ hibernate_write_image(void) // should we come back via regular wake, set the state in memory. gIOHibernateState = kIOHibernateStateInactive; + KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, + wired_pages_encrypted, wired_pages_clear, dirty_pages_encrypted, 0, 0); + if (kIOReturnSuccess == err) { if (kIOHibernateModeSleep & gIOHibernateMode) @@ -2188,6 +2292,7 @@ hibernate_machine_init(void) IOReturn err; uint32_t sum; uint32_t pagesDone; + uint32_t pagesRead = 0; AbsoluteTime allTime, endTime; uint64_t nsec; uint32_t lastProgressStamp = 0; @@ -2218,9 +2323,9 @@ hibernate_machine_init(void) gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1], gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]); - HIBPRINT("video %x %d %d %d\n", + HIBPRINT("video %x %d %d %d status %x\n", gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth, - gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height); + gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus); if ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode) hibernate_page_list_discard(vars->page_list); @@ -2306,7 +2411,9 @@ hibernate_machine_init(void) uint32_t tag; vm_offset_t ppnum, compressedSize; - IOPolledFileRead(vars->fileVars, src, 8, cryptvars); + err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars); + if (kIOReturnSuccess != err) + break; ppnum = header[0]; count = header[1]; @@ -2318,9 +2425,17 @@ hibernate_machine_init(void) for (page = 0; page < count; page++) { - IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars); + err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars); + if (kIOReturnSuccess != err) + break; compressedSize = kIOHibernateTagLength & tag; + if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength)) + { + err = kIOReturnIPCError; + break; + } + if (!compressedSize) { ppnum++; @@ -2328,9 +2443,11 @@ hibernate_machine_init(void) continue; } - IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars); - - if (compressedSize != page_size) + err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars); + if (kIOReturnSuccess != err) + break; + + if (compressedSize < page_size) { decoOffset = page_size; WKdm_decompress((WK_word*) src, (WK_word*) (src + decoOffset), PAGE_SIZE_IN_WORDS); @@ -2338,14 +2455,18 @@ hibernate_machine_init(void) else decoOffset = 0; - sum += hibernate_sum((src + decoOffset), page_size); + sum += hibernate_sum_page((src + decoOffset), ppnum); err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size); if (err) - HIBLOG("IOMemoryDescriptorReadToPhysical [%d] %x\n", ppnum, err); + { + HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err); + break; + } ppnum++; pagesDone++; + pagesRead++; if (vars->videoMapSize && (0 == (1023 & pagesDone))) { @@ -2375,6 +2496,9 @@ hibernate_machine_init(void) } while (true); + if (kIOReturnSuccess != err) + panic("Hibernate restore error %x", err); + gIOHibernateCurrentHeader->actualImage2Sum = sum; if (vars->fileVars->io) @@ -2387,11 +2511,19 @@ hibernate_machine_init(void) (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount); clock_get_uptime(&endTime); + + IOService::getPMRootDomain()->pmStatsRecordEvent( + kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime); + IOService::getPMRootDomain()->pmStatsRecordEvent( + kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime); + SUB_ABSOLUTETIME(&endTime, &allTime); absolutetime_to_nanoseconds(endTime, &nsec); HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %qd ms\n", pagesDone, sum, nsec / 1000000ULL); + + KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_NONE, pagesRead, pagesDone, 0, 0, 0); } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */