/*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2008 Apple Computer, Inc. All rights reserved.
*
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this
- * file.
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ *
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
*
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* Please see the License for the specific language governing rights and
* limitations under the License.
*
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
#include <sys/conf.h>
#include <sys/stat.h>
#include <sys/fcntl.h> // (FWRITE, ...)
-extern "C" {
#include <sys/sysctl.h>
-}
#include <IOKit/IOHibernatePrivate.h>
#include <IOKit/IOPolledInterface.h>
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
-
-OSMetaClassDefineReservedUnused(IOPolledInterface, 0);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
+extern unsigned int save_kdebug_enable;
extern uint32_t gIOHibernateState;
uint32_t gIOHibernateMode;
static char gIOHibernateBootSignature[256+1];
static IODTNVRAM * gIOOptionsEntry;
static IORegistryEntry * gIOChosenEntry;
+#if defined(__i386__) || defined(__x86_64__)
+static const OSSymbol * gIOCreateEFIDevicePathSymbol;
+#endif
static IOPolledFileIOVars gFileVars;
static IOHibernateVars gIOHibernateVars;
addr64_t dstAddr64;
IOByteCount dstLen;
- dstAddr64 = md->getPhysicalSegment64(offset, &dstLen);
+ dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
if (!dstAddr64)
break;
addr64_t srcAddr64;
IOByteCount dstLen;
- srcAddr64 = md->getPhysicalSegment64(offset, &dstLen);
+ srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
if (!srcAddr64)
break;
}
static vm_offset_t
-hibernate_page_list_iterate(hibernate_page_list_t * list,
- void ** iterator, vm_offset_t * ppnum)
+hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
{
- uint32_t count, idx;
-
- idx = (uint32_t) *iterator;
+ uint32_t page = *pPage;
+ uint32_t count;
+ hibernate_bitmap_t * bitmap;
- if (!idx)
- idx = hibernate_page_list_count(list, TRUE, idx);
+ while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
+ {
+ count = hibernate_page_bitmap_count(bitmap, TRUE, page);
+ if (!count)
+ break;
+ page += count;
+ if (page <= bitmap->last_page)
+ break;
+ }
- *ppnum = idx;
- count = hibernate_page_list_count(list, FALSE, idx);
- idx += count;
- idx += hibernate_page_list_count(list, TRUE, idx);
- *iterator = (void *) idx;
+ *pPage = page;
+ if (bitmap)
+ count = hibernate_page_bitmap_count(bitmap, FALSE, page);
+ else
+ count = 0;
return (count);
}
}
static IOReturn
-IOHibernatePollerIODone(IOPolledFileIOVars * vars)
+IOHibernatePollerIODone(IOPolledFileIOVars * vars, bool abortable)
{
- IOReturn err = kIOReturnError;
- int32_t idx;
+ IOReturn err = kIOReturnSuccess;
+ int32_t idx = 0;
IOPolledInterface * poller;
while (-1 == vars->ioStatus)
{
- for (idx = 0;
- (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
+ for (idx = 0;
+ (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
idx++)
{
- err = poller->checkForWork();
- if (err)
- HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
+ IOReturn newErr;
+ newErr = poller->checkForWork();
+ if ((newErr == kIOReturnAborted) && !abortable)
+ newErr = kIOReturnSuccess;
+ if (kIOReturnSuccess == err)
+ err = newErr;
}
}
- if (kIOReturnSuccess != vars->ioStatus)
- HIBLOG("IOPolledInterface::ioStatus 0x%x\n", vars->ioStatus);
+ if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort())
+ {
+ err = kIOReturnAborted;
+ HIBLOG("IOPolledInterface::checkForWork sw abort\n");
+ }
+
+ if (err)
+ {
+ HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
+ }
+ else
+ {
+ err = vars->ioStatus;
+ if (kIOReturnSuccess != err)
+ HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
+ }
- return (vars->ioStatus);
+ return (err);
}
IOReturn
&file_extent_callback, &ctx,
&hibernate_image_dev,
&vars->block0,
- &maxiobytes);
+ &maxiobytes,
+ &vars->solid_state);
if (!vars->fileRef)
{
err = kIOReturnNoSpace;
break;
}
- HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx\n", filename, ctx.size,
- vars->block0, maxiobytes);
+ gIOHibernateFileRef = vars->fileRef;
+
+ if (kIOHibernateModeSSDInvert & gIOHibernateMode)
+ vars->solid_state = vars->solid_state ? false : true;
+
+ HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx ssd %d\n", filename, ctx.size,
+ vars->block0, maxiobytes, vars->solid_state);
if (ctx.size < 1*1024*1024) // check against image size estimate!
{
err = kIOReturnNoSpace;
part->retain();
iter->release();
}
-
+ if (!part)
+ break;
+
int minor, major;
IORegistryEntry * next;
IORegistryEntry * child;
do
{
IOPolledInterface * poller;
- if ((poller = OSDynamicCast(IOPolledInterface, next->getProperty(kIOPolledInterfaceSupportKey))))
+ OSObject * obj;
+
+ obj = next->getProperty(kIOPolledInterfaceSupportKey);
+ if (kOSBooleanFalse == obj)
+ {
+ vars->pollers->flushCollection();
+ break;
+ }
+ else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
vars->pollers->setObject(poller);
if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey))))
vars->blockSize = num->unsigned32BitValue();
&& child->isParent(next, gIOServicePlane, true));
HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
- major, minor, vars->blockSize, vars->pollers->getCount());
+ major, minor, (long)vars->blockSize, vars->pollers->getCount());
if (vars->pollers->getCount() < kIOHibernateMinPollersNeeded)
continue;
*fileExtents = extentsData;
// make imagePath
- char str1[256];
- char str2[24];
- int len = sizeof(str1);
- if ((extentsData->getLength() >= sizeof(IOPolledFileExtent))
- && part->getPath(str1, &len, gIODTPlane))
+ if ((extentsData->getLength() >= sizeof(IOPolledFileExtent)))
{
- // (strip the plane name)
- char * tail = strchr(str1, ':');
- if (!tail)
- tail = str1 - 1;
- data = OSData::withBytes(tail + 1, strlen(tail + 1));
- sprintf(str2, ",%qx", vars->extentMap[0]);
- data->appendBytes(str2, strlen(str2));
+ char str2[24];
+
+#if defined(__i386__) || defined(__x86_64__)
+ if (!gIOCreateEFIDevicePathSymbol)
+ gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
+
+ snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
+
+ err = IOService::getPlatform()->callPlatformFunction(
+ gIOCreateEFIDevicePathSymbol, false,
+ (void *) part, (void *) str2, (void *) true,
+ (void *) &data);
+#else
+ char str1[256];
+ int len = sizeof(str1);
+
+ if (!part->getPath(str1, &len, gIODTPlane))
+ err = kIOReturnNotFound;
+ else
+ {
+ snprintf(str2, sizeof(str2), ",%qx", vars->extentMap[0].start);
+ // (strip the plane name)
+ char * tail = strchr(str1, ':');
+ if (!tail)
+ tail = str1 - 1;
+ data = OSData::withBytes(tail + 1, strlen(tail + 1));
+ data->appendBytes(str2, strlen(str2));
+ }
+#endif
+ if (kIOReturnSuccess == err)
*imagePath = data;
+ else
+ HIBLOG("error 0x%x getting path\n", err);
}
}
while (false);
{
HIBLOG("error 0x%x opening hibernation file\n", err);
if (vars->fileRef)
+ {
kern_close_file_for_direct_io(vars->fileRef);
+ gIOHibernateFileRef = vars->fileRef = NULL;
+ }
}
if (part)
vars->pollers->release();
}
- gIOHibernateFileRef = vars->fileRef;
-
bzero(vars, sizeof(IOPolledFileIOVars));
return (kIOReturnSuccess);
- vars->extentPosition + vars->currentExtent->start);
uint32_t length = (vars->bufferOffset);
- if (cryptvars && vars->encryptStart && (vars->position > vars->encryptStart))
+#if CRYPTO
+ if (cryptvars && vars->encryptStart
+ && (vars->position > vars->encryptStart)
+ && ((vars->position - length) < vars->encryptEnd))
{
uint32_t encryptLen, encryptStart;
encryptLen = vars->position - vars->encryptStart;
if (encryptLen > length)
encryptLen = length;
encryptStart = length - encryptLen;
-
+ if (vars->position > vars->encryptEnd)
+ encryptLen -= (vars->position - vars->encryptEnd);
+
// encrypt the buffer
aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
&cryptvars->aes_iv[0],
&cryptvars->aes_iv[0],
AES_BLOCK_SIZE);
}
+#endif /* CRYPTO */
if (vars->io)
{
- err = IOHibernatePollerIODone(vars);
+ err = IOHibernatePollerIODone(vars, true);
if (kIOReturnSuccess != err)
break;
}
{
if (vars->io)
{
- err = IOHibernatePollerIODone(vars);
+ err = IOHibernatePollerIODone(vars, false);
if (kIOReturnSuccess != err)
break;
}
}
}
- if (vars->extentRemaining <= vars->bufferSize)
- vars->lastRead = vars->extentRemaining;
- else
- vars->lastRead = vars->bufferSize;
-
+ uint64_t length;
+ uint64_t lastReadLength = vars->lastRead;
uint64_t offset = (vars->position
- vars->extentPosition + vars->currentExtent->start);
- uint64_t length = (vars->lastRead);
+ if (vars->extentRemaining <= vars->bufferSize)
+ length = vars->extentRemaining;
+ else
+ length = vars->bufferSize;
+ vars->lastRead = length;
//if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
vars->bufferOffset = 0;
+#if CRYPTO
if (cryptvars)
{
uint8_t thisVector[AES_BLOCK_SIZE];
// save initial vector for following decrypts
bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
- bcopy(vars->buffer + vars->bufferHalf + vars->lastRead - AES_BLOCK_SIZE,
+ bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
&cryptvars->aes_iv[0], AES_BLOCK_SIZE);
// decrypt the buffer
aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
&thisVector[0],
- vars->lastRead / AES_BLOCK_SIZE,
+ lastReadLength / AES_BLOCK_SIZE,
vars->buffer + vars->bufferHalf,
&cryptvars->ctx.decrypt);
}
+#endif /* CRYPTO */
}
}
while (size);
OSData * data;
OSObject * obj;
OSString * str;
- OSNumber * num;
+ bool dsSSD;
IOHibernateVars * vars = &gIOHibernateVars;
gIOHibernateState = kIOHibernateStateInactive;
- if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateModeKey)))
- {
- if ((num = OSDynamicCast(OSNumber, obj)))
- gIOHibernateMode = num->unsigned32BitValue();
+ gIOHibernateDebugFlags = 0;
+ if (kIOLogHibernate & gIOKitDebug)
+ gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
+
+ if (IOService::getPMRootDomain()->getHibernateSettings(
+ &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime))
if (kIOHibernateModeSleep & gIOHibernateMode)
// default to discard clean for safe sleep
gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
| kIOHibernateModeDiscardCleanActive);
- obj->release();
- }
- if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeRatioKey)))
- {
- if ((num = OSDynamicCast(OSNumber, obj)))
- gIOHibernateFreeRatio = num->unsigned32BitValue();
- obj->release();
- }
- if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeTimeKey)))
- {
- if ((num = OSDynamicCast(OSNumber, obj)))
- gIOHibernateFreeTime = num->unsigned32BitValue();
- obj->release();
- }
if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
{
if ((str = OSDynamicCast(OSString, obj)))
- strcpy(gIOHibernateFilename, str->getCStringNoCopy());
+ strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
+ sizeof(gIOHibernateFilename));
obj->release();
}
HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
+
do
{
- vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(0, 4 * page_size, page_size);
- vars->ioBuffer = IOBufferMemoryDescriptor::withOptions(0, 2 * kDefaultIOSize, page_size);
+ vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
+ 4 * page_size, page_size);
+ vars->ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
+ 2 * kDefaultIOSize, page_size);
if (!vars->srcBuffer || !vars->ioBuffer)
{
}
bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
+ gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
+
+ dsSSD = (vars->fileVars->solid_state
+ && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
+
+ if (dsSSD)
+ {
+ gIOHibernateCurrentHeader->options |=
+ kIOHibernateOptionSSD
+ | kIOHibernateOptionColor;
+ }
+ else
+ {
+ gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
+ }
boolean_t encryptedswap;
+ AbsoluteTime startTime, endTime;
+ uint64_t nsec;
+
+ clock_get_uptime(&startTime);
err = hibernate_setup(gIOHibernateCurrentHeader,
gIOHibernateFreeRatio, gIOHibernateFreeTime,
+ dsSSD,
&vars->page_list, &vars->page_list_wired, &encryptedswap);
+ clock_get_uptime(&endTime);
+ SUB_ABSOLUTETIME(&endTime, &startTime);
+ absolutetime_to_nanoseconds(endTime, &nsec);
+ HIBLOG("hibernate_setup(%d) took %qd ms\n", err, nsec / 1000000ULL);
+
if (KERN_SUCCESS != err)
- {
- HIBLOG("hibernate_setup(%d)\n", err);
break;
- }
if (encryptedswap)
gIOHibernateMode ^= kIOHibernateModeEncrypt;
- vars->videoAllocSize = kVideoMapSize;
- if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
- vars->videoMapping = 0;
+ if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
+ {
+ vars->videoAllocSize = kVideoMapSize;
+ if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
+ vars->videoMapping = 0;
+ }
// generate crypt keys
for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
if (gIOOptionsEntry)
{
const OSSymbol * sym;
- size_t len;
- char valueString[16];
sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
if (sym)
}
data->release();
+#if defined(__ppc__)
+ size_t len;
+ char valueString[16];
+
vars->saveBootDevice = gIOOptionsEntry->copyProperty(kIOSelectedBootDeviceKey);
if (gIOChosenEntry)
{
if (str2)
str2->release();
}
-
data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMemorySignatureKey));
if (data)
{
vars->haveFastBoot = true;
- len = sprintf(valueString, "0x%lx", *((UInt32 *)data->getBytesNoCopy()));
+ len = snprintf(valueString, sizeof(valueString), "0x%lx", *((UInt32 *)data->getBytesNoCopy()));
data = OSData::withBytes(valueString, len + 1);
sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
if (sym && data)
if (data)
gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
}
-
+#endif /* __ppc__ */
+#if defined(__i386__) || defined(__x86_64__)
+ struct AppleRTCHibernateVars
+ {
+ uint8_t signature[4];
+ uint32_t revision;
+ uint8_t booterSignature[20];
+ uint8_t wiredCryptKey[16];
+ };
+ AppleRTCHibernateVars rtcVars;
+
+ rtcVars.signature[0] = 'A';
+ rtcVars.signature[1] = 'A';
+ rtcVars.signature[2] = 'P';
+ rtcVars.signature[3] = 'L';
+ rtcVars.revision = 1;
+ bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
+ if (gIOHibernateBootSignature[0])
+ {
+ char c;
+ uint8_t value = 0;
+ for (uint32_t i = 0;
+ (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
+ i++)
+ {
+ if (c >= 'a')
+ c -= 'a' - 10;
+ else if (c >= 'A')
+ c -= 'A' - 10;
+ else if (c >= '0')
+ c -= '0';
+ else
+ continue;
+ value = (value << 4) | c;
+ if (i & 1)
+ rtcVars.booterSignature[i >> 1] = value;
+ }
+ }
+ data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
+ if (data)
+ {
+ IOService::getPMRootDomain()->setProperty(kIOHibernateRTCVariablesKey, data);
+
+ if( gIOOptionsEntry )
+ {
+ if( gIOHibernateMode & kIOHibernateModeSwitch )
+ {
+ const OSSymbol *sym;
+ sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSwitchVarsKey);
+ if( sym )
+ {
+ gIOOptionsEntry->setProperty(sym, data); /* intentional insecure backup of rtc boot vars */
+ sym->release();
+ }
+ }
+ }
+
+ data->release();
+ }
+ if (gIOChosenEntry)
+ {
+ data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
+ if (data)
+ gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
+ }
+#else /* !i386 && !x86_64 */
if (kIOHibernateModeEncrypt & gIOHibernateMode)
{
data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
sym->release();
if (data)
data->release();
- if (gIOHibernateBootSignature[0])
+ if (false && gIOHibernateBootSignature[0])
{
data = OSData::withCapacity(16);
sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
if (sym && data)
{
char c;
- uint8_t value;
+ uint8_t value = 0;
for (uint32_t i = 0; (c = gIOHibernateBootSignature[i]); i++)
{
if (c >= 'a')
data->release();
}
}
-
if (!vars->haveFastBoot)
{
// set boot volume to zero
&newVolume, sizeof(newVolume));
}
}
+#endif /* !i386 && !x86_64 */
}
// --
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+DECLARE_IOHIBERNATEPROGRESSALPHA
+
+static void
+ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
+{
+ uint32_t rowBytes, pixelShift;
+ uint32_t x, y;
+ int32_t blob;
+ uint32_t alpha, in, color, result;
+ uint8_t * out;
+ uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
+
+ rowBytes = display->rowBytes;
+ pixelShift = display->depth >> 4;
+ if (pixelShift < 1) return;
+
+ screen += ((display->width
+ - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
+ + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
+
+ for (y = 0; y < kIOHibernateProgressHeight; y++)
+ {
+ out = screen + y * rowBytes;
+ for (blob = 0; blob < kIOHibernateProgressCount; blob++)
+ {
+ color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
+ for (x = 0; x < kIOHibernateProgressWidth; x++)
+ {
+ alpha = gIOHibernateProgressAlpha[y][x];
+ result = color;
+ if (alpha)
+ {
+ if (0xff != alpha)
+ {
+ if (1 == pixelShift)
+ {
+ in = *((uint16_t *)out) & 0x1f; // 16
+ in = (in << 3) | (in >> 2);
+ }
+ else
+ in = *((uint32_t *)out) & 0xff; // 32
+ saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
+ result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
+ }
+ if (1 == pixelShift)
+ {
+ result >>= 3;
+ *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
+ }
+ else
+ *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
+ }
+ out += (1 << pixelShift);
+ }
+ out += (kIOHibernateProgressSpacing << pixelShift);
+ }
+ }
+}
+
+
+static void
+ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
+{
+ uint32_t rowBytes, pixelShift;
+ uint32_t x, y;
+ int32_t blob, lastBlob;
+ uint32_t alpha, in, color, result;
+ uint8_t * out;
+ uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
+
+ pixelShift = display->depth >> 4;
+ if (pixelShift < 1)
+ return;
+
+ rowBytes = display->rowBytes;
+
+ screen += ((display->width
+ - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
+ + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
+
+ lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
+
+ screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
+
+ for (y = 0; y < kIOHibernateProgressHeight; y++)
+ {
+ out = screen + y * rowBytes;
+ for (blob = firstBlob; blob <= lastBlob; blob++)
+ {
+ color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
+ for (x = 0; x < kIOHibernateProgressWidth; x++)
+ {
+ alpha = gIOHibernateProgressAlpha[y][x];
+ result = color;
+ if (alpha)
+ {
+ if (0xff != alpha)
+ {
+ in = display->progressSaveUnder[blob][saveindex[blob]++];
+ result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
+ }
+ if (1 == pixelShift)
+ {
+ result >>= 3;
+ *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
+ }
+ else
+ *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
+ }
+ out += (1 << pixelShift);
+ }
+ out += (kIOHibernateProgressSpacing << pixelShift);
+ }
+ }
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
IOReturn
IOHibernateSystemHasSlept(void)
{
IOHibernateVars * vars = &gIOHibernateVars;
+ OSObject * obj;
+ OSData * data;
+
+ obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
+ vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
+ if (obj && !vars->previewBuffer)
+ obj->release();
- if ((vars->previewData = OSDynamicCast(OSData,
- IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewBufferKey))))
+ vars->consoleMapping = NULL;
+ if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
{
- vars->previewBuffer = IOMemoryDescriptor::withAddress(
- (void *) vars->previewData->getBytesNoCopy(),
- vars->previewData->getLength(),
- kIODirectionInOut);
+ vars->previewBuffer->release();
+ vars->previewBuffer = 0;
+ }
- if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
- {
- vars->previewBuffer->release();
- vars->previewBuffer = 0;
- }
- if (!vars->previewBuffer)
- vars->previewData = 0;
+ if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
+ && vars->previewBuffer
+ && (data = OSDynamicCast(OSData,
+ IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
+ {
+ UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
+ HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
+
+ IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
+
+ if (kIOHibernatePreviewUpdates & flags)
+ {
+ PE_Video consoleInfo;
+ hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
+
+ IOService::getPlatform()->getConsoleInfo(&consoleInfo);
+
+ graphicsInfo->width = consoleInfo.v_width;
+ graphicsInfo->height = consoleInfo.v_height;
+ graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
+ graphicsInfo->depth = consoleInfo.v_depth;
+ vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
+
+ HIBPRINT("video %p %d %d %d\n",
+ vars->consoleMapping, graphicsInfo->depth,
+ graphicsInfo->width, graphicsInfo->height);
+ if (vars->consoleMapping)
+ ProgressInit(graphicsInfo, vars->consoleMapping,
+ &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
+ }
}
+
if (gIOOptionsEntry)
gIOOptionsEntry->sync();
IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
if (vars->videoAllocSize)
// dealloc range
- kmem_free(kernel_map, trunc_page_32(vars->videoMapping), vars->videoAllocSize);
+ kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
}
if (vars->previewBuffer)
vars->previewBuffer = 0;
}
+ if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
+ {
+ IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
+ gIOHibernateCurrentHeader->options, 32);
+ }
+ else
+ {
+ IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
+ }
+
+ if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
+ && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus))
+ {
+ IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
+ &gIOHibernateGraphicsInfo->gfxStatus,
+ sizeof(gIOHibernateGraphicsInfo->gfxStatus));
+ }
+ else
+ {
+ IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
+ }
+
+
if (vars->fileVars)
{
IOPolledFileClose(vars->fileVars);
// invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
+#ifdef __ppc__
OSData * data = OSData::withCapacity(4);
if (gIOOptionsEntry && data)
{
// just sync the variables in case a later panic syncs nvram (it won't sync variables)
gIOOptionsEntry->syncOFVariables();
}
+#endif
+
+#if defined(__i386__) || defined(__x86_64__)
+ IOService::getPMRootDomain()->removeProperty(kIOHibernateRTCVariablesKey);
+
+ /*
+ * Hibernate variable is written to NVRAM on platforms in which RtcRam
+ * is not backed by coin cell. Remove Hibernate data from NVRAM.
+ */
+ if (gIOOptionsEntry) {
+ const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
+
+ if (sym) {
+ if (gIOOptionsEntry->getProperty(sym)) {
+ gIOOptionsEntry->removeProperty(sym);
+ gIOOptionsEntry->sync();
+ }
+ sym->release();
+ }
+ }
+#endif
if (vars->srcBuffer)
vars->srcBuffer->release();
{
if (gIOHibernateFileRef)
{
+ // invalidate the image file
+ gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
+ int err = kern_write_file(gIOHibernateFileRef, 0,
+ (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
+ if (KERN_SUCCESS != err)
+ HIBLOG("kern_write_file(%d)\n", err);
+
kern_close_file_for_direct_io(gIOHibernateFileRef);
gIOHibernateFileRef = 0;
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
+ CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
+ gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
+SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
+ CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
+ gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
+ CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
+ &gIOHibernateMode, 0, "");
+
void
IOHibernateSystemInit(IOPMrootDomain * rootDomain)
{
data->release();
}
- if (PE_parse_boot_arg("hfile", gIOHibernateFilename))
+ if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))
gIOHibernateMode = kIOHibernateModeOn;
else
gIOHibernateFilename[0] = 0;
- static SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
- CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
- gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
sysctl_register_oid(&sysctl__kern_hibernatefile);
-
- static SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
- CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
- gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
sysctl_register_oid(&sysctl__kern_bootsignature);
-
- static SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
- CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
- &gIOHibernateMode, 0, "");
sysctl_register_oid(&sysctl__kern_hibernatemode);
}
+
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
static void
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-extern "C" boolean_t
+#define C_ASSERT(e) typedef char __C_ASSERT__[(e) ? 1 : -1]
+
+static bool
+no_encrypt_page(vm_offset_t ppnum)
+{
+ if (pmap_is_noencrypt((ppnum_t)ppnum) == TRUE)
+ {
+ return true;
+ }
+ return false;
+}
+
+uint32_t wired_pages_encrypted = 0;
+uint32_t dirty_pages_encrypted = 0;
+uint32_t wired_pages_clear = 0;
+
+extern "C" uint32_t
hibernate_write_image(void)
{
IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
IOHibernateVars * vars = &gIOHibernateVars;
IOPolledFileExtent * fileExtents;
+ C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
+
uint32_t pageCount, pagesDone;
IOReturn err;
- vm_offset_t ppnum;
- IOItemCount page, count;
+ vm_offset_t ppnum, page;
+ IOItemCount count;
uint8_t * src;
uint8_t * data;
IOByteCount pageCompressedSize;
uint64_t compressedSize, uncompressedSize;
uint64_t image1Size = 0;
uint32_t bitmap_size;
- bool iterDone, pollerOpen, needEncryptStart;
+ bool iterDone, pollerOpen, needEncrypt;
uint32_t restore1Sum, sum, sum1, sum2;
uint32_t tag;
uint32_t pageType;
AbsoluteTime startTime, endTime;
AbsoluteTime allTime, compTime, decoTime;
+ uint64_t compBytes;
uint64_t nsec;
uint32_t lastProgressStamp = 0;
uint32_t progressStamp;
+ uint32_t blob, lastBlob = (uint32_t) -1L;
hibernate_cryptvars_t _cryptvars;
hibernate_cryptvars_t * cryptvars = 0;
+ wired_pages_encrypted = 0;
+ dirty_pages_encrypted = 0;
+ wired_pages_clear = 0;
+
if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
return (false /* sleep */ );
+ if (kIOHibernateModeSleep & gIOHibernateMode)
+ kdebug_enable = save_kdebug_enable;
+
+ KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START, 0, 0, 0, 0, 0);
+
restore1Sum = sum1 = sum2 = 0;
+#if CRYPTO
// encryption data. "iv" is the "initial vector".
if (kIOHibernateModeEncrypt & gIOHibernateMode)
{
bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
bzero(gIOHibernateCryptWakeVars, sizeof(hibernate_cryptwakevars_t));
}
+#endif /* CRYPTO */
hibernate_setup_for_wake();
}
#endif
- needEncryptStart = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
+ needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
AbsoluteTime_to_scalar(&compTime) = 0;
AbsoluteTime_to_scalar(&decoTime) = 0;
clock_get_uptime(&allTime);
+ IOService::getPMRootDomain()->pmStatsRecordEvent(
+ kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
do
{
compressedSize = 0;
uncompressedSize = 0;
- iterDone = false;
- pageType = 0; // wired pages first
IOPolledFileSeek(vars->fileVars, sizeof(IOHibernateImageHeader));
break;
}
+ uintptr_t hibernateBase;
+ uintptr_t hibernateEnd;
+
+#if defined(__i386__) || defined(__x86_64__)
+ hibernateBase = sectINITPTB;
+#else
+ hibernateBase = sectHIBB;
+#endif
+
+ hibernateEnd = (sectHIBB + sectSizeHIB);
// copy out restore1 code
- page = atop_32(sectHIBB);
- count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
+ page = atop_32(hibernateBase);
+ count = atop_32(round_page(hibernateEnd)) - page;
header->restore1CodePage = page;
header->restore1PageCount = count;
- header->restore1CodeOffset = ((uint32_t) &hibernate_machine_entrypoint) - sectHIBB;
- header->restore1StackOffset = ((uint32_t) &gIOHibernateRestoreStackEnd[0]) - 64 - sectHIBB;
+ header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
+ header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
// sum __HIB sect, with zeros for the stack
- src = (uint8_t *) trunc_page(sectHIBB);
+ src = (uint8_t *) trunc_page(hibernateBase);
for (page = 0; page < count; page++)
{
if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
- restore1Sum += hibernate_sum(src, page_size);
+ restore1Sum += hibernate_sum_page(src, header->restore1CodePage + page);
else
- restore1Sum += 0x10000001;
+ restore1Sum += 0x00000000;
src += page_size;
}
sum1 = restore1Sum;
// write the __HIB sect, with zeros for the stack
- src = (uint8_t *) trunc_page(sectHIBB);
- count = ((uint32_t) &gIOHibernateRestoreStack[0]) - trunc_page(sectHIBB);
+ src = (uint8_t *) trunc_page(hibernateBase);
+ count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
if (count)
{
err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
if (kIOReturnSuccess != err)
break;
src = &gIOHibernateRestoreStackEnd[0];
- count = round_page(sectHIBB + sectSizeHIB) - ((uint32_t) src);
+ count = round_page(hibernateEnd) - ((uintptr_t) src);
if (count)
{
err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
addr64_t phys64;
IOByteCount segLen;
- if (vars->previewData)
+ if (vars->previewBuffer)
{
ppnum = 0;
count = 0;
do
{
- phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen);
+ phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
pageAndCount[0] = atop_64(phys64);
pageAndCount[1] = atop_32(segLen);
err = IOPolledFileWrite(vars->fileVars,
if (kIOReturnSuccess != err)
break;
- src = (uint8_t *) vars->previewData->getBytesNoCopy();
- count = vars->previewData->getLength();
+ src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
+ count = vars->previewBuffer->getLength();
header->previewPageListSize = ppnum;
header->previewSize = count + ppnum;
for (page = 0; page < count; page += page_size)
- sum1 += hibernate_sum(src + page, page_size);
-
+ {
+ phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
+ sum1 += hibernate_sum_page(src + page, atop_64(phys64));
+ }
err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
if (kIOReturnSuccess != err)
break;
// mark areas for no save
for (count = 0;
- (phys64 = vars->ioBuffer->getPhysicalSegment64(count, &segLen));
+ (phys64 = vars->ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
count += segLen)
{
hibernate_set_page_state(vars->page_list, vars->page_list_wired,
}
for (count = 0;
- (phys64 = vars->srcBuffer->getPhysicalSegment64(count, &segLen));
+ (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
count += segLen)
{
hibernate_set_page_state(vars->page_list, vars->page_list_wired,
// mark more areas for no save, but these are not available
// for trashing during restore
+
+ hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
-#if !__i386__
- page = atop_32(sectHIBB);
- count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
-#else
- // XXX
- page = atop_32(sectHIBB & 0x3FFFFFFF);
- count = atop_32(round_page((sectHIBB + sectSizeHIB) & 0x3FFFFFFF)) - page;
-#endif
+ page = atop_32(hibernateBase);
+ count = atop_32(round_page(hibernateEnd)) - page;
hibernate_set_page_state(vars->page_list, vars->page_list_wired,
page, count,
kIOHibernatePageStateFree);
pageCount -= count;
-
-
if (vars->previewBuffer) for (count = 0;
- (phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen));
+ (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
count += segLen)
{
hibernate_set_page_state(vars->page_list, vars->page_list_wired,
src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
- void * iter = 0;
- pagesDone = 0;
+ pagesDone = 0;
+ lastBlob = 0;
HIBLOG("writing %d pages\n", pageCount);
- do
+ enum
+ // pageType
+ {
+ kWired = 0x02,
+ kEncrypt = 0x01,
+ kWiredEncrypt = kWired | kEncrypt,
+ kWiredClear = kWired,
+ kUnwiredEncrypt = kEncrypt
+ };
+
+ for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)
{
- count = hibernate_page_list_iterate(pageType ? vars->page_list : vars->page_list_wired,
- &iter, &ppnum);
-// kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
-
- iterDone = !count;
-
- pageAndCount[0] = ppnum;
- pageAndCount[1] = count;
- err = IOPolledFileWrite(vars->fileVars,
- (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
- cryptvars);
- if (kIOReturnSuccess != err)
- break;
-
- for (page = 0; page < count; page++)
+ if (needEncrypt && (kEncrypt & pageType))
{
- err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(ppnum), page_size);
- if (err)
+ vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
+ vars->fileVars->encryptEnd = UINT64_MAX;
+ HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
+
+ if (kUnwiredEncrypt == pageType)
{
- HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%d] %x\n", __LINE__, ppnum, err);
- break;
+ // start unwired image
+ bcopy(&cryptvars->aes_iv[0],
+ &gIOHibernateCryptWakeContext.aes_iv[0],
+ sizeof(cryptvars->aes_iv));
+ cryptvars = &gIOHibernateCryptWakeContext;
}
+ }
+ for (iterDone = false, ppnum = 0; !iterDone; )
+ {
+ count = hibernate_page_list_iterate((kWired & pageType)
+ ? vars->page_list_wired : vars->page_list,
+ &ppnum);
+// kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
+ iterDone = !count;
- sum = hibernate_sum(src, page_size);
-
- clock_get_uptime(&startTime);
-
- pageCompressedSize = WKdm_compress ((WK_word*) src, (WK_word*) (src + page_size), PAGE_SIZE_IN_WORDS);
-
- clock_get_uptime(&endTime);
- ADD_ABSOLUTETIME(&compTime, &endTime);
- SUB_ABSOLUTETIME(&compTime, &startTime);
-
- if (kIOHibernateModeEncrypt & gIOHibernateMode)
- pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
-
- if (pageCompressedSize > page_size)
+ if (count && (kWired & pageType) && needEncrypt)
{
-// HIBLOG("------------lose: %d\n", pageCompressedSize);
- pageCompressedSize = page_size;
+ uint32_t checkIndex;
+ for (checkIndex = 0;
+ (checkIndex < count)
+ && (((kEncrypt & pageType) == 0) == no_encrypt_page(ppnum + checkIndex));
+ checkIndex++)
+ {}
+ if (!checkIndex)
+ {
+ ppnum++;
+ continue;
+ }
+ count = checkIndex;
}
- if (pageCompressedSize != page_size)
- data = (src + page_size);
- else
- data = src;
-
- tag = pageCompressedSize | kIOHibernateTagSignature;
-
- if (pageType)
- sum2 += sum;
+ switch (pageType)
+ {
+ case kWiredEncrypt: wired_pages_encrypted += count; break;
+ case kWiredClear: wired_pages_clear += count; break;
+ case kUnwiredEncrypt: dirty_pages_encrypted += count; break;
+ }
+
+ if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */}
else
- sum1 += sum;
-
- if (needEncryptStart && (ppnum >= atop_32(sectDATAB)))
{
- // start encrypting partway into the data about to be written
- vars->fileVars->encryptStart = (vars->fileVars->position + AES_BLOCK_SIZE - 1)
- & ~(AES_BLOCK_SIZE - 1);
- needEncryptStart = false;
+ pageAndCount[0] = ppnum;
+ pageAndCount[1] = count;
+ err = IOPolledFileWrite(vars->fileVars,
+ (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
+ cryptvars);
+ if (kIOReturnSuccess != err)
+ break;
}
-
- err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
- if (kIOReturnSuccess != err)
- break;
-
- err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
- if (kIOReturnSuccess != err)
- break;
-
- compressedSize += pageCompressedSize;
- if (pageCompressedSize)
- uncompressedSize += page_size;
- ppnum++;
- pagesDone++;
- if (0 == (8191 & pagesDone))
+ for (page = ppnum; page < (ppnum + count); page++)
{
+ err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
+ if (err)
+ {
+ HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
+ break;
+ }
+
+ sum = hibernate_sum_page(src, page);
+ if (kWired & pageType)
+ sum1 += sum;
+ else
+ sum2 += sum;
+
+ clock_get_uptime(&startTime);
+
+ pageCompressedSize = WKdm_compress ((WK_word*) src, (WK_word*) (src + page_size), PAGE_SIZE_IN_WORDS);
+
clock_get_uptime(&endTime);
- SUB_ABSOLUTETIME(&endTime, &allTime);
- absolutetime_to_nanoseconds(endTime, &nsec);
- progressStamp = nsec / 750000000ULL;
- if (progressStamp != lastProgressStamp)
+ ADD_ABSOLUTETIME(&compTime, &endTime);
+ SUB_ABSOLUTETIME(&compTime, &startTime);
+ compBytes += page_size;
+
+ if (kIOHibernateModeEncrypt & gIOHibernateMode)
+ pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
+
+ if (pageCompressedSize > page_size)
+ {
+// HIBLOG("------------lose: %d\n", pageCompressedSize);
+ pageCompressedSize = page_size;
+ }
+
+ if (pageCompressedSize != page_size)
+ data = (src + page_size);
+ else
+ data = src;
+
+ tag = pageCompressedSize | kIOHibernateTagSignature;
+ err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
+ if (kIOReturnSuccess != err)
+ break;
+
+ err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
+ if (kIOReturnSuccess != err)
+ break;
+
+ compressedSize += pageCompressedSize;
+ if (pageCompressedSize)
+ uncompressedSize += page_size;
+ pagesDone++;
+
+ if (vars->consoleMapping && (0 == (1023 & pagesDone)))
+ {
+ blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
+ if (blob != lastBlob)
+ {
+ ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
+ lastBlob = blob;
+ }
+ }
+ if (0 == (8191 & pagesDone))
{
- lastProgressStamp = progressStamp;
- HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
+ clock_get_uptime(&endTime);
+ SUB_ABSOLUTETIME(&endTime, &allTime);
+ absolutetime_to_nanoseconds(endTime, &nsec);
+ progressStamp = nsec / 750000000ULL;
+ if (progressStamp != lastProgressStamp)
+ {
+ lastProgressStamp = progressStamp;
+ HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
+ }
}
}
+ if (kIOReturnSuccess != err)
+ break;
+ ppnum = page;
}
+
if (kIOReturnSuccess != err)
break;
- if (iterDone && !pageType)
+
+ if ((kEncrypt & pageType))
+ {
+ vars->fileVars->encryptEnd = (vars->fileVars->position + AES_BLOCK_SIZE - 1)
+ & ~(AES_BLOCK_SIZE - 1);
+ HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
+ }
+
+ if (kWiredEncrypt != pageType)
{
+ // end of image1/2 - fill to next block
err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
if (kIOReturnSuccess != err)
break;
-
- iterDone = false;
- pageType = 1;
- iter = 0;
+ }
+ if (kWiredClear == pageType)
+ {
+ // end wired image
+ header->encryptStart = vars->fileVars->encryptStart;
+ header->encryptEnd = vars->fileVars->encryptEnd;
image1Size = vars->fileVars->position;
- if (cryptvars)
- {
- bcopy(&cryptvars->aes_iv[0],
- &gIOHibernateCryptWakeContext.aes_iv[0],
- sizeof(cryptvars->aes_iv));
- cryptvars = &gIOHibernateCryptWakeContext;
- }
- HIBLOG("image1Size %qd\n", image1Size);
+ HIBLOG("image1Size %qd, encryptStart1 %qx, End1 %qx\n",
+ image1Size, header->encryptStart, header->encryptEnd);
}
}
- while (!iterDone);
- if (kIOReturnSuccess != err)
- break;
- err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
if (kIOReturnSuccess != err)
break;
header->image1Size = image1Size;
header->bitmapSize = bitmap_size;
header->pageCount = pageCount;
- header->encryptStart = vars->fileVars->encryptStart;
header->restore1Sum = restore1Sum;
header->image1Sum = sum1;
header->fileExtentMapSize = sizeof(header->fileExtentMap);
bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
+ header->deviceBase = vars->fileVars->block0;
+
IOPolledFileSeek(vars->fileVars, 0);
err = IOPolledFileWrite(vars->fileVars,
(uint8_t *) header, sizeof(IOHibernateImageHeader),
err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
if (kIOReturnSuccess != err)
break;
- err = IOHibernatePollerIODone(vars->fileVars);
+ err = IOHibernatePollerIODone(vars->fileVars, true);
if (kIOReturnSuccess != err)
break;
}
while (false);
clock_get_uptime(&endTime);
+
+ IOService::getPMRootDomain()->pmStatsRecordEvent(
+ kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
+
SUB_ABSOLUTETIME(&endTime, &allTime);
absolutetime_to_nanoseconds(endTime, &nsec);
HIBLOG("all time: %qd ms, ",
HIBLOG("\nimage %qd, uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
header->imageSize,
uncompressedSize, atop_32(uncompressedSize), compressedSize,
- (int) ((compressedSize * 100ULL) / uncompressedSize),
+ uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
sum1, sum2);
+ HIBLOG("wired_pages_encrypted %d, wired_pages_clear %d, dirty_pages_encrypted %d\n",
+ wired_pages_encrypted, wired_pages_clear, dirty_pages_encrypted);
+
+ if (vars->fileVars->io)
+ (void) IOHibernatePollerIODone(vars->fileVars, false);
+
if (pollerOpen)
IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
+ if (vars->consoleMapping)
+ ProgressUpdate(gIOHibernateGraphicsInfo,
+ vars->consoleMapping, 0, kIOHibernateProgressCount);
+
HIBLOG("hibernate_write_image done(%x)\n", err);
// should we come back via regular wake, set the state in memory.
gIOHibernateState = kIOHibernateStateInactive;
- if ((kIOReturnSuccess == err) && !(kIOHibernateModeSleep & gIOHibernateMode))
- return (true /* power down */ );
- else
- return (false /* sleep */ );
-}
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-DECLARE_IOHIBERNATEPROGRESSALPHA
-
-static void
-ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
-{
- uint32_t rowBytes, pixelShift;
- uint32_t x, y;
- int32_t blob, lastBlob;
- uint32_t alpha, in, color, result;
- uint8_t * out;
- uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
-
- pixelShift = display->depth >> 4;
- if (pixelShift < 1)
- return;
-
- rowBytes = display->rowBytes;
-
- screen += ((display->width
- - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
- + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
+ KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END,
+ wired_pages_encrypted, wired_pages_clear, dirty_pages_encrypted, 0, 0);
- lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
-
- screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
-
- for (y = 0; y < kIOHibernateProgressHeight; y++)
+ if (kIOReturnSuccess == err)
{
- out = screen + y * rowBytes;
- for (blob = firstBlob; blob <= lastBlob; blob++)
- {
- color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
- for (x = 0; x < kIOHibernateProgressWidth; x++)
- {
- alpha = gIOHibernateProgressAlpha[y][x];
- result = color;
- if (alpha)
- {
- if (0xff != alpha)
- {
- in = display->progressSaveUnder[blob][saveindex[blob]++];
- result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
- }
- if (1 == pixelShift)
- {
- result >>= 3;
- *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
- }
- else
- *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
- }
- out += (1 << pixelShift);
- }
- out += (kIOHibernateProgressSpacing << pixelShift);
- }
+ if (kIOHibernateModeSleep & gIOHibernateMode)
+ {
+ return (kIOHibernatePostWriteSleep);
+ }
+ else if(kIOHibernateModeRestart & gIOHibernateMode)
+ {
+ return (kIOHibernatePostWriteRestart);
+ }
+ else
+ {
+ /* by default, power down */
+ return (kIOHibernatePostWriteHalt);
+ }
+ }
+ else if (kIOReturnAborted == err)
+ {
+ return (kIOHibernatePostWriteWake);
+ }
+ else
+ {
+ /* on error, sleep */
+ return (kIOHibernatePostWriteSleep);
}
}
IOReturn err;
uint32_t sum;
uint32_t pagesDone;
+ uint32_t pagesRead = 0;
AbsoluteTime allTime, endTime;
uint64_t nsec;
uint32_t lastProgressStamp = 0;
if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
return;
- if ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)
- hibernate_page_list_discard(vars->page_list);
-
-
sum = gIOHibernateCurrentHeader->actualImage1Sum;
pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
- HIBPRINT("video %x %d %d %d\n",
+ HIBPRINT("video %x %d %d %d status %x\n",
gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
- gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height);
+ gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
+
+ if ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)
+ hibernate_page_list_discard(vars->page_list);
+
+ boot_args *args = (boot_args *) PE_state.bootArgs;
- if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)
+ if (vars->videoMapping
+ && gIOHibernateGraphicsInfo->physicalAddress
+ && (args->Video.v_baseAddr == gIOHibernateGraphicsInfo->physicalAddress))
{
vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
* gIOHibernateGraphicsInfo->rowBytes);
vars->videoMapSize, kIOMapInhibitCache );
}
- uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();;
+ uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
+
+ if (gIOHibernateWakeMapSize)
+ {
+ err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(gIOHibernateWakeMap),
+ gIOHibernateWakeMapSize);
+ if (kIOReturnSuccess == err)
+ hibernate_newruntime_map(src, gIOHibernateWakeMapSize,
+ gIOHibernateCurrentHeader->systemTableOffset);
+ gIOHibernateWakeMap = 0;
+ gIOHibernateWakeMapSize = 0;
+ }
+
uint32_t decoOffset;
clock_get_uptime(&allTime);
IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
- if (vars->videoMapping)
+ if (vars->videoMapSize)
{
lastBlob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
/ (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
uint32_t tag;
vm_offset_t ppnum, compressedSize;
- IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
+ err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
+ if (kIOReturnSuccess != err)
+ break;
ppnum = header[0];
count = header[1];
for (page = 0; page < count; page++)
{
- IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
+ err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
+ if (kIOReturnSuccess != err)
+ break;
compressedSize = kIOHibernateTagLength & tag;
+ if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength))
+ {
+ err = kIOReturnIPCError;
+ break;
+ }
+
if (!compressedSize)
{
ppnum++;
continue;
}
- IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
-
- if (compressedSize != page_size)
+ err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
+ if (kIOReturnSuccess != err)
+ break;
+
+ if (compressedSize < page_size)
{
decoOffset = page_size;
WKdm_decompress((WK_word*) src, (WK_word*) (src + decoOffset), PAGE_SIZE_IN_WORDS);
else
decoOffset = 0;
- sum += hibernate_sum((src + decoOffset), page_size);
+ sum += hibernate_sum_page((src + decoOffset), ppnum);
err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
if (err)
- HIBLOG("IOMemoryDescriptorReadToPhysical [%d] %x\n", ppnum, err);
+ {
+ HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
+ break;
+ }
ppnum++;
pagesDone++;
+ pagesRead++;
- if (vars->videoMapping && (0 == (255 & pagesDone)))
+ if (vars->videoMapSize && (0 == (1023 & pagesDone)))
{
blob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
/ (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
}
while (true);
+ if (kIOReturnSuccess != err)
+ panic("Hibernate restore error %x", err);
+
gIOHibernateCurrentHeader->actualImage2Sum = sum;
if (vars->fileVars->io)
- (void) IOHibernatePollerIODone(vars->fileVars);
+ (void) IOHibernatePollerIODone(vars->fileVars, false);
err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
- if (vars->videoMapping)
+ if (vars->videoMapSize)
ProgressUpdate(gIOHibernateGraphicsInfo,
(uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
clock_get_uptime(&endTime);
+
+ IOService::getPMRootDomain()->pmStatsRecordEvent(
+ kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
+ IOService::getPMRootDomain()->pmStatsRecordEvent(
+ kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
+
SUB_ABSOLUTETIME(&endTime, &allTime);
absolutetime_to_nanoseconds(endTime, &nsec);
HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %qd ms\n",
pagesDone, sum, nsec / 1000000ULL);
+
+ KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_NONE, pagesRead, pagesDone, 0, 0, 0);
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
projects / apple / xnu.git / blobdiff