X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/fe8ab488e9161c46dd9885d58fc52996dc0249ff..e8c3f78193f1895ea514044358b93b1add9322f3:/iokit/Kernel/IOHibernateIO.cpp?ds=sidebyside diff --git a/iokit/Kernel/IOHibernateIO.cpp b/iokit/Kernel/IOHibernateIO.cpp index f85f2ab0e..94d5b465e 100644 --- a/iokit/Kernel/IOHibernateIO.cpp +++ b/iokit/Kernel/IOHibernateIO.cpp @@ -1,8 +1,8 @@ /* - * Copyright (c) 2004-2008 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2004-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,21 +22,20 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ - /* Sleep: - PMRootDomain calls IOHibernateSystemSleep() before system sleep (devices awake, normal execution context) -- IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level, +- IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level, grabs its extents and searches for a polling driver willing to work with that IOMedia. The BSD code makes an ioctl to the storage driver to get the partition base offset to - the disk, and other ioctls to get the transfer constraints + the disk, and other ioctls to get the transfer constraints If successful, the file is written to make sure its initially not bootable (in case of later failure) and nvram set to point to the first block of the file. (Has to be done here so blocking is possible in nvram support). @@ -46,7 +45,7 @@ Sleep: pages for eviction. It also copies processor flags needed for the restore path and sets a flag in the boot processor proc info. gIOHibernateState = kIOHibernateStateHibernating. -- Regular sleep progresses - some drivers may inspect the root domain property +- Regular sleep progresses - some drivers may inspect the root domain property kIOHibernateStateKey to modify behavior. The platform driver saves state to memory as usual but leaves motherboard I/O on. - Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu, @@ -58,17 +57,17 @@ Sleep: by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits. The image header and block list are written. The header includes the second file extent so only the header block is needed to read the file, regardless of filesystem. - The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data + The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data (only) is used to decompress the image during wake/boot. Some additional pages are removed from the bitmaps - the buffers used for hibernation. The bitmaps are written to the image. - More areas are removed from the bitmaps (after they have been written to the image) - the + More areas are removed from the bitmaps (after they have been written to the image) - the segment "__HIB" pages and interrupt stack. - Each wired page is compressed and written and then each non-wired page. Compression and + Each wired page is compressed and written and then each non-wired page. Compression and disk writes are in parallel. The image header is written to the start of the file and the polling driver closed. The machine powers down (or sleeps). - + Boot/Wake: - BootX sees the boot-image nvram variable containing the device and block number of the image, @@ -77,8 +76,8 @@ Boot/Wake: in the OF memory environment, and the image is decrypted. There is no decompression in BootX, that is in the kernel's __HIB section. - BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry - hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off, - only code & data in that section is safe to call since all the other wired pages are still + hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off, + only code & data in that section is safe to call since all the other wired pages are still compressed in the image. - hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps. It uses the bitmaps to work out which pages can be uncompressed from the image to their final @@ -88,11 +87,11 @@ Boot/Wake: is used to get pages into place for 64bit. - the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings - are removed from the software strutures, and the hash table is reinitialized. + are removed from the software strutures, and the hash table is reinitialized. - After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest of memory, using the polled mode driver, before other threads can run or any devices are turned on. This reduces the memory usage for BootX and allows decompression in parallel with disk reads, - for the remaining non wired pages. + for the remaining non wired pages. - The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file is closed via bsd. @@ -107,7 +106,7 @@ partition) that the image is going to live, looking for polled interface propert one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the interfaces found are kept in an ordered list. -There is an Open/Close pair of calls made to each of the interfaces at various stages since there are +There is an Open/Close pair of calls made to each of the interfaces at various stages since there are few different contexts things happen in: - there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all @@ -117,7 +116,7 @@ immediately wakes back up for the image write. - there is an Open/Close (BeforeSleep) pair made around the image write operations that happen immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart -from the low level bits (motherboard I/O etc). There is only one thread running. The close can be +from the low level bits (motherboard I/O etc). There is only one thread running. The close can be used to flush and set the disk to sleep. - there is an Open/Close (AfterSleep) pair made around the image read operations that happen @@ -147,6 +146,7 @@ to restrict I/O ops. #include #include #include +#include #include "RootDomainUserClient.h" #include #include "IOPMPowerStateQueue.h" @@ -167,12 +167,15 @@ to restrict I/O ops. #include #include "IOHibernateInternal.h" #include +#include #include "IOKitKernelInternal.h" #include #include #include #include +#include +#include extern "C" addr64_t kvtophys(vm_offset_t va); extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va); @@ -180,68 +183,69 @@ extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va); /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #define DISABLE_TRIM 0 -#define TRIM_DELAY 5000 +#define TRIM_DELAY 25000 extern unsigned int save_kdebug_enable; extern uint32_t gIOHibernateState; uint32_t gIOHibernateMode; static char gIOHibernateBootSignature[256+1]; static char gIOHibernateFilename[MAXPATHLEN+1]; + +static uuid_string_t gIOHibernateBridgeBootSessionUUIDString; + static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent) uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms) static uint64_t gIOHibernateCompression = 0x80; // default compression 50% +boolean_t gIOHibernateStandbyDisabled; static IODTNVRAM * gIOOptionsEntry; static IORegistryEntry * gIOChosenEntry; + +static const OSSymbol * gIOHibernateBootImageKey; +static const OSSymbol * gIOHibernateBootSignatureKey; +static const OSSymbol * gIOBridgeBootSessionUUIDKey; + #if defined(__i386__) || defined(__x86_64__) -static const OSSymbol * gIOCreateEFIDevicePathSymbol; + static const OSSymbol * gIOHibernateRTCVariablesKey; static const OSSymbol * gIOHibernateBoot0082Key; static const OSSymbol * gIOHibernateBootNextKey; static OSData * gIOHibernateBoot0082Data; static OSData * gIOHibernateBootNextData; static OSObject * gIOHibernateBootNextSave; -static struct kern_direct_file_io_ref_t * gDebugImageFileRef; -#endif + +#endif /* defined(__i386__) || defined(__x86_64__) */ static IOLock * gFSLock; -static uint32_t gFSState; + uint32_t gFSState; +static thread_call_t gIOHibernateTrimCalloutEntry; static IOPolledFileIOVars gFileVars; static IOHibernateVars gIOHibernateVars; -static struct kern_direct_file_io_ref_t * gIOHibernateFileRef; -static hibernate_cryptvars_t gIOHibernateCryptWakeContext; +static IOPolledFileCryptVars gIOHibernateCryptWakeContext; static hibernate_graphics_t _hibernateGraphics; static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics; static hibernate_statistics_t _hibernateStats; static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats; -enum +enum { - kFSIdle = 0, - kFSOpening = 2, - kFSOpened = 3, - kFSTimedOut = 4, + kFSIdle = 0, + kFSOpening = 2, + kFSOpened = 3, + kFSTimedOut = 4, + kFSTrimDelay = 5 }; static IOReturn IOHibernateDone(IOHibernateVars * vars); +static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature); +static void IOSetBootImageNVRAM(OSData * data); +static void IOHibernateSystemPostWakeTrim(void * p1, void * p2); /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -enum { kXPRamAudioVolume = 8 }; enum { kDefaultIOSize = 128 * 1024 }; enum { kVideoMapSize = 80 * 1024 * 1024 }; -#ifndef kIOMediaPreferredBlockSizeKey -#define kIOMediaPreferredBlockSizeKey "Preferred Block Size" -#endif - -#ifndef kIOBootPathKey -#define kIOBootPathKey "bootpath" -#endif -#ifndef kIOSelectedBootDeviceKey -#define kIOSelectedBootDeviceKey "boot-device" -#endif - /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ // copy from phys addr to MD @@ -386,1191 +390,218 @@ hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage) /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -static IOReturn -IOHibernatePollerProbe(IOPolledFileIOVars * vars, IOService * target) +IOReturn +IOHibernateSystemSleep(void) { - IOReturn err = kIOReturnError; - int32_t idx; - IOPolledInterface * poller; - - for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--) - { - poller = (IOPolledInterface *) vars->pollers->getObject(idx); - err = poller->probe(target); - if (err) - { - HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err); - break; - } - } + IOReturn err; + OSData * nvramData; + OSObject * obj; + OSString * str; + OSNumber * num; + bool dsSSD, vmflush, swapPinned; + IOHibernateVars * vars; + uint64_t setFileSize = 0; - return (err); -} + gIOHibernateState = kIOHibernateStateInactive; -static IOReturn -IOHibernatePollerOpen(IOPolledFileIOVars * vars, uint32_t state, IOMemoryDescriptor * md) -{ - IOReturn err = kIOReturnError; - int32_t idx; - IOPolledInterface * poller; + gIOHibernateDebugFlags = 0; + if (kIOLogHibernate & gIOKitDebug) + gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs; - for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--) + if (IOService::getPMRootDomain()->getHibernateSettings( + &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime)) { - poller = (IOPolledInterface *) vars->pollers->getObject(idx); - err = poller->open(state, md); - if (err) - { - HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err); - break; - } + if (kIOHibernateModeSleep & gIOHibernateMode) + // default to discard clean for safe sleep + gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive + | kIOHibernateModeDiscardCleanActive); } - return (err); -} - -static IOReturn -IOHibernatePollerClose(IOPolledFileIOVars * vars, uint32_t state) -{ - IOReturn err = kIOReturnError; - int32_t idx; - IOPolledInterface * poller; - - for (idx = 0; - (poller = (IOPolledInterface *) vars->pollers->getObject(idx)); - idx++) + if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey))) { - err = poller->close(state); - if (err) - HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err); + if ((str = OSDynamicCast(OSString, obj))) + strlcpy(gIOHibernateFilename, str->getCStringNoCopy(), + sizeof(gIOHibernateFilename)); + obj->release(); } - return (err); -} - -static void -IOHibernatePollerIOComplete(void * target, - void * parameter, - IOReturn status, - UInt64 actualByteCount) -{ - IOPolledFileIOVars * vars = (IOPolledFileIOVars *) parameter; - - vars->ioStatus = status; -} - -static IOReturn -IOHibernatePollerIO(IOPolledFileIOVars * vars, - uint32_t operation, uint32_t bufferOffset, - uint64_t deviceOffset, uint64_t length) -{ - - IOReturn err = kIOReturnError; - IOPolledInterface * poller; - IOPolledCompletion completion; - - completion.target = 0; - completion.action = &IOHibernatePollerIOComplete; - completion.parameter = vars; - - vars->ioStatus = -1; - - poller = (IOPolledInterface *) vars->pollers->getObject(0); - err = poller->startIO(operation, bufferOffset, deviceOffset + vars->block0, length, completion); - if (err) - HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err); - - return (err); -} - -static IOReturn -IOHibernatePollerIODone(IOPolledFileIOVars * vars, bool abortable) -{ - IOReturn err = kIOReturnSuccess; - int32_t idx = 0; - IOPolledInterface * poller; + if (!gIOHibernateMode || !gIOHibernateFilename[0]) + return (kIOReturnUnsupported); - while (-1 == vars->ioStatus) - { - for (idx = 0; - (poller = (IOPolledInterface *) vars->pollers->getObject(idx)); - idx++) - { - IOReturn newErr; - newErr = poller->checkForWork(); - if ((newErr == kIOReturnAborted) && !abortable) - newErr = kIOReturnSuccess; - if (kIOReturnSuccess == err) - err = newErr; - } - } + HIBLOG("hibernate image path: %s\n", gIOHibernateFilename); - if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort()) - { - err = kIOReturnAborted; - HIBLOG("IOPolledInterface::checkForWork sw abort\n"); - } + vars = IONew(IOHibernateVars, 1); + if (!vars) return (kIOReturnNoMemory); + bzero(vars, sizeof(*vars)); - if (err) + IOLockLock(gFSLock); + if (!gIOHibernateTrimCalloutEntry) { - HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err); + gIOHibernateTrimCalloutEntry = thread_call_allocate(&IOHibernateSystemPostWakeTrim, &gFSLock); } - else + IOHibernateSystemPostWakeTrim(NULL, NULL); + thread_call_cancel(gIOHibernateTrimCalloutEntry); + if (kFSIdle != gFSState) { - err = vars->ioStatus; - if (kIOReturnSuccess != err) - HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err); + HIBLOG("hibernate file busy\n"); + IOLockUnlock(gFSLock); + IODelete(vars, IOHibernateVars, 1); + return (kIOReturnBusy); } + gFSState = kFSOpening; + IOLockUnlock(gFSLock); - return (err); -} - -IOReturn -IOPolledInterface::checkAllForWork(void) -{ - IOReturn err = kIOReturnNotReady; - int32_t idx; - IOPolledInterface * poller; - - IOHibernateVars * vars = &gIOHibernateVars; - - if (!vars->fileVars || !vars->fileVars->pollers) - return (err); - - for (idx = 0; - (poller = (IOPolledInterface *) vars->fileVars->pollers->getObject(idx)); - idx++) + swapPinned = false; + do { - err = poller->checkForWork(); - if (err) - HIBLOG("IOPolledInterface::checkAllForWork[%d] 0x%x\n", idx, err); - } - - return (err); -} - -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -struct _OpenFileContext -{ - OSData * extents; - uint64_t size; -}; - -static void -file_extent_callback(void * ref, uint64_t start, uint64_t length) -{ - _OpenFileContext * ctx = (_OpenFileContext *) ref; - IOPolledFileExtent extent; - - extent.start = start; - extent.length = length; - - HIBLOG("[0x%qx, 0x%qx]\n", start, length); + vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, + 2 * page_size + WKdm_SCRATCH_BUF_SIZE_INTERNAL, page_size); - ctx->extents->appendBytes(&extent, sizeof(extent)); - ctx->size += length; -} + vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, + ptoa_64(gIOHibernateHandoffPageCount), page_size); -static IOService * -IOCopyMediaForDev(dev_t device) -{ - OSDictionary * matching; - OSNumber * num; - OSIterator * iter; - IOService * result = 0; - - matching = IOService::serviceMatching("IOMedia"); - if (!matching) - return (0); - do - { - num = OSNumber::withNumber(major(device), 32); - if (!num) - break; - matching->setObject(kIOBSDMajorKey, num); - num->release(); - num = OSNumber::withNumber(minor(device), 32); - if (!num) - break; - matching->setObject(kIOBSDMinorKey, num); - num->release(); - if (!num) - break; - iter = IOService::getMatchingServices(matching); - if (iter) + if (!vars->srcBuffer || !vars->handoffBuffer) { - result = (IOService *) iter->getNextObject(); - result->retain(); - iter->release(); + err = kIOReturnNoMemory; + break; } - } - while (false); - matching->release(); - return (result); -} + if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey))) + { + if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMinSize = num->unsigned64BitValue(); + obj->release(); + } + if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey))) + { + if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMaxSize = num->unsigned64BitValue(); + obj->release(); + } -/* - * Writes header to disk with signature, block size and file extents data. - * If there are more than 2 extents, then they are written on second block. - */ -static IOReturn -WriteExtentsToFile(struct kern_direct_file_io_ref_t * fileRef, - uint32_t signature, uint32_t blockSize, - IOPolledFileExtent *fileExtents, - IOByteCount size) -{ - IOHibernateImageHeader hdr; - IOItemCount count; - IOReturn err = kIOReturnSuccess; - int rc; + boolean_t encryptedswap = true; + uint32_t pageCount; + AbsoluteTime startTime, endTime; + uint64_t nsec; - memset(&hdr, 0, sizeof(IOHibernateImageHeader)); - count = size; - if (count > sizeof(hdr.fileExtentMap)) - { - hdr.fileExtentMapSize = count; - count = sizeof(hdr.fileExtentMap); - } - else - hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap); + bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader)); + gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags; + gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature; - bcopy(fileExtents, &hdr.fileExtentMap[0], count); + vmflush = ((kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey))); + err = hibernate_alloc_page_lists(&vars->page_list, + &vars->page_list_wired, + &vars->page_list_pal); + if (KERN_SUCCESS != err) break; - // copy file block extent list if larger than header - if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap)) - { - count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap); - rc = kern_write_file(fileRef, blockSize, - (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)), - count, IO_SKIP_ENCRYPTION); - if (rc != 0) { - HIBLOG("kern_write_file returned %d\n", rc); - err = kIOReturnIOError; - goto exit; - } - } - hdr.signature = signature; - hdr.deviceBlockSize = blockSize; + err = hibernate_pin_swap(TRUE); + if (KERN_SUCCESS != err) break; + swapPinned = true; - rc = kern_write_file(fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION); - if (rc != 0) { - HIBLOG("kern_write_file returned %d\n", rc); - err = kIOReturnIOError; - goto exit; - } + if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode)) + { + hibernate_page_list_setall(vars->page_list, + vars->page_list_wired, + vars->page_list_pal, + true /* preflight */, + vmflush /* discard */, + &pageCount); + PE_Video consoleInfo; + bzero(&consoleInfo, sizeof(consoleInfo)); + IOService::getPlatform()->getConsoleInfo(&consoleInfo); -exit: - return err; -} + // estimate: 6% increase in pages compressed + // screen preview 2 images compressed 0% + setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8) + + vars->page_list->list_size + + (consoleInfo.v_width * consoleInfo.v_height * 8); + enum { setFileRound = 1024*1024ULL }; + setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1)); -static IOReturn -GetImageBlockSize(IOService *part, OSArray *pollers, IOByteCount *blockSize) -{ - IOService * service; - IORegistryEntry * next; - IORegistryEntry * child; + HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n", + pageCount, (100ULL * gIOHibernateCompression) >> 8, + setFileSize, vars->fileMinSize); - IOReturn err = kIOReturnSuccess; + if (!(kIOHibernateModeFileResize & gIOHibernateMode) + && (setFileSize < vars->fileMinSize)) + { + setFileSize = vars->fileMinSize; + } + } + vars->volumeCryptKeySize = sizeof(vars->volumeCryptKey); + err = IOPolledFileOpen(gIOHibernateFilename, + (kIOPolledFileCreate | kIOPolledFileHibernate), + setFileSize, 0, + gIOHibernateCurrentHeader, sizeof(gIOHibernateCurrentHeader), + &vars->fileVars, &nvramData, + &vars->volumeCryptKey[0], &vars->volumeCryptKeySize); - next = part; - do - { - IOPolledInterface * poller; - OSObject * obj; - OSNumber * num; + if (KERN_SUCCESS != err) + { + IOLockLock(gFSLock); + if (kFSOpening != gFSState) err = kIOReturnTimeout; + IOLockUnlock(gFSLock); + } - obj = next->getProperty(kIOPolledInterfaceSupportKey); - if (kOSBooleanFalse == obj) + if (KERN_SUCCESS != err) { - pollers->flushCollection(); + HIBLOG("IOPolledFileOpen(%x)\n", err); break; } - else if ((poller = OSDynamicCast(IOPolledInterface, obj))) - pollers->setObject(poller); - - if ((service = OSDynamicCast(IOService, next)) - && service->getDeviceMemory() - && !pollers->getCount()) break; - - if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey)))) - *blockSize = num->unsigned32BitValue(); - child = next; - } - while ((next = child->getParentEntry(gIOServicePlane)) - && child->isParent(next, gIOServicePlane, true)); - - if (*blockSize < 4096) *blockSize = 4096; - - if (!pollers->getCount()) - err = kIOReturnUnsupported; - return err; -} + // write extents for debug data usage in EFI + IOWriteExtentsToFile(vars->fileVars, kIOHibernateHeaderOpenSignature); -IOReturn -IOPolledFileOpen( const char * filename, uint64_t setFileSize, - IOBufferMemoryDescriptor * ioBuffer, - IOPolledFileIOVars ** fileVars, OSData ** fileExtents, - OSData ** imagePath, uint8_t * volumeCryptKey) -{ - IOReturn err = kIOReturnSuccess; - IOPolledFileIOVars * vars; - _OpenFileContext ctx; - OSData * extentsData; - IOService * part = 0; - OSString * keyUUID = 0; - OSString * keyStoreUUID = 0; - dev_t block_dev; - dev_t hibernate_image_dev; - uint64_t maxiobytes; - AbsoluteTime startTime, endTime; - uint64_t nsec; - caddr_t write_file_addr = NULL; - vm_size_t write_file_len = 0; - - vars = IONew(IOPolledFileIOVars, 1); - if (!vars) return (kIOReturnNoMemory); - bzero(vars, sizeof(*vars)); + err = IOPolledFilePollersSetup(vars->fileVars, kIOPolledPreflightState); + if (KERN_SUCCESS != err) break; - do - { - vars->io = false; - vars->buffer = (uint8_t *) ioBuffer->getBytesNoCopy(); - vars->bufferHalf = 0; - vars->bufferOffset = 0; - vars->bufferSize = ioBuffer->getLength() >> 1; - - extentsData = OSData::withCapacity(32); - ctx.extents = extentsData; - ctx.size = 0; - clock_get_uptime(&startTime); - if (!gDebugImageFileRef) - { - // Avoid writing the header if it is written when file is prep'd for debug data - // Image is locked during prep for debug data. So, write may fail. - write_file_addr = (caddr_t)gIOHibernateCurrentHeader; - write_file_len = sizeof(IOHibernateImageHeader); - } - vars->fileRef = kern_open_file_for_direct_io(filename, - true, - &file_extent_callback, &ctx, - setFileSize, - // write file: - 0, write_file_addr, - write_file_len, - // results - &block_dev, - &hibernate_image_dev, - &vars->block0, - &maxiobytes, - &vars->flags); -#if 0 - uint32_t msDelay = (131071 & random()); - HIBLOG("sleep %d\n", msDelay); - IOSleep(msDelay); -#endif + clock_get_uptime(&startTime); + err = hibernate_setup(gIOHibernateCurrentHeader, + vmflush, + vars->page_list, vars->page_list_wired, vars->page_list_pal); clock_get_uptime(&endTime); SUB_ABSOLUTETIME(&endTime, &startTime); absolutetime_to_nanoseconds(endTime, &nsec); - if (!vars->fileRef) err = kIOReturnNoSpace; + boolean_t haveSwapPin, hibFileSSD; + haveSwapPin = vm_swap_files_pinned(); - IOLockLock(gFSLock); - if (kFSOpening != gFSState) err = kIOReturnTimeout; - IOLockUnlock(gFSLock); + hibFileSSD = (kIOPolledFileSSD & vars->fileVars->flags); - HIBLOG("kern_open_file_for_direct_io(%d) took %qd ms\n", err, nsec / 1000000ULL); - if (kIOReturnSuccess != err) break; + HIBLOG("hibernate_setup(%d) took %qd ms, swapPin(%d) ssd(%d)\n", + err, nsec / 1000000ULL, + haveSwapPin, hibFileSSD); + if (KERN_SUCCESS != err) break; - if (kIOHibernateModeSSDInvert & gIOHibernateMode) - vars->flags ^= kIOHibernateOptionSSD; + gIOHibernateStandbyDisabled = ((!haveSwapPin || !hibFileSSD)); - HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx ssd %d\n", filename, ctx.size, - vars->block0, maxiobytes, kIOHibernateOptionSSD & vars->flags); - if (ctx.size < 1*1024*1024) // check against image size estimate! + dsSSD = ((0 != (kIOPolledFileSSD & vars->fileVars->flags)) + && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey))); + + if (dsSSD) gIOHibernateCurrentHeader->options |= kIOHibernateOptionSSD | kIOHibernateOptionColor; + else gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress; + + +#if defined(__i386__) || defined(__x86_64__) + if (vars->volumeCryptKeySize && + (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey))) { - err = kIOReturnNoSpace; - break; + uintptr_t smcVars[2]; + smcVars[0] = vars->volumeCryptKeySize; + smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0]; + + IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars)); + bzero(smcVars, sizeof(smcVars)); } +#endif - vars->fileSize = ctx.size; - if (maxiobytes < vars->bufferSize) vars->bufferSize = maxiobytes; - - vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy(); - part = IOCopyMediaForDev(block_dev); - if (!part) - { - err = kIOReturnNotFound; - break; - } - err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false, - (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL); - if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID) - { -// IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy()); - uuid_t volumeKeyUUID; - aks_volume_key_t vek; - static IOService * sKeyStore; - static const OSSymbol * sAKSGetKey; - - if (!sAKSGetKey) - sAKSGetKey = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY); - if (!sKeyStore) - sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane); - if (sKeyStore) - err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID); - else - err = kIOReturnNoResources; - if (kIOReturnSuccess == err) - err = sKeyStore->callPlatformFunction(sAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL); - if (kIOReturnSuccess != err) - IOLog("volume key err 0x%x\n", err); - else - { - size_t bytes = (kIOHibernateAESKeySize / 8); - if (vek.key.keybytecount < bytes) - bytes = vek.key.keybytecount; - bcopy(&vek.key.keybytes[0], volumeCryptKey, bytes); - } - bzero(&vek, sizeof(vek)); - } - part->release(); - - part = IOCopyMediaForDev(hibernate_image_dev); - if (!part) - { - err = kIOReturnNotFound; - break; - } - - vars->pollers = OSArray::withCapacity(4); - if (!vars->pollers) - { - err = kIOReturnNoMemory; - break; - } - - err = GetImageBlockSize(part, vars->pollers, &vars->blockSize); - - HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n", - major(hibernate_image_dev), minor(hibernate_image_dev), (long)vars->blockSize, - vars->pollers->getCount()); - - if (err != kIOReturnSuccess) - break; - - IORegistryEntry * next; - OSData * data; - if (vars->blockSize < sizeof(IOHibernateImageHeader)) - { - err = kIOReturnError; - continue; - } - - err = IOHibernatePollerProbe(vars, (IOService *) part); - if (kIOReturnSuccess != err) break; - - err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer); - if (kIOReturnSuccess != err) break; - - vars->media = part; - next = part; - while (next) - { - next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue); - next = next->getParentEntry(gIOServicePlane); - } - - *fileVars = vars; - *fileExtents = extentsData; - - // make imagePath - - if ((extentsData->getLength() >= sizeof(IOPolledFileExtent))) - { - char str2[24 + sizeof(uuid_string_t) + 2]; - -#if defined(__i386__) || defined(__x86_64__) - if (!gIOCreateEFIDevicePathSymbol) - gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath"); - - if (keyUUID) - snprintf(str2, sizeof(str2), "%qx:%s", - vars->extentMap[0].start, keyUUID->getCStringNoCopy()); - else - snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start); - - err = IOService::getPlatform()->callPlatformFunction( - gIOCreateEFIDevicePathSymbol, false, - (void *) part, (void *) str2, - (void *) (uintptr_t) true, (void *) &data); -#else - char str1[256]; - int len = sizeof(str1); - - if (!part->getPath(str1, &len, gIODTPlane)) - err = kIOReturnNotFound; - else - { - snprintf(str2, sizeof(str2), ",%qx", vars->extentMap[0].start); - // (strip the plane name) - char * tail = strchr(str1, ':'); - if (!tail) - tail = str1 - 1; - data = OSData::withBytes(tail + 1, strlen(tail + 1)); - data->appendBytes(str2, strlen(str2)); - } -#endif - if (kIOReturnSuccess == err) - *imagePath = data; - else - HIBLOG("error 0x%x getting path\n", err); - } - } - while (false); - - if (kIOReturnSuccess != err) - { - HIBLOG("error 0x%x opening hibernation file\n", err); - if (vars->fileRef) - { - kern_close_file_for_direct_io(vars->fileRef, 0, 0, 0, 0, 0); - vars->fileRef = NULL; - } - } - else - { - WriteExtentsToFile(vars->fileRef, kIOHibernateHeaderOpenSignature, vars->blockSize, - (IOPolledFileExtent *)extentsData->getBytesNoCopy(), - extentsData->getLength()); - } - - if (part) - part->release(); - - return (err); -} - -IOReturn -IOPolledFileClose( IOPolledFileIOVars * vars ) -{ - if (vars->pollers) - { - IOHibernatePollerClose(vars, kIOPolledPostflightState); - vars->pollers->release(); - } - - bzero(vars, sizeof(IOPolledFileIOVars)); - - return (kIOReturnSuccess); -} - -static IOReturn -IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position) -{ - IOPolledFileExtent * extentMap; - - extentMap = vars->extentMap; - - vars->position = position; - - while (position >= extentMap->length) - { - position -= extentMap->length; - extentMap++; - } - - vars->currentExtent = extentMap; - vars->extentRemaining = extentMap->length - position; - vars->extentPosition = vars->position - position; - - if (vars->bufferSize <= vars->extentRemaining) - vars->bufferLimit = vars->bufferSize; - else - vars->bufferLimit = vars->extentRemaining; - - return (kIOReturnSuccess); -} - -static IOReturn -IOPolledFileWrite(IOPolledFileIOVars * vars, - const uint8_t * bytes, IOByteCount size, - hibernate_cryptvars_t * cryptvars) -{ - IOReturn err = kIOReturnSuccess; - IOByteCount copy; - bool flush = false; - - do - { - if (!bytes && !size) - { - // seek to end of block & flush - size = vars->position & (vars->blockSize - 1); - if (size) - size = vars->blockSize - size; - flush = true; - // use some garbage for the fill - bytes = vars->buffer + vars->bufferOffset; - } - - copy = vars->bufferLimit - vars->bufferOffset; - if (copy > size) - copy = size; - else - flush = true; - - if (bytes) - { - bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy); - bytes += copy; - } - else - bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy); - - size -= copy; - vars->bufferOffset += copy; - vars->position += copy; - - if (flush && vars->bufferOffset) - { - uint64_t offset = (vars->position - vars->bufferOffset - - vars->extentPosition + vars->currentExtent->start); - uint32_t length = (vars->bufferOffset); - -#if CRYPTO - if (cryptvars && vars->encryptStart - && (vars->position > vars->encryptStart) - && ((vars->position - length) < vars->encryptEnd)) - { - AbsoluteTime startTime, endTime; - - uint64_t encryptLen, encryptStart; - encryptLen = vars->position - vars->encryptStart; - if (encryptLen > length) - encryptLen = length; - encryptStart = length - encryptLen; - if (vars->position > vars->encryptEnd) - encryptLen -= (vars->position - vars->encryptEnd); - - clock_get_uptime(&startTime); - - // encrypt the buffer - aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart, - &cryptvars->aes_iv[0], - encryptLen / AES_BLOCK_SIZE, - vars->buffer + vars->bufferHalf + encryptStart, - &cryptvars->ctx.encrypt); - - clock_get_uptime(&endTime); - ADD_ABSOLUTETIME(&vars->cryptTime, &endTime); - SUB_ABSOLUTETIME(&vars->cryptTime, &startTime); - vars->cryptBytes += encryptLen; - - // save initial vector for following encrypts - bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE, - &cryptvars->aes_iv[0], - AES_BLOCK_SIZE); - } -#endif /* CRYPTO */ - - if (vars->io) - { - err = IOHibernatePollerIODone(vars, true); - if (kIOReturnSuccess != err) - break; - } - -if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position); -//if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length); - - err = IOHibernatePollerIO(vars, kIOPolledWrite, vars->bufferHalf, offset, length); - if (kIOReturnSuccess != err) - break; - vars->io = true; - - vars->extentRemaining -= vars->bufferOffset; - if (!vars->extentRemaining) - { - vars->currentExtent++; - vars->extentRemaining = vars->currentExtent->length; - vars->extentPosition = vars->position; - if (!vars->extentRemaining) - { - err = kIOReturnOverrun; - break; - } - } - - vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize; - vars->bufferOffset = 0; - if (vars->bufferSize <= vars->extentRemaining) - vars->bufferLimit = vars->bufferSize; - else - vars->bufferLimit = vars->extentRemaining; - - flush = false; - } - } - while (size); - - return (err); -} - -static IOReturn -IOPolledFileRead(IOPolledFileIOVars * vars, - uint8_t * bytes, IOByteCount size, - hibernate_cryptvars_t * cryptvars) -{ - IOReturn err = kIOReturnSuccess; - IOByteCount copy; - -// bytesWritten += size; - - do - { - copy = vars->bufferLimit - vars->bufferOffset; - if (copy > size) - copy = size; - - if (bytes) - { - bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy); - bytes += copy; - } - size -= copy; - vars->bufferOffset += copy; -// vars->position += copy; - - if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd)) - { - if (vars->io) - { - err = IOHibernatePollerIODone(vars, false); - if (kIOReturnSuccess != err) - break; - } - else - cryptvars = 0; - -if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position); - - vars->position += vars->lastRead; - vars->extentRemaining -= vars->lastRead; - vars->bufferLimit = vars->lastRead; - - if (!vars->extentRemaining) - { - vars->currentExtent++; - vars->extentRemaining = vars->currentExtent->length; - vars->extentPosition = vars->position; - if (!vars->extentRemaining) - { - err = kIOReturnOverrun; - break; - } - } - - uint64_t length; - uint64_t lastReadLength = vars->lastRead; - uint64_t offset = (vars->position - - vars->extentPosition + vars->currentExtent->start); - if (vars->extentRemaining <= vars->bufferSize) - length = vars->extentRemaining; - else - length = vars->bufferSize; - if ((length + vars->position) > vars->readEnd) - length = vars->readEnd - vars->position; - - vars->lastRead = length; - if (length) - { -//if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length); - err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length); - if (kIOReturnSuccess != err) - break; - vars->io = true; - } - - vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize; - vars->bufferOffset = 0; - -#if CRYPTO - if (cryptvars) - { - uint8_t thisVector[AES_BLOCK_SIZE]; - AbsoluteTime startTime, endTime; - - // save initial vector for following decrypts - bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE); - bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE, - &cryptvars->aes_iv[0], AES_BLOCK_SIZE); - - // decrypt the buffer - clock_get_uptime(&startTime); - - aes_decrypt_cbc(vars->buffer + vars->bufferHalf, - &thisVector[0], - lastReadLength / AES_BLOCK_SIZE, - vars->buffer + vars->bufferHalf, - &cryptvars->ctx.decrypt); - - clock_get_uptime(&endTime); - ADD_ABSOLUTETIME(&vars->cryptTime, &endTime); - SUB_ABSOLUTETIME(&vars->cryptTime, &startTime); - vars->cryptBytes += lastReadLength; - } -#endif /* CRYPTO */ - } - } - while (size); - - return (err); -} - -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#if HIBERNATION -IOReturn -IOHibernateOpenForDebugData( ) -{ - dev_t image_dev; - OSData *extentsData = NULL; - OSObject *obj; - OSString *str; - IOByteCount blockSize = 0; - IOByteCount size; - IOService * part = 0; - OSData * data = NULL; - - IOPolledFileExtent * fileExtents; - IOReturn err = kIOReturnSuccess; - IORegistryEntry * regEntry; - OSArray * pollers = NULL; - - _OpenFileContext ctx; - - if (gDebugImageFileRef != NULL) - return kIOReturnError; - - if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey))) - { - if ((str = OSDynamicCast(OSString, obj))) - strlcpy(gIOHibernateFilename, str->getCStringNoCopy(), - sizeof(gIOHibernateFilename)); - obj->release(); - } - - if (!gIOHibernateFilename[0]) { - HIBLOG("Failed to get hibernate image filename\n"); - return (kIOReturnUnsupported); - } - - extentsData = OSData::withCapacity(32); - ctx.extents = extentsData; - ctx.size = 0; - - bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader)); - gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags; - gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature; - - gDebugImageFileRef = kern_open_file_for_direct_io(gIOHibernateFilename, - false, - &file_extent_callback, &ctx, - 0, 0, - (caddr_t)gIOHibernateCurrentHeader, - sizeof(IOHibernateImageHeader), - NULL, &image_dev, NULL, NULL, NULL); - - if (gDebugImageFileRef == NULL) - { - HIBLOG("Failed to open the file \n"); - err = kIOReturnError; - goto exit; - } - fileExtents = (IOPolledFileExtent *)extentsData->getBytesNoCopy(); - size = extentsData->getLength(); - - part = IOCopyMediaForDev(image_dev); - if (!part) - { - HIBLOG("Failed to get the media device\n"); - err = kIOReturnNotFound; - goto exit; - } - - - pollers = OSArray::withCapacity(4); - if (!pollers) - { - err = kIOReturnNoMemory; - goto exit; - } - - err = GetImageBlockSize(part, pollers, &blockSize); - if (err != kIOReturnSuccess) - { - HIBLOG("Failed to get block size\n"); - goto exit; - } - if (blockSize < sizeof(IOHibernateImageHeader)) - { - HIBLOG("block size %llu is less than the size of the header\n", blockSize); - err = kIOReturnError; - goto exit; - } - - WriteExtentsToFile(gDebugImageFileRef, kIOHibernateHeaderOpenSignature, - blockSize, fileExtents, size); - - char str2[24 + sizeof(uuid_string_t) + 2]; - - if (!gIOCreateEFIDevicePathSymbol) - gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath"); - - snprintf(str2, sizeof(str2), "%qx", fileExtents[0].start); - - err = IOService::getPlatform()->callPlatformFunction( - gIOCreateEFIDevicePathSymbol, false, - (void *) part, (void *) str2, - (void *) (uintptr_t) true, (void *) &data); - - if (!gIOOptionsEntry) - { - regEntry = IORegistryEntry::fromPath("/options", gIODTPlane); - gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry); - if (regEntry && !gIOOptionsEntry) - regEntry->release(); - } - if (gIOOptionsEntry) - { - const OSSymbol * sym; - - sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey); - if (sym) - { - gIOOptionsEntry->setProperty(sym, data); - sym->release(); - } - } - - -exit: - - if ( (err != kIOReturnSuccess) && gDebugImageFileRef) { - kern_close_file_for_direct_io(gDebugImageFileRef, 0, 0, 0, 0, 0); - gDebugImageFileRef = NULL; - } - if (extentsData) extentsData->release(); - if (part) part->release(); - if (pollers) pollers->release(); - if (data) data->release(); - - return err; -} -#endif - -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -IOReturn -IOHibernateSystemSleep(void) -{ - IOReturn err; - OSData * data; - OSObject * obj; - OSString * str; - OSNumber * num; - bool dsSSD, vmflush; - IOHibernateVars * vars; - - gIOHibernateState = kIOHibernateStateInactive; - - if (!gIOChosenEntry) - gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane); - - gIOHibernateDebugFlags = 0; - if (kIOLogHibernate & gIOKitDebug) - gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs; - - if (IOService::getPMRootDomain()->getHibernateSettings( - &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime)) - { - if (kIOHibernateModeSleep & gIOHibernateMode) - // default to discard clean for safe sleep - gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive - | kIOHibernateModeDiscardCleanActive); - } - - if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey))) - { - if ((str = OSDynamicCast(OSString, obj))) - strlcpy(gIOHibernateFilename, str->getCStringNoCopy(), - sizeof(gIOHibernateFilename)); - obj->release(); - } - - if (!gIOHibernateMode || !gIOHibernateFilename[0]) - return (kIOReturnUnsupported); - - HIBLOG("hibernate image path: %s\n", gIOHibernateFilename); - - vars = IONew(IOHibernateVars, 1); - if (!vars) return (kIOReturnNoMemory); - bzero(vars, sizeof(*vars)); - - IOLockLock(gFSLock); - if (kFSIdle != gFSState) - { - HIBLOG("hibernate file busy\n"); - IOLockUnlock(gFSLock); - IODelete(vars, IOHibernateVars, 1); - return (kIOReturnBusy); - } - gFSState = kFSOpening; - IOLockUnlock(gFSLock); - - do - { - vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, - 2 * page_size + WKdm_SCRATCH_BUF_SIZE, page_size); - vars->ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, - 2 * kDefaultIOSize, page_size); - - vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, - ptoa_64(gIOHibernateHandoffPageCount), page_size); - - if (!vars->srcBuffer || !vars->ioBuffer || !vars->handoffBuffer) - { - err = kIOReturnNoMemory; - break; - } - - if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey))) - { - if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMinSize = num->unsigned64BitValue(); - obj->release(); - } - if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey))) - { - if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMaxSize = num->unsigned64BitValue(); - obj->release(); - } - - boolean_t encryptedswap = true; - uint32_t pageCount; - AbsoluteTime startTime, endTime; - uint64_t nsec; - - bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader)); - gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags; - gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature; - - vmflush = (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)); - uint64_t setFileSize = 0; - err = hibernate_alloc_page_lists(&vars->page_list, - &vars->page_list_wired, - &vars->page_list_pal); - if (KERN_SUCCESS != err) - break; - - if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode)) - { - hibernate_page_list_setall(vars->page_list, - vars->page_list_wired, - vars->page_list_pal, - true /* preflight */, - vmflush /* discard */, - &pageCount); - PE_Video consoleInfo; - bzero(&consoleInfo, sizeof(consoleInfo)); - IOService::getPlatform()->getConsoleInfo(&consoleInfo); - - // estimate: 6% increase in pages compressed - // screen preview 2 images compressed 0% - setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8) - + vars->page_list->list_size - + (consoleInfo.v_width * consoleInfo.v_height * 8); - enum { setFileRound = 1024*1024ULL }; - setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1)); - - HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n", - pageCount, (100ULL * gIOHibernateCompression) >> 8, - setFileSize, vars->fileMinSize); - - if (!(kIOHibernateModeFileResize & gIOHibernateMode) - && (setFileSize < vars->fileMinSize)) - { - setFileSize = vars->fileMinSize; - } - } - - // open & invalidate the image file - - if (gDebugImageFileRef) { - kern_close_file_for_direct_io(gDebugImageFileRef, 0, 0, 0, 0, 0); - gDebugImageFileRef = NULL; - } - - err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, vars->ioBuffer, - &vars->fileVars, &vars->fileExtents, &data, - &vars->volumeCryptKey[0]); - - if (KERN_SUCCESS != err) - { - HIBLOG("IOPolledFileOpen(%x)\n", err); - break; - } - - clock_get_uptime(&startTime); - err = hibernate_setup(gIOHibernateCurrentHeader, - gIOHibernateFreeRatio, gIOHibernateFreeTime, - vmflush, - vars->page_list, vars->page_list_wired, vars->page_list_pal); - clock_get_uptime(&endTime); - SUB_ABSOLUTETIME(&endTime, &startTime); - absolutetime_to_nanoseconds(endTime, &nsec); - HIBLOG("hibernate_setup(%d) took %qd ms\n", err, nsec / 1000000ULL); - - dsSSD = ((0 != (kIOHibernateOptionSSD & vars->fileVars->flags)) - && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey))); - if (dsSSD) - { - gIOHibernateCurrentHeader->options |= - kIOHibernateOptionSSD - | kIOHibernateOptionColor; - -#if defined(__i386__) || defined(__x86_64__) - if (!uuid_is_null(vars->volumeCryptKey) && - (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey))) - { - uintptr_t smcVars[2]; - smcVars[0] = sizeof(vars->volumeCryptKey); - smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0]; - - IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars)); - bzero(smcVars, sizeof(smcVars)); - } -#endif - } - else - { - gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress; - } - - - if (KERN_SUCCESS != err) - break; - - if (encryptedswap || !uuid_is_null(vars->volumeCryptKey)) - gIOHibernateMode ^= kIOHibernateModeEncrypt; + if (encryptedswap || vars->volumeCryptKeySize) + gIOHibernateMode ^= kIOHibernateModeEncrypt; if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options) { vars->videoAllocSize = kVideoMapSize; - if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize)) + if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize, VM_KERN_MEMORY_IOKIT)) vars->videoMapping = 0; } @@ -1582,28 +613,11 @@ IOHibernateSystemSleep(void) // set nvram - IORegistryEntry * regEntry; - if (!gIOOptionsEntry) - { - regEntry = IORegistryEntry::fromPath("/options", gIODTPlane); - gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry); - if (regEntry && !gIOOptionsEntry) - regEntry->release(); - } - - if (gIOOptionsEntry) - { - const OSSymbol * sym; - - sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey); - if (sym) - { - gIOOptionsEntry->setProperty(sym, data); - sym->release(); - } - data->release(); + IOSetBootImageNVRAM(nvramData); + nvramData->release(); #if defined(__i386__) || defined(__x86_64__) + { struct AppleRTCHibernateVars { uint8_t signature[4]; @@ -1612,6 +626,7 @@ IOHibernateSystemSleep(void) uint8_t wiredCryptKey[16]; }; AppleRTCHibernateVars rtcVars; + OSData * data; rtcVars.signature[0] = 'A'; rtcVars.signature[1] = 'A'; @@ -1619,175 +634,154 @@ IOHibernateSystemSleep(void) rtcVars.signature[3] = 'L'; rtcVars.revision = 1; bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey)); - if (gIOHibernateBootSignature[0]) + + if (gIOChosenEntry + && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOHibernateBootSignatureKey))) + && (sizeof(rtcVars.booterSignature) <= data->getLength())) + { + bcopy(data->getBytesNoCopy(), &rtcVars.booterSignature[0], sizeof(rtcVars.booterSignature)); + } + else if (gIOHibernateBootSignature[0]) { char c; uint8_t value = 0; - for (uint32_t i = 0; - (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1)); - i++) + uint32_t in, out, digits; + for (in = out = digits = 0; + (c = gIOHibernateBootSignature[in]) && (in < sizeof(gIOHibernateBootSignature)); + in++) { - if (c >= 'a') - c -= 'a' - 10; - else if (c >= 'A') - c -= 'A' - 10; - else if (c >= '0') - c -= '0'; + if ((c >= 'a') && (c <= 'f')) c -= 'a' - 10; + else if ((c >= 'A') && (c <= 'F')) c -= 'A' - 10; + else if ((c >= '0') && (c <= '9')) c -= '0'; else - continue; + { + if (c == '=') out = digits = value = 0; + continue; + } value = (value << 4) | c; - if (i & 1) - rtcVars.booterSignature[i >> 1] = value; + if (digits & 1) + { + rtcVars.booterSignature[out++] = value; + if (out >= sizeof(rtcVars.booterSignature)) break; + } + digits++; } } +#if DEBUG || DEVELOPMENT + if (kIOLogHibernate & gIOKitDebug) IOKitKernelLogBuffer("H> rtc:", + &rtcVars, sizeof(rtcVars), &kprintf); +#endif /* DEBUG || DEVELOPMENT */ + data = OSData::withBytes(&rtcVars, sizeof(rtcVars)); if (data) - { - if (!gIOHibernateRTCVariablesKey) - gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey); + { if (gIOHibernateRTCVariablesKey) IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data); - - if( gIOOptionsEntry ) - { - if( gIOHibernateMode & kIOHibernateModeSwitch ) - { - const OSSymbol *sym; - sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSwitchVarsKey); - if( sym ) - { - gIOOptionsEntry->setProperty(sym, data); /* intentional insecure backup of rtc boot vars */ - sym->release(); - } - } - } - data->release(); } - if (gIOChosenEntry) + if (gIOChosenEntry && gIOOptionsEntry) { data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey)); - if (data) - gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy()); + if (data) gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy()); + // set BootNext + if (!gIOHibernateBoot0082Data) { - // set BootNext - - if (!gIOHibernateBoot0082Data) + OSData * fileData = 0; + data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path")); + if (data && data->getLength() >= 4) fileData = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-file-path")); + if (data) { - data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path")); - if (data) + // AppleNVRAM_EFI_LOAD_OPTION + struct { + uint32_t Attributes; + uint16_t FilePathLength; + uint16_t Desc; + } loadOptionHeader; + loadOptionHeader.Attributes = 1; + loadOptionHeader.FilePathLength = data->getLength(); + loadOptionHeader.Desc = 0; + if (fileData) { - // AppleNVRAM_EFI_LOAD_OPTION - struct { - uint32_t Attributes; - uint16_t FilePathLength; - uint16_t Desc; - } loadOptionHeader; - loadOptionHeader.Attributes = 1; - loadOptionHeader.FilePathLength = data->getLength(); - loadOptionHeader.Desc = 0; - gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength); - if (gIOHibernateBoot0082Data) + loadOptionHeader.FilePathLength -= 4; + loadOptionHeader.FilePathLength += fileData->getLength(); + } + gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength); + if (gIOHibernateBoot0082Data) + { + gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader)); + if (fileData) { - gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader)); - gIOHibernateBoot0082Data->appendBytes(data); + gIOHibernateBoot0082Data->appendBytes(data->getBytesNoCopy(), data->getLength() - 4); + gIOHibernateBoot0082Data->appendBytes(fileData); } + else gIOHibernateBoot0082Data->appendBytes(data); } } - if (!gIOHibernateBoot0082Key) - gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082"); - if (!gIOHibernateBootNextKey) - gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext"); - if (!gIOHibernateBootNextData) - { - uint16_t bits = 0x0082; - gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits)); - } - if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData) - { - gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey); - gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data); - gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData); - } - } - } -#else /* !i386 && !x86_64 */ - if (kIOHibernateModeEncrypt & gIOHibernateMode) - { - data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey)); - sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey); - if (sym && data) - gIOOptionsEntry->setProperty(sym, data); - if (sym) - sym->release(); - if (data) - data->release(); - if (false && gIOHibernateBootSignature[0]) - { - data = OSData::withCapacity(16); - sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey); - if (sym && data) - { - char c; - uint8_t value = 0; - for (uint32_t i = 0; (c = gIOHibernateBootSignature[i]); i++) - { - if (c >= 'a') - c -= 'a' - 10; - else if (c >= 'A') - c -= 'A' - 10; - else if (c >= '0') - c -= '0'; - else - continue; - value = (value << 4) | c; - if (i & 1) - data->appendBytes(&value, sizeof(value)); - } - gIOOptionsEntry->setProperty(sym, data); - } - if (sym) - sym->release(); - if (data) - data->release(); - } - } - if (!vars->haveFastBoot) - { - // set boot volume to zero - IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform()); - if (platform && (kIOReturnSuccess == platform->readXPRAM(kXPRamAudioVolume, - &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume)))) - { - uint8_t newVolume; - newVolume = vars->saveBootAudioVolume & 0xf8; - platform->writeXPRAM(kXPRamAudioVolume, - &newVolume, sizeof(newVolume)); - } + } + if (!gIOHibernateBootNextData) + { + uint16_t bits = 0x0082; + gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits)); + } + +#if DEBUG || DEVELOPMENT + if (kIOLogHibernate & gIOKitDebug) IOKitKernelLogBuffer("H> bootnext:", + gIOHibernateBoot0082Data->getBytesNoCopy(), gIOHibernateBoot0082Data->getLength(), &kprintf); +#endif /* DEBUG || DEVELOPMENT */ + if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData) + { + gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey); + gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data); + gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData); + } + // BootNext } -#endif /* !i386 && !x86_64 */ } - // -- - +#endif /* !i386 && !x86_64 */ } while (false); + if (swapPinned) hibernate_pin_swap(FALSE); + IOLockLock(gFSLock); - if ((kIOReturnSuccess == err) && (kFSOpening == gFSState)) + if ((kIOReturnSuccess == err) && (kFSOpening != gFSState)) + { + HIBLOG("hibernate file close due timeout\n"); + err = kIOReturnTimeout; + } + if (kIOReturnSuccess == err) { gFSState = kFSOpened; gIOHibernateVars = *vars; gFileVars = *vars->fileVars; + gFileVars.allocated = false; gIOHibernateVars.fileVars = &gFileVars; - gIOHibernateFileRef = gFileVars.fileRef; gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature; gIOHibernateState = kIOHibernateStateHibernating; + +#if DEBUG || DEVELOPMENT + if (kIOLogHibernate & gIOKitDebug) + { + OSData * data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey)); + if (data) + { + uintptr_t * smcVars = (typeof(smcVars)) data->getBytesNoCopy(); + IOKitKernelLogBuffer("H> smc:", + (const void *)smcVars[1], smcVars[0], &kprintf); + } + } +#endif /* DEBUG || DEVELOPMENT */ } else { - HIBLOG("hibernate file close due timeout\n"); - if (vars->fileVars && vars->fileVars->fileRef) kern_close_file_for_direct_io(vars->fileVars->fileRef, 0, 0, 0, 0, 0); + IOPolledFileIOVars * fileVars = vars->fileVars; IOHibernateDone(vars); + IOPolledFileClose(&fileVars, +#if DISABLE_TRIM + 0, NULL, 0, 0, 0); +#else + 0, NULL, 0, sizeof(IOHibernateImageHeader), setFileSize); +#endif gFSState = kFSIdle; } IOLockUnlock(gFSLock); @@ -1800,6 +794,93 @@ IOHibernateSystemSleep(void) /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +static void +IOSetBootImageNVRAM(OSData * data) +{ + IORegistryEntry * regEntry; + + if (!gIOOptionsEntry) + { + regEntry = IORegistryEntry::fromPath("/options", gIODTPlane); + gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry); + if (regEntry && !gIOOptionsEntry) + regEntry->release(); + } + if (gIOOptionsEntry && gIOHibernateBootImageKey) + { + if (data) + { + gIOOptionsEntry->setProperty(gIOHibernateBootImageKey, data); +#if DEBUG || DEVELOPMENT + if (kIOLogHibernate & gIOKitDebug) IOKitKernelLogBuffer("H> boot-image:", + data->getBytesNoCopy(), data->getLength(), &kprintf); +#endif /* DEBUG || DEVELOPMENT */ + } + else + { + gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey); + gIOOptionsEntry->sync(); + } + } +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +/* + * Writes header to disk with signature, block size and file extents data. + * If there are more than 2 extents, then they are written on second block. + */ +static IOReturn +IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature) +{ + IOHibernateImageHeader hdr; + IOItemCount count; + IOReturn err = kIOReturnSuccess; + int rc; + IOPolledFileExtent * fileExtents; + + fileExtents = (typeof(fileExtents)) vars->fileExtents->getBytesNoCopy(); + + memset(&hdr, 0, sizeof(IOHibernateImageHeader)); + count = vars->fileExtents->getLength(); + if (count > sizeof(hdr.fileExtentMap)) + { + hdr.fileExtentMapSize = count; + count = sizeof(hdr.fileExtentMap); + } + else + hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap); + + bcopy(fileExtents, &hdr.fileExtentMap[0], count); + + // copy file block extent list if larger than header + if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap)) + { + count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap); + rc = kern_write_file(vars->fileRef, vars->blockSize, + (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)), + count, IO_SKIP_ENCRYPTION); + if (rc != 0) { + HIBLOG("kern_write_file returned %d\n", rc); + err = kIOReturnIOError; + goto exit; + } + } + hdr.signature = signature; + hdr.deviceBlockSize = vars->blockSize; + + rc = kern_write_file(vars->fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION); + if (rc != 0) { + HIBLOG("kern_write_file returned %d\n", rc); + err = kIOReturnIOError; + goto exit; + } + +exit: + return err; +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + DECLARE_IOHIBERNATEPROGRESSALPHA static void @@ -1815,11 +896,11 @@ ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveund rowBytes = display->rowBytes; pixelShift = display->depth >> 4; if (pixelShift < 1) return; - + screen += ((display->width - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1)) + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes; - + for (y = 0; y < kIOHibernateProgressHeight; y++) { out = screen + y * rowBytes; @@ -1876,7 +957,7 @@ ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBl rowBytes = display->rowBytes; - screen += ((display->width + screen += ((display->width - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1)) + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes; @@ -1964,8 +1045,8 @@ IOHibernateSystemHasSlept(void) } if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options) - && vars->previewBuffer - && (data = OSDynamicCast(OSData, + && vars->previewBuffer + && (data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey)))) { UInt32 flags = *((UInt32 *)data->getBytesNoCopy()); @@ -1987,7 +1068,7 @@ IOHibernateSystemHasSlept(void) vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr; HIBPRINT("video %p %d %d %d\n", - vars->consoleMapping, graphicsInfo->depth, + vars->consoleMapping, graphicsInfo->depth, graphicsInfo->width, graphicsInfo->height); if (vars->consoleMapping) ProgressInit(graphicsInfo, vars->consoleMapping, @@ -2040,6 +1121,7 @@ IOHibernateSystemWake(void) { if (kFSOpened == gFSState) { + IOPolledFilePollersClose(gIOHibernateVars.fileVars, kIOPolledPostflightState); IOHibernateDone(&gIOHibernateVars); } else @@ -2053,7 +1135,8 @@ IOHibernateSystemWake(void) static IOReturn IOHibernateDone(IOHibernateVars * vars) { - IORegistryEntry * next; + IOReturn err; + OSData * data; hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal); @@ -2075,7 +1158,7 @@ IOHibernateDone(IOHibernateVars * vars) if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) { - IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey, + IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey, gIOHibernateCurrentHeader->options, 32); } else @@ -2086,7 +1169,7 @@ IOHibernateDone(IOHibernateVars * vars) if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus)) { - IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey, + IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey, &gIOHibernateGraphicsInfo->gfxStatus, sizeof(gIOHibernateGraphicsInfo->gfxStatus)); } @@ -2095,17 +1178,6 @@ IOHibernateDone(IOHibernateVars * vars) IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey); } - if (vars->fileVars) - { - if ((next = vars->fileVars->media)) do - { - next->removeProperty(kIOPolledInterfaceActiveKey); - next = next->getParentEntry(gIOServicePlane); - } - while (next); - IOPolledFileClose(vars->fileVars); - } - // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched #if defined(__i386__) || defined(__x86_64__) @@ -2139,10 +1211,7 @@ IOHibernateDone(IOHibernateVars * vars) } #endif - if (vars->srcBuffer) - vars->srcBuffer->release(); - if (vars->ioBuffer) - vars->ioBuffer->release(); + if (vars->srcBuffer) vars->srcBuffer->release(); bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0])); if (vars->handoffBuffer) { @@ -2165,7 +1234,7 @@ IOHibernateDone(IOHibernateVars * vars) case kIOHibernateHandoffTypeDeviceTree: MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot()); break; - + case kIOHibernateHandoffTypeKeyStore: #if defined(__i386__) || defined(__x86_64__) { @@ -2178,17 +1247,41 @@ IOHibernateDone(IOHibernateVars * vars) } #endif break; - + default: done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000)); break; - } + } + } +#if defined(__i386__) || defined(__x86_64__) + if (vars->volumeCryptKeySize) + { + IOBufferMemoryDescriptor * + bmd = IOBufferMemoryDescriptor::withBytes(&vars->volumeCryptKey[0], + vars->volumeCryptKeySize, kIODirectionOutIn); + if (!bmd) panic("IOBufferMemoryDescriptor"); + IOSetAPFSKeyStoreData(bmd); + bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey)); } +#endif + } vars->handoffBuffer->release(); } - if (vars->fileExtents) - vars->fileExtents->release(); + + if (gIOChosenEntry + && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey))) + && (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength())) + { + bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0], + sizeof(gIOHibernateBridgeBootSessionUUIDString)); + } + + if (vars->hwEncrypt) + { + err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, NULL, 0); + HIBLOG("IOPolledFilePollersSetEncryptionKey(0,%x)\n", err); + } bzero(vars, sizeof(*vars)); @@ -2197,116 +1290,122 @@ IOHibernateDone(IOHibernateVars * vars) return (kIOReturnSuccess); } -IOReturn -IOHibernateSystemPostWake(void) +static void +IOHibernateSystemPostWakeTrim(void * p1, void * p2) { - struct kern_direct_file_io_ref_t * fileRef; - - if (kFSOpened == gFSState) + // invalidate & close the image file + if (p1) IOLockLock(gFSLock); + if (kFSTrimDelay == gFSState) { - // invalidate & close the image file - gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature; - if ((fileRef = gIOHibernateFileRef)) - { - gIOHibernateFileRef = 0; - IOSleep(TRIM_DELAY); - kern_close_file_for_direct_io(fileRef, + IOPolledFileIOVars * vars = &gFileVars; + IOPolledFileClose(&vars, #if DISABLE_TRIM - 0, 0, 0, 0, 0); + 0, NULL, 0, 0, 0); #else - 0, (caddr_t) gIOHibernateCurrentHeader, - sizeof(IOHibernateImageHeader), - 0, - gIOHibernateCurrentHeader->imageSize); + 0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader), + sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize); #endif - } - gFSState = kFSIdle; + gFSState = kFSIdle; } + if (p1) IOLockUnlock(gFSLock); +} - if (gDebugImageFileRef) { - kern_close_file_for_direct_io(gDebugImageFileRef, 0, 0, 0, 0, 0); - gDebugImageFileRef = NULL; - } +IOReturn +IOHibernateSystemPostWake(bool now) +{ + gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature; + IOSetBootImageNVRAM(0); - if (!gIOOptionsEntry) + IOLockLock(gFSLock); + if (kFSTrimDelay == gFSState) { - IORegistryEntry * regEntry; - regEntry = IORegistryEntry::fromPath("/options", gIODTPlane); - gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry); - if (regEntry && !gIOOptionsEntry) - regEntry->release(); + thread_call_cancel(gIOHibernateTrimCalloutEntry); + IOHibernateSystemPostWakeTrim(NULL, NULL); } - if (gIOOptionsEntry) + else if (kFSOpened != gFSState) gFSState = kFSIdle; + else { - const OSSymbol * sym; - - sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey); - if (sym) - { - gIOOptionsEntry->removeProperty(sym); - gIOOptionsEntry->sync(); - sym->release(); - } + gFSState = kFSTrimDelay; + if (now) + { + thread_call_cancel(gIOHibernateTrimCalloutEntry); + IOHibernateSystemPostWakeTrim(NULL, NULL); + } + else + { + AbsoluteTime deadline; + clock_interval_to_deadline(TRIM_DELAY, kMillisecondScale, &deadline ); + thread_call_enter1_delayed(gIOHibernateTrimCalloutEntry, NULL, deadline); + } } + IOLockUnlock(gFSLock); return (kIOReturnSuccess); } -bool IOHibernateWasScreenLocked(void) +uint32_t IOHibernateWasScreenLocked(void) { - bool ret = false; + uint32_t ret = 0; if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry) { OSData * data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey)); - if (data) switch (*((uint32_t *)data->getBytesNoCopy())) + if (data) { - case kIOScreenLockLocked: - case kIOScreenLockFileVaultDialog: - ret = true; - break; - case kIOScreenLockNoLock: - case kIOScreenLockUnlocked: - default: - ret = false; - break; - } + ret = ((uint32_t *)data->getBytesNoCopy())[0]; + gIOChosenEntry->setProperty(kIOBooterScreenLockStateKey, data); + } } + else gIOChosenEntry->removeProperty(kIOBooterScreenLockStateKey); + return (ret); } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -SYSCTL_STRING(_kern, OID_AUTO, hibernatefile, +SYSCTL_STRING(_kern, OID_AUTO, hibernatefile, CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED, gIOHibernateFilename, sizeof(gIOHibernateFilename), ""); -SYSCTL_STRING(_kern, OID_AUTO, bootsignature, +SYSCTL_STRING(_kern, OID_AUTO, bootsignature, CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED, gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), ""); -SYSCTL_UINT(_kern, OID_AUTO, hibernatemode, +SYSCTL_UINT(_kern, OID_AUTO, hibernatemode, CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED, &gIOHibernateMode, 0, ""); SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics, CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED, - gIOHibernateStats, hibernate_statistics_t, ""); + &_hibernateStats, hibernate_statistics_t, ""); +SYSCTL_STRING(_kern_bridge, OID_AUTO, bootsessionuuid, + CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED, + gIOHibernateBridgeBootSessionUUIDString, sizeof(gIOHibernateBridgeBootSessionUUIDString), ""); SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready, CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY, - &gIOHibernateStats->graphicsReadyTime, 0, ""); + &_hibernateStats.graphicsReadyTime, 0, ""); SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification, CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY, - &gIOHibernateStats->wakeNotificationTime, 0, ""); + &_hibernateStats.wakeNotificationTime, 0, ""); SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready, CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY, - &gIOHibernateStats->lockScreenReadyTime, 0, ""); + &_hibernateStats.lockScreenReadyTime, 0, ""); SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready, CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY, - &gIOHibernateStats->hidReadyTime, 0, ""); - + &_hibernateStats.hidReadyTime, 0, ""); void IOHibernateSystemInit(IOPMrootDomain * rootDomain) { + gIOHibernateBootImageKey = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey); + gIOHibernateBootSignatureKey = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey); + gIOBridgeBootSessionUUIDKey = OSSymbol::withCStringNoCopy(kIOBridgeBootSessionUUIDKey); + +#if defined(__i386__) || defined(__x86_64__) + gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey); + gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082"); + gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext"); + gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey); +#endif /* defined(__i386__) || defined(__x86_64__) */ + OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState)); if (data) { @@ -2328,58 +1427,35 @@ IOHibernateSystemInit(IOPMrootDomain * rootDomain) sysctl_register_oid(&sysctl__kern_hibernatelockscreenready); sysctl_register_oid(&sysctl__kern_hibernatehidready); - gFSLock = IOLockAlloc(); -} - - -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -static void -hibernate_setup_for_wake(void) -{ -} - -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#define C_ASSERT(e) typedef char __C_ASSERT__[(e) ? 1 : -1] + gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane); -static bool -no_encrypt_page(vm_offset_t ppnum) -{ - if (pmap_is_noencrypt((ppnum_t)ppnum) == TRUE) + if (gIOChosenEntry + && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey))) + && (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength())) { - return true; + sysctl_register_oid(&sysctl__kern_bridge_bootsessionuuid); + bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0], sizeof(gIOHibernateBridgeBootSessionUUIDString)); } - return false; -} - -static void -hibernate_pal_callback(void *vars_arg, vm_offset_t addr) -{ - IOHibernateVars *vars = (IOHibernateVars *)vars_arg; - /* Make sure it's not in either of the save lists */ - hibernate_set_page_state(vars->page_list, vars->page_list_wired, atop_64(addr), 1, kIOHibernatePageStateFree); - /* Set it in the bitmap of pages owned by the PAL */ - hibernate_page_bitset(vars->page_list_pal, TRUE, atop_64(addr)); + gFSLock = IOLockAlloc(); } -static struct hibernate_cryptvars_t *local_cryptvars; +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -extern "C" int -hibernate_pal_write(void *buffer, size_t size) +static IOReturn +IOHibernatePolledFileWrite(IOPolledFileIOVars * vars, + const uint8_t * bytes, IOByteCount size, + IOPolledFileCryptVars * cryptvars) { - IOHibernateVars * vars = &gIOHibernateVars; + IOReturn err; - IOReturn err = IOPolledFileWrite(vars->fileVars, (const uint8_t *)buffer, size, local_cryptvars); - if (kIOReturnSuccess != err) { - kprintf("epic hibernate fail! %d\n", err); - return err; - } + err = IOPolledFileWrite(vars, bytes, size, cryptvars); + if ((kIOReturnSuccess == err) && hibernate_should_abort()) err = kIOReturnAborted; - return 0; + return (err); } +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ extern "C" uint32_t hibernate_write_image(void) @@ -2388,7 +1464,7 @@ hibernate_write_image(void) IOHibernateVars * vars = &gIOHibernateVars; IOPolledFileExtent * fileExtents; - C_ASSERT(sizeof(IOHibernateImageHeader) == 512); + _static_assert_1_arg(sizeof(IOHibernateImageHeader) == 512); uint32_t pageCount, pagesDone; IOReturn err; @@ -2398,8 +1474,7 @@ hibernate_write_image(void) uint8_t * data; uint8_t * compressed; uint8_t * scratch; - void * zerosCompressed; - IOByteCount pageCompressedSize, zerosCompressedLen; + IOByteCount pageCompressedSize; uint64_t compressedSize, uncompressedSize; uint64_t image1Size = 0; uint32_t bitmap_size; @@ -2411,6 +1486,8 @@ hibernate_write_image(void) uint32_t pageAndCount[2]; addr64_t phys64; IOByteCount segLen; + uintptr_t hibernateBase; + uintptr_t hibernateEnd; AbsoluteTime startTime, endTime; AbsoluteTime allTime, compTime; @@ -2423,29 +1500,30 @@ hibernate_write_image(void) uint32_t wiredPagesEncrypted; uint32_t dirtyPagesEncrypted; uint32_t wiredPagesClear; - uint32_t zeroPageCount; + uint32_t svPageCount; + uint32_t zvPageCount; - hibernate_cryptvars_t _cryptvars; - hibernate_cryptvars_t * cryptvars = 0; + IOPolledFileCryptVars _cryptvars; + IOPolledFileCryptVars * cryptvars = 0; wiredPagesEncrypted = 0; dirtyPagesEncrypted = 0; wiredPagesClear = 0; - zeroPageCount = 0; + svPageCount = 0; + zvPageCount = 0; - if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents) - return (false /* sleep */ ); + if (!vars->fileVars + || !vars->fileVars->pollers + || !(kIOHibernateModeOn & gIOHibernateMode)) return (kIOHibernatePostWriteSleep); if (kIOHibernateModeSleep & gIOHibernateMode) kdebug_enable = save_kdebug_enable; - KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START, 0, 0, 0, 0, 0); + KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START); IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate); restore1Sum = sum1 = sum2 = 0; - hibernate_pal_prepare(); - #if CRYPTO // encryption data. "iv" is the "initial vector". if (kIOHibernateModeEncrypt & gIOHibernateMode) @@ -2453,9 +1531,9 @@ hibernate_write_image(void) static const unsigned char first_iv[AES_BLOCK_SIZE] = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c, 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda }; - + cryptvars = &gIOHibernateCryptWakeContext; - bzero(cryptvars, sizeof(hibernate_cryptvars_t)); + bzero(cryptvars, sizeof(IOPolledFileCryptVars)); aes_encrypt_key(vars->cryptKey, kIOHibernateAESKeySize, &cryptvars->ctx.encrypt); @@ -2464,10 +1542,9 @@ hibernate_write_image(void) &cryptvars->ctx.decrypt); cryptvars = &_cryptvars; - bzero(cryptvars, sizeof(hibernate_cryptvars_t)); + bzero(cryptvars, sizeof(IOPolledFileCryptVars)); for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++) vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount]; - bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey)); aes_encrypt_key(vars->wiredCryptKey, kIOHibernateAESKeySize, &cryptvars->ctx.encrypt); @@ -2475,31 +1552,27 @@ hibernate_write_image(void) bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE); bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey)); bzero(&vars->cryptKey[0], sizeof(vars->cryptKey)); - - local_cryptvars = cryptvars; } #endif /* CRYPTO */ - hibernate_setup_for_wake(); - hibernate_page_list_setall(vars->page_list, vars->page_list_wired, vars->page_list_pal, false /* !preflight */, /* discard_all */ - ((0 == (kIOHibernateModeSleep & gIOHibernateMode)) + ((0 == (kIOHibernateModeSleep & gIOHibernateMode)) && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))), &pageCount); HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount); - fileExtents = (IOPolledFileExtent *) vars->fileExtents->getBytesNoCopy(); + fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy(); #if 0 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent); for (page = 0; page < count; page++) { - HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page, + HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page, fileExtents[page].start, fileExtents[page].length, fileExtents[page].start + fileExtents[page].length); } @@ -2510,41 +1583,49 @@ hibernate_write_image(void) compBytes = 0; clock_get_uptime(&allTime); - IOService::getPMRootDomain()->pmStatsRecordEvent( + IOService::getPMRootDomain()->pmStatsRecordEvent( kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime); - do + do { compressedSize = 0; uncompressedSize = 0; - zeroPageCount = 0; + svPageCount = 0; + zvPageCount = 0; IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize); - - HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n", + + HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled()); - err = IOHibernatePollerOpen(vars->fileVars, kIOPolledBeforeSleepState, vars->ioBuffer); + err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState, + // abortable if not low battery + !IOService::getPMRootDomain()->mustHibernate()); HIBLOG("IOHibernatePollerOpen(%x)\n", err); pollerOpen = (kIOReturnSuccess == err); if (!pollerOpen) break; - + + if (vars->volumeCryptKeySize) + { + err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, &vars->volumeCryptKey[0], vars->volumeCryptKeySize); + HIBLOG("IOPolledFilePollersSetEncryptionKey(%x)\n", err); + vars->hwEncrypt = (kIOReturnSuccess == err); + bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey)); + if (vars->hwEncrypt) header->options |= kIOHibernateOptionHWEncrypt; + } + // copy file block extent list if larger than header - - count = vars->fileExtents->getLength(); + + count = vars->fileVars->fileExtents->getLength(); if (count > sizeof(header->fileExtentMap)) { count -= sizeof(header->fileExtentMap); - err = IOPolledFileWrite(vars->fileVars, + err = IOHibernatePolledFileWrite(vars->fileVars, ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars); if (kIOReturnSuccess != err) break; } - uintptr_t hibernateBase; - uintptr_t hibernateEnd; - hibernateBase = HIB_BASE; /* Defined in PAL headers */ - hibernateEnd = (segHIBB + segSizeHIB); // copy out restore1 code @@ -2567,6 +1648,11 @@ hibernate_write_image(void) header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase; header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase; + if (uuid_parse(&gIOHibernateBridgeBootSessionUUIDString[0], &header->bridgeBootSessionUUID[0])) + { + bzero(&header->bridgeBootSessionUUID[0], sizeof(header->bridgeBootSessionUUID)); + } + // sum __HIB seg, with zeros for the stack src = (uint8_t *) trunc_page(hibernateBase); for (page = 0; page < count; page++) @@ -2578,18 +1664,18 @@ hibernate_write_image(void) src += page_size; } sum1 = restore1Sum; - + // write the __HIB seg, with zeros for the stack src = (uint8_t *) trunc_page(hibernateBase); count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase); if (count) { - err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars); + err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars); if (kIOReturnSuccess != err) break; } - err = IOPolledFileWrite(vars->fileVars, + err = IOHibernatePolledFileWrite(vars->fileVars, (uint8_t *) 0, &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0], cryptvars); @@ -2599,12 +1685,12 @@ hibernate_write_image(void) count = round_page(hibernateEnd) - ((uintptr_t) src); if (count) { - err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars); + err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars); if (kIOReturnSuccess != err) break; } - if (kIOHibernateModeEncrypt & gIOHibernateMode) + if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) { vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1)); vars->fileVars->encryptEnd = UINT64_MAX; @@ -2622,8 +1708,8 @@ hibernate_write_image(void) phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone); pageAndCount[0] = atop_64(phys64); pageAndCount[1] = atop_32(segLen); - err = IOPolledFileWrite(vars->fileVars, - (const uint8_t *) &pageAndCount, sizeof(pageAndCount), + err = IOHibernatePolledFileWrite(vars->fileVars, + (const uint8_t *) &pageAndCount, sizeof(pageAndCount), cryptvars); if (kIOReturnSuccess != err) break; @@ -2648,46 +1734,47 @@ hibernate_write_image(void) phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone); sum1 += hibernate_sum_page(src + page, atop_64(phys64)); } - err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars); + err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars); if (kIOReturnSuccess != err) break; } // mark areas for no save - + IOMemoryDescriptor * ioBuffer; + ioBuffer = IOPolledFileGetIOBuffer(vars->fileVars); for (count = 0; - (phys64 = vars->ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone)); + (phys64 = ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone)); count += segLen) { - hibernate_set_page_state(vars->page_list, vars->page_list_wired, + hibernate_set_page_state(vars->page_list, vars->page_list_wired, atop_64(phys64), atop_32(segLen), kIOHibernatePageStateFree); pageCount -= atop_32(segLen); } - + for (count = 0; (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone)); count += segLen) { - hibernate_set_page_state(vars->page_list, vars->page_list_wired, + hibernate_set_page_state(vars->page_list, vars->page_list_wired, atop_64(phys64), atop_32(segLen), kIOHibernatePageStateFree); pageCount -= atop_32(segLen); } // copy out bitmap of pages available for trashing during restore - + bitmap_size = vars->page_list_wired->list_size; src = (uint8_t *) vars->page_list_wired; - err = IOPolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars); + err = IOHibernatePolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars); if (kIOReturnSuccess != err) break; - // mark more areas for no save, but these are not available + // mark more areas for no save, but these are not available // for trashing during restore hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount); - + page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase)); count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page; @@ -2700,7 +1787,7 @@ hibernate_write_image(void) (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone)); count += segLen) { - hibernate_set_page_state(vars->page_list, vars->page_list_wired, + hibernate_set_page_state(vars->page_list, vars->page_list_wired, atop_64(phys64), atop_32(segLen), kIOHibernatePageStateFree); pageCount -= atop_32(segLen); @@ -2710,36 +1797,36 @@ hibernate_write_image(void) (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone)); count += segLen) { - hibernate_set_page_state(vars->page_list, vars->page_list_wired, + hibernate_set_page_state(vars->page_list, vars->page_list_wired, atop_64(phys64), atop_32(segLen), kIOHibernatePageStateFree); pageCount -= atop_32(segLen); } - (void)hibernate_pal_callback; +#if KASAN + vm_size_t shadow_pages_free = atop_64(shadow_ptop) - atop_64(shadow_pnext); + + /* no need to save unused shadow pages */ + hibernate_set_page_state(vars->page_list, vars->page_list_wired, + atop_64(shadow_pnext), + shadow_pages_free, + kIOHibernatePageStateFree); +#endif src = (uint8_t *) vars->srcBuffer->getBytesNoCopy(); compressed = src + page_size; scratch = compressed + page_size; - // compress a zero page - bzero(src, page_size); - zerosCompressed = vars->handoffBuffer->getBytesNoCopy(); - zerosCompressedLen = WKdm_compress_new((WK_word*) src, - (WK_word*) zerosCompressed, - (WK_word*) scratch, - page_size - 4); - pagesDone = 0; lastBlob = 0; - - HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n", + + HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n", bitmap_size, header->previewSize, pageCount, vars->fileVars->position); enum // pageType - { + { kWired = 0x02, kEncrypt = 0x01, kWiredEncrypt = kWired | kEncrypt, @@ -2747,45 +1834,56 @@ hibernate_write_image(void) kUnwiredEncrypt = kEncrypt }; + bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features())); + for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--) { if (kUnwiredEncrypt == pageType) - { + { // start unwired image - if (kIOHibernateModeEncrypt & gIOHibernateMode) + if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) { vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1)); vars->fileVars->encryptEnd = UINT64_MAX; HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart); } - bcopy(&cryptvars->aes_iv[0], - &gIOHibernateCryptWakeContext.aes_iv[0], + bcopy(&cryptvars->aes_iv[0], + &gIOHibernateCryptWakeContext.aes_iv[0], sizeof(cryptvars->aes_iv)); cryptvars = &gIOHibernateCryptWakeContext; } for (iterDone = false, ppnum = 0; !iterDone; ) { - count = hibernate_page_list_iterate((kWired & pageType) - ? vars->page_list_wired : vars->page_list, - &ppnum); + if (cpuAES && (pageType == kWiredClear)) + { + count = 0; + } + else + { + count = hibernate_page_list_iterate((kWired & pageType) ? vars->page_list_wired : vars->page_list, + &ppnum); + } // kprintf("[%d](%x : %x)\n", pageType, ppnum, count); iterDone = !count; - - if (count && (kWired & pageType) && needEncrypt) - { - uint32_t checkIndex; - for (checkIndex = 0; - (checkIndex < count) - && (((kEncrypt & pageType) == 0) == no_encrypt_page(ppnum + checkIndex)); - checkIndex++) - {} - if (!checkIndex) - { - ppnum++; - continue; - } - count = checkIndex; - } + + if (!cpuAES) + { + if (count && (kWired & pageType) && needEncrypt) + { + uint32_t checkIndex; + for (checkIndex = 0; + (checkIndex < count) + && (((kEncrypt & pageType) == 0) == pmap_is_noencrypt(ppnum + checkIndex)); + checkIndex++) + {} + if (!checkIndex) + { + ppnum++; + continue; + } + count = checkIndex; + } + } switch (pageType) { @@ -2793,19 +1891,19 @@ hibernate_write_image(void) case kWiredClear: wiredPagesClear += count; break; case kUnwiredEncrypt: dirtyPagesEncrypted += count; break; } - + if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */} else { pageAndCount[0] = ppnum; pageAndCount[1] = count; - err = IOPolledFileWrite(vars->fileVars, - (const uint8_t *) &pageAndCount, sizeof(pageAndCount), + err = IOHibernatePolledFileWrite(vars->fileVars, + (const uint8_t *) &pageAndCount, sizeof(pageAndCount), cryptvars); if (kIOReturnSuccess != err) break; } - + for (page = ppnum; page < (ppnum + count); page++) { err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size); @@ -2814,16 +1912,16 @@ hibernate_write_image(void) HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err); break; } - + sum = hibernate_sum_page(src, page); if (kWired & pageType) sum1 += sum; else sum2 += sum; - + clock_get_uptime(&startTime); - wkresult = WKdm_compress_new((WK_word*) src, - (WK_word*) compressed, + wkresult = WKdm_compress_new((const WK_word*) src, + (WK_word*) compressed, (WK_word*) scratch, page_size - 4); @@ -2834,34 +1932,37 @@ hibernate_write_image(void) compBytes += page_size; pageCompressedSize = (-1 == wkresult) ? page_size : wkresult; - if ((pageCompressedSize == zerosCompressedLen) - && !bcmp(compressed, zerosCompressed, zerosCompressedLen)) + if (pageCompressedSize == 0) { - pageCompressedSize = 0; - zeroPageCount++; - } + pageCompressedSize = 4; + data = src; - if (kIOHibernateModeEncrypt & gIOHibernateMode) - pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1); + if (*(uint32_t *)src) + svPageCount++; + else + zvPageCount++; + } + else + { + if (pageCompressedSize != page_size) + data = compressed; + else + data = src; + } - if (pageCompressedSize != page_size) - data = compressed; - else - data = src; - tag = pageCompressedSize | kIOHibernateTagSignature; - err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars); + err = IOHibernatePolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars); if (kIOReturnSuccess != err) break; - - err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars); + + err = IOHibernatePolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars); if (kIOReturnSuccess != err) break; - + compressedSize += pageCompressedSize; uncompressedSize += page_size; pagesDone++; - + if (vars->consoleMapping && (0 == (1023 & pagesDone))) { blob = ((pagesDone * kIOHibernateProgressCount) / pageCount); @@ -2901,14 +2002,14 @@ hibernate_write_image(void) if (kWiredEncrypt != pageType) { // end of image1/2 - fill to next block - err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars); + err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars); if (kIOReturnSuccess != err) break; } if (kWiredClear == pageType) { // enlarge wired image for test -// err = IOPolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars); +// err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars); // end wired image header->encryptStart = vars->fileVars->encryptStart; @@ -2922,19 +2023,22 @@ hibernate_write_image(void) { if (kIOReturnOverrun == err) { - // update actual compression ratio on not enough space + // update actual compression ratio on not enough space (for retry) gIOHibernateCompression = (compressedSize << 8) / uncompressedSize; } + + // update partial amount written (for IOPolledFileClose cleanup/unmap) + header->imageSize = vars->fileVars->position; break; } // Header: - + header->imageSize = vars->fileVars->position; header->image1Size = image1Size; header->bitmapSize = bitmap_size; header->pageCount = pageCount; - + header->restore1Sum = restore1Sum; header->image1Sum = sum1; header->image2Sum = sum2; @@ -2942,8 +2046,8 @@ hibernate_write_image(void) header->compression = (compressedSize << 8) / uncompressedSize; gIOHibernateCompression = header->compression; - - count = vars->fileExtents->getLength(); + + count = vars->fileVars->fileExtents->getLength(); if (count > sizeof(header->fileExtentMap)) { header->fileExtentMapSize = count; @@ -2955,25 +2059,20 @@ hibernate_write_image(void) header->deviceBase = vars->fileVars->block0; header->deviceBlockSize = vars->fileVars->blockSize; - + IOPolledFileSeek(vars->fileVars, 0); - err = IOPolledFileWrite(vars->fileVars, - (uint8_t *) header, sizeof(IOHibernateImageHeader), + err = IOHibernatePolledFileWrite(vars->fileVars, + (uint8_t *) header, sizeof(IOHibernateImageHeader), cryptvars); if (kIOReturnSuccess != err) break; - err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars); - if (kIOReturnSuccess != err) - break; - err = IOHibernatePollerIODone(vars->fileVars, true); - if (kIOReturnSuccess != err) - break; + err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars); } while (false); - + clock_get_uptime(&endTime); - IOService::getPMRootDomain()->pmStatsRecordEvent( + IOService::getPMRootDomain()->pmStatsRecordEvent( kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime); SUB_ABSOLUTETIME(&endTime, &allTime); @@ -2981,34 +2080,31 @@ hibernate_write_image(void) HIBLOG("all time: %qd ms, ", nsec / 1000000ULL); absolutetime_to_nanoseconds(compTime, &nsec); - HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ", - compBytes, + HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ", + compBytes, nsec / 1000000ULL, nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0); absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec); - HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ", - vars->fileVars->cryptBytes, - nsec / 1000000ULL, + HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ", + vars->fileVars->cryptBytes, + nsec / 1000000ULL, nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0); - HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n", + HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n", header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize, uncompressedSize, atop_32(uncompressedSize), compressedSize, uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0, sum1, sum2); - HIBLOG("zeroPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n", - zeroPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted); - - if (vars->fileVars->io) - (void) IOHibernatePollerIODone(vars->fileVars, false); + HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n", + svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted); if (pollerOpen) - IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState); + IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted ); if (vars->consoleMapping) - ProgressUpdate(gIOHibernateGraphicsInfo, + ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, 0, kIOHibernateProgressCount); HIBLOG("hibernate_write_image done(%x)\n", err); @@ -3016,8 +2112,8 @@ hibernate_write_image(void) // should we come back via regular wake, set the state in memory. gIOHibernateState = kIOHibernateStateInactive; - KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, - wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted, 0, 0); + KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, wiredPagesEncrypted, + wiredPagesClear, dirtyPagesEncrypted); if (kIOReturnSuccess == err) { @@ -3048,7 +2144,7 @@ hibernate_write_image(void) /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -extern "C" void +extern "C" void hibernate_machine_init(void) { IOReturn err; @@ -3062,12 +2158,12 @@ hibernate_machine_init(void) uint64_t compBytes; uint32_t lastProgressStamp = 0; uint32_t progressStamp; - hibernate_cryptvars_t * cryptvars = 0; + IOPolledFileCryptVars * cryptvars = 0; IOHibernateVars * vars = &gIOHibernateVars; bzero(gIOHibernateStats, sizeof(hibernate_statistics_t)); - if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents) + if (!vars->fileVars || !vars->fileVars->pollers) return; sum = gIOHibernateCurrentHeader->actualImage1Sum; @@ -3080,13 +2176,13 @@ hibernate_machine_init(void) } HIBPRINT("diag %x %x %x %x\n", - gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1], + gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1], gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]); #define t40ms(x) (tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000) #define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y); tStat(booterStart, booterStart); - gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart, + gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart; tStat(booterDuration0, booterTime0); tStat(booterDuration1, booterTime1); tStat(booterDuration2, booterTime2); @@ -3099,7 +2195,12 @@ hibernate_machine_init(void) gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize; gIOHibernateStats->image1Pages = pagesDone; - HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n", + /* HIBERNATE_stats */ + KDBG(IOKDBG_CODE(DBG_HIBERNATE, 14), gIOHibernateStats->smcStart, + gIOHibernateStats->booterStart, gIOHibernateStats->booterDuration, + gIOHibernateStats->trampolineDuration); + + HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n", gIOHibernateStats->booterStart, gIOHibernateStats->smcStart, gIOHibernateStats->booterDuration0, @@ -3114,7 +2215,7 @@ hibernate_machine_init(void) gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size, gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree); - if ((0 != (kIOHibernateModeSleep & gIOHibernateMode)) + if ((0 != (kIOHibernateModeSleep & gIOHibernateMode)) && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))) { hibernate_page_list_discard(vars->page_list); @@ -3126,8 +2227,9 @@ hibernate_machine_init(void) panic("handoff overflow"); IOHibernateHandoff * handoff; - bool done = false; - bool foundCryptData = false; + bool done = false; + bool foundCryptData = false; + bool foundVolumeEncryptData = false; for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy(); !done; @@ -3142,7 +2244,10 @@ hibernate_machine_init(void) break; case kIOHibernateHandoffTypeGraphicsInfo: - bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo)); + if (handoff->bytecount == sizeof(*gIOHibernateGraphicsInfo)) + { + bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo)); + } break; case kIOHibernateHandoffTypeCryptVars: @@ -3156,18 +2261,27 @@ hibernate_machine_init(void) bzero(data, handoff->bytecount); break; + case kIOHibernateHandoffTypeVolumeCryptKey: + if (handoff->bytecount == vars->volumeCryptKeySize) + { + bcopy(data, &vars->volumeCryptKey[0], vars->volumeCryptKeySize); + foundVolumeEncryptData = true; + } + else panic("kIOHibernateHandoffTypeVolumeCryptKey(%d)", handoff->bytecount); + break; + case kIOHibernateHandoffTypeMemoryMap: clock_get_uptime(&allTime); - hibernate_newruntime_map(data, handoff->bytecount, + hibernate_newruntime_map(data, handoff->bytecount, gIOHibernateCurrentHeader->systemTableOffset); clock_get_uptime(&endTime); - + SUB_ABSOLUTETIME(&endTime, &allTime); absolutetime_to_nanoseconds(endTime, &nsec); - + HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL); break; @@ -3182,30 +2296,33 @@ hibernate_machine_init(void) default: done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000)); break; - } + } } - if (cryptvars && !foundCryptData) - panic("hibernate handoff"); - HIBPRINT("video %x %d %d %d status %x\n", - gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth, - gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus); + if (vars->hwEncrypt && !foundVolumeEncryptData) + panic("no volumeCryptKey"); + else if (cryptvars && !foundCryptData) + panic("hibernate handoff"); + + HIBPRINT("video 0x%llx %d %d %d status %x\n", + gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth, + gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus); if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress) { - vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height + vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height * gIOHibernateGraphicsInfo->rowBytes); if (vars->videoMapSize > vars->videoAllocSize) vars->videoMapSize = 0; else { - IOMapPages(kernel_map, + IOMapPages(kernel_map, vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress, vars->videoMapSize, kIOMapInhibitCache ); } } if (vars->videoMapSize) - ProgressUpdate(gIOHibernateGraphicsInfo, + ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount); uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy(); @@ -3217,18 +2334,26 @@ hibernate_machine_init(void) AbsoluteTime_to_scalar(&compTime) = 0; compBytes = 0; - HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled()); - err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0); + HIBLOG("IOPolledFilePollersOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled()); + err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledAfterSleepState, false); clock_get_uptime(&startIOTime); endTime = startIOTime; SUB_ABSOLUTETIME(&endTime, &allTime); absolutetime_to_nanoseconds(endTime, &nsec); - HIBLOG("IOHibernatePollerOpen(%x) %qd ms\n", err, nsec / 1000000ULL); + HIBLOG("IOPolledFilePollersOpen(%x) %qd ms\n", err, nsec / 1000000ULL); + + if (vars->hwEncrypt) + { + err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, + &vars->volumeCryptKey[0], vars->volumeCryptKeySize); + HIBLOG("IOPolledFilePollersSetEncryptionKey(%x) %ld\n", err, vars->volumeCryptKeySize); + if (kIOReturnSuccess != err) panic("IOPolledFilePollersSetEncryptionKey(0x%x)", err); + cryptvars = 0; + } IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size); // kick off the read ahead - vars->fileVars->io = false; vars->fileVars->bufferHalf = 0; vars->fileVars->bufferLimit = 0; vars->fileVars->lastRead = 0; @@ -3278,30 +2403,39 @@ hibernate_machine_init(void) break; } - if (!compressedSize) bzero_phys(ptoa_64(ppnum), page_size); - else + err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars); + if (kIOReturnSuccess != err) break; + + if (compressedSize < page_size) { - err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars); - if (kIOReturnSuccess != err) break; - if (compressedSize < page_size) - { - decoOffset = page_size; - clock_get_uptime(&startTime); - WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, page_size); - clock_get_uptime(&endTime); - ADD_ABSOLUTETIME(&compTime, &endTime); - SUB_ABSOLUTETIME(&compTime, &startTime); - compBytes += page_size; + decoOffset = page_size; + clock_get_uptime(&startTime); + + if (compressedSize == 4) { + int i; + uint32_t *s, *d; + + s = (uint32_t *)src; + d = (uint32_t *)(uintptr_t)compressed; + + for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++) + *d++ = *s; } - else decoOffset = 0; + else + WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, compressedSize); + clock_get_uptime(&endTime); + ADD_ABSOLUTETIME(&compTime, &endTime); + SUB_ABSOLUTETIME(&compTime, &startTime); + compBytes += page_size; + } + else decoOffset = 0; - sum += hibernate_sum_page((src + decoOffset), ppnum); - err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size); - if (err) - { + sum += hibernate_sum_page((src + decoOffset), ppnum); + err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size); + if (err) + { HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err); break; - } } ppnum++; @@ -3317,7 +2451,7 @@ hibernate_machine_init(void) if (progressStamp != lastProgressStamp) { lastProgressStamp = progressStamp; - HIBPRINT("pages %d (%d%%)\n", pagesDone, + HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount); } } @@ -3332,18 +2466,15 @@ hibernate_machine_init(void) gIOHibernateCurrentHeader->actualImage2Sum = sum; gIOHibernateCompression = gIOHibernateCurrentHeader->compression; - if (vars->fileVars->io) - (void) IOHibernatePollerIODone(vars->fileVars, false); - clock_get_uptime(&endIOTime); - err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState); + err = IOPolledFilePollersClose(vars->fileVars, kIOPolledAfterSleepState); clock_get_uptime(&endTime); - IOService::getPMRootDomain()->pmStatsRecordEvent( + IOService::getPMRootDomain()->pmStatsRecordEvent( kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime); - IOService::getPMRootDomain()->pmStatsRecordEvent( + IOService::getPMRootDomain()->pmStatsRecordEvent( kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime); SUB_ABSOLUTETIME(&endTime, &allTime); @@ -3355,23 +2486,23 @@ hibernate_machine_init(void) gIOHibernateStats->kernelImageReadDuration = nsec / 1000000ULL; gIOHibernateStats->imagePages = pagesDone; - HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ", + HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ", pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize, nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0); absolutetime_to_nanoseconds(compTime, &nsec); - HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ", - compBytes, + HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ", + compBytes, nsec / 1000000ULL, nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0); absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec); - HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n", - vars->fileVars->cryptBytes, - nsec / 1000000ULL, + HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n", + vars->fileVars->cryptBytes, + nsec / 1000000ULL, nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0); - KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_NONE, pagesRead, pagesDone, 0, 0, 0); + KDBG(IOKDBG_CODE(DBG_HIBERNATE, 2), pagesRead, pagesDone); } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -3435,6 +2566,3 @@ void IOHibernateSystemRestart(void) if (noteProp) noteProp->release(); if (sym) sym->release(); } - - -