+ /*
+ * Hibernate variable is written to NVRAM on platforms in which RtcRam
+ * is not backed by coin cell. Remove Hibernate data from NVRAM.
+ */
+ if (gIOOptionsEntry) {
+ if (gIOHibernateRTCVariablesKey) {
+ if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
+ gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
+ }
+ }
+
+ if (gIOHibernateBootNextKey) {
+ if (gIOHibernateBootNextSave) {
+ gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
+ gIOHibernateBootNextSave->release();
+ gIOHibernateBootNextSave = NULL;
+ } else {
+ gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
+ }
+ }
+ if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) {
+ gIOOptionsEntry->sync();
+ }
+ }
+#endif
+
+ if (vars->srcBuffer) {
+ vars->srcBuffer->release();
+ }
+ bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
+ if (vars->handoffBuffer) {
+ if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
+ IOHibernateHandoff * handoff;
+ bool done = false;
+ for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
+ !done;
+ handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount]) {
+ HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
+ uint8_t * data = &handoff->data[0];
+ switch (handoff->type) {
+ case kIOHibernateHandoffTypeEnd:
+ done = true;
+ break;
+
+ case kIOHibernateHandoffTypeDeviceTree:
+ MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
+ break;
+
+ case kIOHibernateHandoffTypeKeyStore:
+#if defined(__i386__) || defined(__x86_64__)
+ {
+ IOBufferMemoryDescriptor *
+ md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
+ if (md) {
+ IOSetKeyStoreData(md);
+ }
+ }
+#endif
+ break;
+
+ default:
+ done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
+ break;
+ }
+ }
+#if defined(__i386__) || defined(__x86_64__)
+ if (vars->volumeCryptKeySize) {
+ IOBufferMemoryDescriptor *
+ bmd = IOBufferMemoryDescriptor::withBytes(&vars->volumeCryptKey[0],
+ vars->volumeCryptKeySize, kIODirectionOutIn);
+ if (!bmd) {
+ panic("IOBufferMemoryDescriptor");
+ }
+ IOSetAPFSKeyStoreData(bmd);
+ bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
+ }
+#endif
+ }
+ vars->handoffBuffer->release();
+ }
+
+ if (gIOChosenEntry
+ && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey)))
+ && (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength())) {
+ bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0],
+ sizeof(gIOHibernateBridgeBootSessionUUIDString));
+ }
+
+ if (vars->hwEncrypt) {
+ err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, NULL, 0);
+ HIBLOG("IOPolledFilePollersSetEncryptionKey(0,%x)\n", err);
+ }
+
+ bzero(vars, sizeof(*vars));
+
+// gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
+
+ return kIOReturnSuccess;
+}
+
+static void
+IOHibernateSystemPostWakeTrim(void * p1, void * p2)
+{
+ // invalidate & close the image file
+ if (p1) {
+ IOLockLock(gFSLock);
+ }
+ if (kFSTrimDelay == gFSState) {
+ IOPolledFileIOVars * vars = &gFileVars;
+ IOPolledFileClose(&vars,
+#if DISABLE_TRIM
+ 0, NULL, 0, 0, 0);
+#else
+ 0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader),
+ sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize);
+#endif
+ gFSState = kFSIdle;
+ }
+ if (p1) {
+ IOLockUnlock(gFSLock);
+ }
+}
+
+IOReturn
+IOHibernateSystemPostWake(bool now)
+{
+ gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
+ IOSetBootImageNVRAM(NULL);
+
+ IOLockLock(gFSLock);
+ if (kFSTrimDelay == gFSState) {
+ thread_call_cancel(gIOHibernateTrimCalloutEntry);
+ IOHibernateSystemPostWakeTrim(NULL, NULL);
+ } else if (kFSOpened != gFSState) {
+ gFSState = kFSIdle;
+ } else {
+ gFSState = kFSTrimDelay;
+ if (now) {
+ thread_call_cancel(gIOHibernateTrimCalloutEntry);
+ IOHibernateSystemPostWakeTrim(NULL, NULL);
+ } else {
+ AbsoluteTime deadline;
+ clock_interval_to_deadline(TRIM_DELAY, kMillisecondScale, &deadline );
+ thread_call_enter1_delayed(gIOHibernateTrimCalloutEntry, NULL, deadline);
+ }
+ }
+ IOLockUnlock(gFSLock);
+
+ return kIOReturnSuccess;
+}
+
+uint32_t
+IOHibernateWasScreenLocked(void)
+{
+ uint32_t ret = 0;
+ if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry) {
+ OSData *
+ data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
+ if (data) {
+ ret = ((uint32_t *)data->getBytesNoCopy())[0];
+ gIOChosenEntry->setProperty(kIOBooterScreenLockStateKey, data);
+ }
+ } else {
+ gIOChosenEntry->removeProperty(kIOBooterScreenLockStateKey);
+ }
+
+ return ret;
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
+ CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
+ gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
+SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
+ CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
+ gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
+ CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
+ &gIOHibernateMode, 0, "");
+SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
+ CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
+ &_hibernateStats, hibernate_statistics_t, "");
+SYSCTL_STRING(_kern_bridge, OID_AUTO, bootsessionuuid,
+ CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
+ gIOHibernateBridgeBootSessionUUIDString, sizeof(gIOHibernateBridgeBootSessionUUIDString), "");
+
+SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
+ CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+ &_hibernateStats.graphicsReadyTime, 0, "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
+ CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+ &_hibernateStats.wakeNotificationTime, 0, "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
+ CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+ &_hibernateStats.lockScreenReadyTime, 0, "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
+ CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+ &_hibernateStats.hidReadyTime, 0, "");
+
+void
+IOHibernateSystemInit(IOPMrootDomain * rootDomain)
+{
+ gIOHibernateBootImageKey = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
+ gIOHibernateBootSignatureKey = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
+ gIOBridgeBootSessionUUIDKey = OSSymbol::withCStringNoCopy(kIOBridgeBootSessionUUIDKey);
+
+#if defined(__i386__) || defined(__x86_64__)
+ gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
+ gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
+ gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
+ gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
+#endif /* defined(__i386__) || defined(__x86_64__) */
+
+ OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
+ if (data) {
+ rootDomain->setProperty(kIOHibernateStateKey, data);
+ data->release();
+ }
+
+ if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename))) {
+ gIOHibernateMode = kIOHibernateModeOn;
+ } else {
+ gIOHibernateFilename[0] = 0;
+ }
+
+ sysctl_register_oid(&sysctl__kern_hibernatefile);
+ sysctl_register_oid(&sysctl__kern_bootsignature);
+ sysctl_register_oid(&sysctl__kern_hibernatemode);
+ sysctl_register_oid(&sysctl__kern_hibernatestatistics);
+ sysctl_register_oid(&sysctl__kern_hibernategraphicsready);
+ sysctl_register_oid(&sysctl__kern_hibernatewakenotification);
+ sysctl_register_oid(&sysctl__kern_hibernatelockscreenready);
+ sysctl_register_oid(&sysctl__kern_hibernatehidready);
+
+ gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
+
+ if (gIOChosenEntry
+ && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey)))
+ && (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength())) {
+ sysctl_register_oid(&sysctl__kern_bridge_bootsessionuuid);
+ bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0], sizeof(gIOHibernateBridgeBootSessionUUIDString));
+ }
+
+ gFSLock = IOLockAlloc();
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+static IOReturn
+IOHibernatePolledFileWrite(IOPolledFileIOVars * vars,
+ const uint8_t * bytes, IOByteCount size,
+ IOPolledFileCryptVars * cryptvars)
+{
+ IOReturn err;
+
+ err = IOPolledFileWrite(vars, bytes, size, cryptvars);
+ if ((kIOReturnSuccess == err) && hibernate_should_abort()) {
+ err = kIOReturnAborted;
+ }
+
+ return err;
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+extern "C" uint32_t
+hibernate_write_image(void)
+{
+ IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
+ IOHibernateVars * vars = &gIOHibernateVars;
+ IOPolledFileExtent * fileExtents;
+
+ _static_assert_1_arg(sizeof(IOHibernateImageHeader) == 512);
+
+ uint32_t pageCount, pagesDone;
+ IOReturn err;
+ vm_offset_t ppnum, page;
+ IOItemCount count;
+ uint8_t * src;
+ uint8_t * data;
+ uint8_t * compressed;
+ uint8_t * scratch;
+ IOByteCount pageCompressedSize;
+ uint64_t compressedSize, uncompressedSize;
+ uint64_t image1Size = 0;
+ uint32_t bitmap_size;
+ bool iterDone, pollerOpen, needEncrypt;
+ uint32_t restore1Sum, sum, sum1, sum2;
+ int wkresult;
+ uint32_t tag;
+ uint32_t pageType;
+ uint32_t pageAndCount[2];
+ addr64_t phys64;
+ IOByteCount segLen;
+ uintptr_t hibernateBase;
+ uintptr_t hibernateEnd;
+
+ AbsoluteTime startTime, endTime;
+ AbsoluteTime allTime, compTime;
+ uint64_t compBytes;
+ uint64_t nsec;
+ uint32_t lastProgressStamp = 0;
+ uint32_t progressStamp;
+ uint32_t blob, lastBlob = (uint32_t) -1L;
+
+ uint32_t wiredPagesEncrypted;
+ uint32_t dirtyPagesEncrypted;
+ uint32_t wiredPagesClear;
+ uint32_t svPageCount;
+ uint32_t zvPageCount;
+
+ IOPolledFileCryptVars _cryptvars;
+ IOPolledFileCryptVars * cryptvars = NULL;
+
+ wiredPagesEncrypted = 0;
+ dirtyPagesEncrypted = 0;
+ wiredPagesClear = 0;
+ svPageCount = 0;
+ zvPageCount = 0;
+
+ if (!vars->fileVars
+ || !vars->fileVars->pollers
+ || !(kIOHibernateModeOn & gIOHibernateMode)) {
+ return kIOHibernatePostWriteSleep;
+ }
+
+ if (kIOHibernateModeSleep & gIOHibernateMode) {
+ kdebug_enable = save_kdebug_enable;
+ }
+
+ KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START);
+ IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
+
+ restore1Sum = sum1 = sum2 = 0;
+
+#if CRYPTO
+ // encryption data. "iv" is the "initial vector".
+ if (kIOHibernateModeEncrypt & gIOHibernateMode) {
+ static const unsigned char first_iv[AES_BLOCK_SIZE]
+ = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
+ 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
+
+ cryptvars = &gIOHibernateCryptWakeContext;
+ bzero(cryptvars, sizeof(IOPolledFileCryptVars));
+ aes_encrypt_key(vars->cryptKey,
+ kIOHibernateAESKeySize,
+ &cryptvars->ctx.encrypt);
+ aes_decrypt_key(vars->cryptKey,
+ kIOHibernateAESKeySize,
+ &cryptvars->ctx.decrypt);
+
+ cryptvars = &_cryptvars;
+ bzero(cryptvars, sizeof(IOPolledFileCryptVars));
+ for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++) {
+ vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
+ }
+ aes_encrypt_key(vars->wiredCryptKey,
+ kIOHibernateAESKeySize,
+ &cryptvars->ctx.encrypt);
+
+ bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
+ bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
+ bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
+ }
+#endif /* CRYPTO */
+
+ hibernate_page_list_setall(vars->page_list,
+ vars->page_list_wired,
+ vars->page_list_pal,
+ false /* !preflight */,
+ /* discard_all */
+ ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
+ && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
+ &pageCount);
+
+ HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
+
+ fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy();
+
+#if 0
+ count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
+ for (page = 0; page < count; page++) {
+ HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
+ fileExtents[page].start, fileExtents[page].length,
+ fileExtents[page].start + fileExtents[page].length);
+ }
+#endif
+
+ needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
+ AbsoluteTime_to_scalar(&compTime) = 0;
+ compBytes = 0;
+
+ clock_get_uptime(&allTime);
+ IOService::getPMRootDomain()->pmStatsRecordEvent(
+ kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
+ do{
+ compressedSize = 0;
+ uncompressedSize = 0;
+ svPageCount = 0;
+ zvPageCount = 0;
+
+ IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
+
+ HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
+ ml_get_interrupts_enabled());
+ err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState,
+ // abortable if not low battery
+ !IOService::getPMRootDomain()->mustHibernate());
+ HIBLOG("IOHibernatePollerOpen(%x)\n", err);
+ pollerOpen = (kIOReturnSuccess == err);
+ if (!pollerOpen) {
+ break;
+ }
+
+ if (vars->volumeCryptKeySize) {
+ err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
+ HIBLOG("IOPolledFilePollersSetEncryptionKey(%x)\n", err);
+ vars->hwEncrypt = (kIOReturnSuccess == err);
+ bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
+ if (vars->hwEncrypt) {
+ header->options |= kIOHibernateOptionHWEncrypt;
+ }
+ }
+
+ // copy file block extent list if larger than header
+
+ count = vars->fileVars->fileExtents->getLength();
+ if (count > sizeof(header->fileExtentMap)) {
+ count -= sizeof(header->fileExtentMap);
+ err = IOHibernatePolledFileWrite(vars->fileVars,
+ ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ }
+
+ hibernateBase = HIB_BASE; /* Defined in PAL headers */
+ hibernateEnd = (segHIBB + segSizeHIB);
+
+ // copy out restore1 code
+
+ for (count = 0;
+ (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+ count += segLen) {
+ for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++) {
+ gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
+ }
+ }
+
+ page = atop_32(kvtophys(hibernateBase));
+ count = atop_32(round_page(hibernateEnd) - hibernateBase);
+ header->restore1CodePhysPage = page;
+ header->restore1CodeVirt = hibernateBase;
+ header->restore1PageCount = count;
+ header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
+ header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
+
+ if (uuid_parse(&gIOHibernateBridgeBootSessionUUIDString[0], &header->bridgeBootSessionUUID[0])) {
+ bzero(&header->bridgeBootSessionUUID[0], sizeof(header->bridgeBootSessionUUID));
+ }
+
+ // sum __HIB seg, with zeros for the stack
+ src = (uint8_t *) trunc_page(hibernateBase);
+ for (page = 0; page < count; page++) {
+ if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0])) {
+ restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
+ } else {
+ restore1Sum += 0x00000000;
+ }
+ src += page_size;
+ }
+ sum1 = restore1Sum;
+
+ // write the __HIB seg, with zeros for the stack
+
+ src = (uint8_t *) trunc_page(hibernateBase);
+ count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
+ if (count) {
+ err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ }
+ err = IOHibernatePolledFileWrite(vars->fileVars,
+ (uint8_t *) NULL,
+ &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
+ cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ src = &gIOHibernateRestoreStackEnd[0];
+ count = round_page(hibernateEnd) - ((uintptr_t) src);
+ if (count) {
+ err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ }
+
+ if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) {
+ vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
+ vars->fileVars->encryptEnd = UINT64_MAX;
+ HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
+ }
+
+ // write the preview buffer
+
+ if (vars->previewBuffer) {
+ ppnum = 0;
+ count = 0;
+ do{
+ phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
+ pageAndCount[0] = atop_64(phys64);
+ pageAndCount[1] = atop_32(segLen);
+ err = IOHibernatePolledFileWrite(vars->fileVars,
+ (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
+ cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ count += segLen;
+ ppnum += sizeof(pageAndCount);
+ }while (phys64);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+
+ src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
+
+ ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
+
+ count = vars->previewBuffer->getLength();
+
+ header->previewPageListSize = ppnum;
+ header->previewSize = count + ppnum;
+
+ for (page = 0; page < count; page += page_size) {
+ phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
+ sum1 += hibernate_sum_page(src + page, atop_64(phys64));
+ }
+ err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ }
+
+ // mark areas for no save
+ IOMemoryDescriptor * ioBuffer;
+ ioBuffer = IOPolledFileGetIOBuffer(vars->fileVars);
+ for (count = 0;
+ (phys64 = ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+ count += segLen) {
+ hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+ atop_64(phys64), atop_32(segLen),
+ kIOHibernatePageStateFree);
+ pageCount -= atop_32(segLen);
+ }
+
+ for (count = 0;
+ (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+ count += segLen) {
+ hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+ atop_64(phys64), atop_32(segLen),
+ kIOHibernatePageStateFree);
+ pageCount -= atop_32(segLen);
+ }
+
+ // copy out bitmap of pages available for trashing during restore
+
+ bitmap_size = vars->page_list_wired->list_size;
+ src = (uint8_t *) vars->page_list_wired;
+ err = IOHibernatePolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+
+ // mark more areas for no save, but these are not available
+ // for trashing during restore
+
+ hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
+
+
+ page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
+ count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
+ hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+ page, count,
+ kIOHibernatePageStateFree);
+ pageCount -= count;
+
+ if (vars->previewBuffer) {
+ for (count = 0;
+ (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+ count += segLen) {
+ hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+ atop_64(phys64), atop_32(segLen),
+ kIOHibernatePageStateFree);
+ pageCount -= atop_32(segLen);
+ }
+ }
+
+ for (count = 0;
+ (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+ count += segLen) {
+ hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+ atop_64(phys64), atop_32(segLen),
+ kIOHibernatePageStateFree);
+ pageCount -= atop_32(segLen);
+ }
+
+#if KASAN
+ vm_size_t shadow_pages_free = atop_64(shadow_ptop) - atop_64(shadow_pnext);
+
+ /* no need to save unused shadow pages */
+ hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+ atop_64(shadow_pnext),
+ shadow_pages_free,
+ kIOHibernatePageStateFree);
+#endif
+
+ src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
+ compressed = src + page_size;
+ scratch = compressed + page_size;
+
+ pagesDone = 0;
+ lastBlob = 0;
+
+ HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
+ bitmap_size, header->previewSize,
+ pageCount, vars->fileVars->position);
+
+ enum
+ // pageType
+ {
+ kWired = 0x02,
+ kEncrypt = 0x01,
+ kWiredEncrypt = kWired | kEncrypt,
+ kWiredClear = kWired,
+ kUnwiredEncrypt = kEncrypt
+ };
+
+ bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features()));
+
+ for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--) {
+ if (kUnwiredEncrypt == pageType) {
+ // start unwired image
+ if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) {
+ vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
+ vars->fileVars->encryptEnd = UINT64_MAX;
+ HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
+ }
+ bcopy(&cryptvars->aes_iv[0],
+ &gIOHibernateCryptWakeContext.aes_iv[0],
+ sizeof(cryptvars->aes_iv));
+ cryptvars = &gIOHibernateCryptWakeContext;
+ }
+ for (iterDone = false, ppnum = 0; !iterDone;) {
+ if (cpuAES && (pageType == kWiredClear)) {
+ count = 0;
+ } else {
+ count = hibernate_page_list_iterate((kWired & pageType) ? vars->page_list_wired : vars->page_list,
+ &ppnum);
+ }
+// kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
+ iterDone = !count;
+
+ if (!cpuAES) {
+ if (count && (kWired & pageType) && needEncrypt) {
+ uint32_t checkIndex;
+ for (checkIndex = 0;
+ (checkIndex < count)
+ && (((kEncrypt & pageType) == 0) == pmap_is_noencrypt(ppnum + checkIndex));
+ checkIndex++) {
+ }
+ if (!checkIndex) {
+ ppnum++;
+ continue;
+ }
+ count = checkIndex;
+ }
+ }
+
+ switch (pageType) {
+ case kWiredEncrypt: wiredPagesEncrypted += count; break;
+ case kWiredClear: wiredPagesClear += count; break;
+ case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
+ }
+
+ if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */
+ } else {
+ pageAndCount[0] = ppnum;
+ pageAndCount[1] = count;
+ err = IOHibernatePolledFileWrite(vars->fileVars,
+ (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
+ cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ }
+
+ for (page = ppnum; page < (ppnum + count); page++) {
+ err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
+ if (err) {
+ HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
+ break;
+ }
+
+ sum = hibernate_sum_page(src, page);
+ if (kWired & pageType) {
+ sum1 += sum;
+ } else {
+ sum2 += sum;
+ }
+
+ clock_get_uptime(&startTime);
+ wkresult = WKdm_compress_new((const WK_word*) src,
+ (WK_word*) compressed,
+ (WK_word*) scratch,
+ page_size - 4);
+
+ clock_get_uptime(&endTime);
+ ADD_ABSOLUTETIME(&compTime, &endTime);
+ SUB_ABSOLUTETIME(&compTime, &startTime);
+
+ compBytes += page_size;
+ pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
+
+ if (pageCompressedSize == 0) {
+ pageCompressedSize = 4;
+ data = src;
+
+ if (*(uint32_t *)src) {
+ svPageCount++;
+ } else {
+ zvPageCount++;
+ }
+ } else {
+ if (pageCompressedSize != page_size) {
+ data = compressed;
+ } else {
+ data = src;
+ }
+ }
+
+ tag = pageCompressedSize | kIOHibernateTagSignature;
+ err = IOHibernatePolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+
+ err = IOHibernatePolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+
+ compressedSize += pageCompressedSize;
+ uncompressedSize += page_size;
+ pagesDone++;
+
+ if (vars->consoleMapping && (0 == (1023 & pagesDone))) {
+ blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
+ if (blob != lastBlob) {
+ ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
+ lastBlob = blob;
+ }
+ }
+ if (0 == (8191 & pagesDone)) {
+ clock_get_uptime(&endTime);
+ SUB_ABSOLUTETIME(&endTime, &allTime);
+ absolutetime_to_nanoseconds(endTime, &nsec);
+ progressStamp = nsec / 750000000ULL;
+ if (progressStamp != lastProgressStamp) {
+ lastProgressStamp = progressStamp;
+ HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
+ }
+ }
+ }
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ ppnum = page;
+ }
+
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+
+ if ((kEncrypt & pageType) && vars->fileVars->encryptStart) {
+ vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
+ HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
+ }
+
+ if (kWiredEncrypt != pageType) {
+ // end of image1/2 - fill to next block
+ err = IOHibernatePolledFileWrite(vars->fileVars, NULL, 0, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ }
+ if (kWiredClear == pageType) {
+ // enlarge wired image for test
+// err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
+
+ // end wired image
+ header->encryptStart = vars->fileVars->encryptStart;
+ header->encryptEnd = vars->fileVars->encryptEnd;
+ image1Size = vars->fileVars->position;
+ HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
+ image1Size, header->encryptStart, header->encryptEnd);
+ }
+ }
+ if (kIOReturnSuccess != err) {
+ if (kIOReturnOverrun == err) {
+ // update actual compression ratio on not enough space (for retry)
+ gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
+ }
+
+ // update partial amount written (for IOPolledFileClose cleanup/unmap)
+ header->imageSize = vars->fileVars->position;
+ break;
+ }
+
+ // Header:
+
+ header->imageSize = vars->fileVars->position;
+ header->image1Size = image1Size;
+ header->bitmapSize = bitmap_size;
+ header->pageCount = pageCount;
+
+ header->restore1Sum = restore1Sum;
+ header->image1Sum = sum1;
+ header->image2Sum = sum2;
+ header->sleepTime = gIOLastSleepTime.tv_sec;
+
+ header->compression = (compressedSize << 8) / uncompressedSize;
+ gIOHibernateCompression = header->compression;
+
+ count = vars->fileVars->fileExtents->getLength();
+ if (count > sizeof(header->fileExtentMap)) {
+ header->fileExtentMapSize = count;
+ count = sizeof(header->fileExtentMap);
+ } else {
+ header->fileExtentMapSize = sizeof(header->fileExtentMap);
+ }
+ bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
+
+ header->deviceBase = vars->fileVars->block0;
+ header->deviceBlockSize = vars->fileVars->blockSize;
+
+ IOPolledFileSeek(vars->fileVars, 0);
+ err = IOHibernatePolledFileWrite(vars->fileVars,
+ (uint8_t *) header, sizeof(IOHibernateImageHeader),
+ cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ err = IOHibernatePolledFileWrite(vars->fileVars, NULL, 0, cryptvars);
+ }while (false);
+
+ clock_get_uptime(&endTime);
+
+ IOService::getPMRootDomain()->pmStatsRecordEvent(
+ kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
+
+ SUB_ABSOLUTETIME(&endTime, &allTime);
+ absolutetime_to_nanoseconds(endTime, &nsec);
+ HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
+
+ absolutetime_to_nanoseconds(compTime, &nsec);
+ HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
+ compBytes,
+ nsec / 1000000ULL,
+ nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+ absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
+ HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
+ vars->fileVars->cryptBytes,
+ nsec / 1000000ULL,
+ nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+ HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
+ header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
+ uncompressedSize, atop_32(uncompressedSize), compressedSize,
+ uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
+ sum1, sum2);
+
+ HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
+ svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
+
+ if (pollerOpen) {
+ IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted );
+ }
+
+ if (vars->consoleMapping) {
+ ProgressUpdate(gIOHibernateGraphicsInfo,
+ vars->consoleMapping, 0, kIOHibernateProgressCount);
+ }
+
+ HIBLOG("hibernate_write_image done(%x)\n", err);
+
+ // should we come back via regular wake, set the state in memory.
+ gIOHibernateState = kIOHibernateStateInactive;
+
+ KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, wiredPagesEncrypted,
+ wiredPagesClear, dirtyPagesEncrypted);
+
+ if (kIOReturnSuccess == err) {
+ if (kIOHibernateModeSleep & gIOHibernateMode) {
+ return kIOHibernatePostWriteSleep;
+ } else if (kIOHibernateModeRestart & gIOHibernateMode) {
+ return kIOHibernatePostWriteRestart;
+ } else {
+ /* by default, power down */
+ return kIOHibernatePostWriteHalt;
+ }
+ } else if (kIOReturnAborted == err) {
+ return kIOHibernatePostWriteWake;
+ } else {
+ /* on error, sleep */
+ return kIOHibernatePostWriteSleep;
+ }
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+extern "C" void
+hibernate_machine_init(void)
projects / apple / xnu.git / blobdiff