+ count = vars->fileVars->fileExtents->getLength();
+ if (count > sizeof(header->fileExtentMap)) {
+ count -= sizeof(header->fileExtentMap);
+ err = IOHibernatePolledFileWrite(vars->fileVars,
+ ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ }
+
+ hibernateBase = HIB_BASE; /* Defined in PAL headers */
+ hibernateEnd = (segHIBB + segSizeHIB);
+
+ // copy out restore1 code
+
+ for (count = 0;
+ (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+ count += segLen) {
+ for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++) {
+ gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
+ }
+ }
+
+ page = atop_32(kvtophys(hibernateBase));
+ count = atop_32(round_page(hibernateEnd) - hibernateBase);
+ header->restore1CodePhysPage = page;
+ header->restore1CodeVirt = hibernateBase;
+ header->restore1PageCount = count;
+ header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
+ header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
+
+ if (uuid_parse(&gIOHibernateBridgeBootSessionUUIDString[0], &header->bridgeBootSessionUUID[0])) {
+ bzero(&header->bridgeBootSessionUUID[0], sizeof(header->bridgeBootSessionUUID));
+ }
+
+ // sum __HIB seg, with zeros for the stack
+ src = (uint8_t *) trunc_page(hibernateBase);
+ for (page = 0; page < count; page++) {
+ if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0])) {
+ restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
+ } else {
+ restore1Sum += 0x00000000;
+ }
+ src += page_size;
+ }
+ sum1 = restore1Sum;
+
+ // write the __HIB seg, with zeros for the stack
+
+ src = (uint8_t *) trunc_page(hibernateBase);
+ count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
+ if (count) {
+ err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ }
+ err = IOHibernatePolledFileWrite(vars->fileVars,
+ (uint8_t *) NULL,
+ &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
+ cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ src = &gIOHibernateRestoreStackEnd[0];
+ count = round_page(hibernateEnd) - ((uintptr_t) src);
+ if (count) {
+ err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ }
+
+ if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) {
+ vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
+ vars->fileVars->encryptEnd = UINT64_MAX;
+ HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
+ }
+
+ // write the preview buffer
+
+ if (vars->previewBuffer) {
+ ppnum = 0;
+ count = 0;
+ do{
+ phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
+ pageAndCount[0] = atop_64(phys64);
+ pageAndCount[1] = atop_32(segLen);
+ err = IOHibernatePolledFileWrite(vars->fileVars,
+ (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
+ cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ count += segLen;
+ ppnum += sizeof(pageAndCount);
+ }while (phys64);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+
+ src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
+
+ ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
+
+ count = vars->previewBuffer->getLength();
+
+ header->previewPageListSize = ppnum;
+ header->previewSize = count + ppnum;
+
+ for (page = 0; page < count; page += page_size) {
+ phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
+ sum1 += hibernate_sum_page(src + page, atop_64(phys64));
+ }
+ err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ }
+
+ // mark areas for no save
+ IOMemoryDescriptor * ioBuffer;
+ ioBuffer = IOPolledFileGetIOBuffer(vars->fileVars);
+ for (count = 0;
+ (phys64 = ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+ count += segLen) {
+ hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+ atop_64(phys64), atop_32(segLen),
+ kIOHibernatePageStateFree);
+ pageCount -= atop_32(segLen);
+ }
+
+ for (count = 0;
+ (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+ count += segLen) {
+ hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+ atop_64(phys64), atop_32(segLen),
+ kIOHibernatePageStateFree);
+ pageCount -= atop_32(segLen);
+ }
+
+ // copy out bitmap of pages available for trashing during restore
+
+ bitmap_size = vars->page_list_wired->list_size;
+ src = (uint8_t *) vars->page_list_wired;
+ err = IOHibernatePolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+
+ // mark more areas for no save, but these are not available
+ // for trashing during restore
+
+ hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
+
+
+ page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
+ count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
+ hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+ page, count,
+ kIOHibernatePageStateFree);
+ pageCount -= count;
+
+ if (vars->previewBuffer) {
+ for (count = 0;
+ (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+ count += segLen) {
+ hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+ atop_64(phys64), atop_32(segLen),
+ kIOHibernatePageStateFree);
+ pageCount -= atop_32(segLen);
+ }
+ }
+
+ for (count = 0;
+ (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+ count += segLen) {
+ hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+ atop_64(phys64), atop_32(segLen),
+ kIOHibernatePageStateFree);
+ pageCount -= atop_32(segLen);
+ }
+
+#if KASAN
+ vm_size_t shadow_pages_free = atop_64(shadow_ptop) - atop_64(shadow_pnext);
+
+ /* no need to save unused shadow pages */
+ hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+ atop_64(shadow_pnext),
+ shadow_pages_free,
+ kIOHibernatePageStateFree);
+#endif
+
+ src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
+ compressed = src + page_size;
+ scratch = compressed + page_size;
+
+ pagesDone = 0;
+ lastBlob = 0;
+
+ HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
+ bitmap_size, header->previewSize,
+ pageCount, vars->fileVars->position);
+
+ enum
+ // pageType
+ {
+ kWired = 0x02,
+ kEncrypt = 0x01,
+ kWiredEncrypt = kWired | kEncrypt,
+ kWiredClear = kWired,
+ kUnwiredEncrypt = kEncrypt
+ };
+
+ bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features()));
+
+ for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--) {
+ if (kUnwiredEncrypt == pageType) {
+ // start unwired image
+ if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) {
+ vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
+ vars->fileVars->encryptEnd = UINT64_MAX;
+ HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
+ }
+ bcopy(&cryptvars->aes_iv[0],
+ &gIOHibernateCryptWakeContext.aes_iv[0],
+ sizeof(cryptvars->aes_iv));
+ cryptvars = &gIOHibernateCryptWakeContext;
+ }
+ for (iterDone = false, ppnum = 0; !iterDone;) {
+ if (cpuAES && (pageType == kWiredClear)) {
+ count = 0;
+ } else {
+ count = hibernate_page_list_iterate((kWired & pageType) ? vars->page_list_wired : vars->page_list,
+ &ppnum);
+ }
+// kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
+ iterDone = !count;
+
+ if (!cpuAES) {
+ if (count && (kWired & pageType) && needEncrypt) {
+ uint32_t checkIndex;
+ for (checkIndex = 0;
+ (checkIndex < count)
+ && (((kEncrypt & pageType) == 0) == pmap_is_noencrypt(ppnum + checkIndex));
+ checkIndex++) {
+ }
+ if (!checkIndex) {
+ ppnum++;
+ continue;
+ }
+ count = checkIndex;
+ }
+ }
+
+ switch (pageType) {
+ case kWiredEncrypt: wiredPagesEncrypted += count; break;
+ case kWiredClear: wiredPagesClear += count; break;
+ case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
+ }
+
+ if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */
+ } else {
+ pageAndCount[0] = ppnum;
+ pageAndCount[1] = count;
+ err = IOHibernatePolledFileWrite(vars->fileVars,
+ (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
+ cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ }
+
+ for (page = ppnum; page < (ppnum + count); page++) {
+ err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
+ if (err) {
+ HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
+ break;
+ }
+
+ sum = hibernate_sum_page(src, page);
+ if (kWired & pageType) {
+ sum1 += sum;
+ } else {
+ sum2 += sum;
+ }
+
+ clock_get_uptime(&startTime);
+ wkresult = WKdm_compress_new((const WK_word*) src,
+ (WK_word*) compressed,
+ (WK_word*) scratch,
+ page_size - 4);
+
+ clock_get_uptime(&endTime);
+ ADD_ABSOLUTETIME(&compTime, &endTime);
+ SUB_ABSOLUTETIME(&compTime, &startTime);
+
+ compBytes += page_size;
+ pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
+
+ if (pageCompressedSize == 0) {
+ pageCompressedSize = 4;
+ data = src;
+
+ if (*(uint32_t *)src) {
+ svPageCount++;
+ } else {
+ zvPageCount++;
+ }
+ } else {
+ if (pageCompressedSize != page_size) {
+ data = compressed;
+ } else {
+ data = src;
+ }
+ }
+
+ tag = pageCompressedSize | kIOHibernateTagSignature;
+ err = IOHibernatePolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+
+ err = IOHibernatePolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+
+ compressedSize += pageCompressedSize;
+ uncompressedSize += page_size;
+ pagesDone++;
+
+ if (vars->consoleMapping && (0 == (1023 & pagesDone))) {
+ blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
+ if (blob != lastBlob) {
+ ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
+ lastBlob = blob;
+ }
+ }
+ if (0 == (8191 & pagesDone)) {
+ clock_get_uptime(&endTime);
+ SUB_ABSOLUTETIME(&endTime, &allTime);
+ absolutetime_to_nanoseconds(endTime, &nsec);
+ progressStamp = nsec / 750000000ULL;
+ if (progressStamp != lastProgressStamp) {
+ lastProgressStamp = progressStamp;
+ HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
+ }
+ }
+ }
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ ppnum = page;
+ }
+
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+
+ if ((kEncrypt & pageType) && vars->fileVars->encryptStart) {
+ vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
+ HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
+ }
+
+ if (kWiredEncrypt != pageType) {
+ // end of image1/2 - fill to next block
+ err = IOHibernatePolledFileWrite(vars->fileVars, NULL, 0, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ }
+ if (kWiredClear == pageType) {
+ // enlarge wired image for test
+// err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
+
+ // end wired image
+ header->encryptStart = vars->fileVars->encryptStart;
+ header->encryptEnd = vars->fileVars->encryptEnd;
+ image1Size = vars->fileVars->position;
+ HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
+ image1Size, header->encryptStart, header->encryptEnd);
+ }
+ }
+ if (kIOReturnSuccess != err) {
+ if (kIOReturnOverrun == err) {
+ // update actual compression ratio on not enough space (for retry)
+ gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
+ }
+
+ // update partial amount written (for IOPolledFileClose cleanup/unmap)
+ header->imageSize = vars->fileVars->position;
+ break;
+ }
+
+ // Header:
+
+ header->imageSize = vars->fileVars->position;
+ header->image1Size = image1Size;
+ header->bitmapSize = bitmap_size;
+ header->pageCount = pageCount;
+
+ header->restore1Sum = restore1Sum;
+ header->image1Sum = sum1;
+ header->image2Sum = sum2;
+ header->sleepTime = gIOLastSleepTime.tv_sec;
+
+ header->compression = (compressedSize << 8) / uncompressedSize;
+ gIOHibernateCompression = header->compression;
+
+ count = vars->fileVars->fileExtents->getLength();
+ if (count > sizeof(header->fileExtentMap)) {
+ header->fileExtentMapSize = count;
+ count = sizeof(header->fileExtentMap);
+ } else {
+ header->fileExtentMapSize = sizeof(header->fileExtentMap);
+ }
+ bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
+
+ header->deviceBase = vars->fileVars->block0;
+ header->deviceBlockSize = vars->fileVars->blockSize;
+
+ IOPolledFileSeek(vars->fileVars, 0);
+ err = IOHibernatePolledFileWrite(vars->fileVars,
+ (uint8_t *) header, sizeof(IOHibernateImageHeader),
+ cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ err = IOHibernatePolledFileWrite(vars->fileVars, NULL, 0, cryptvars);
+ }while (false);
+
+ clock_get_uptime(&endTime);
+
+ IOService::getPMRootDomain()->pmStatsRecordEvent(
+ kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
+
+ SUB_ABSOLUTETIME(&endTime, &allTime);
+ absolutetime_to_nanoseconds(endTime, &nsec);
+ HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
+
+ absolutetime_to_nanoseconds(compTime, &nsec);
+ HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
+ compBytes,
+ nsec / 1000000ULL,
+ nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+ absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
+ HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
+ vars->fileVars->cryptBytes,
+ nsec / 1000000ULL,
+ nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+ HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
+ header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
+ uncompressedSize, atop_32(uncompressedSize), compressedSize,
+ uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
+ sum1, sum2);
+
+ HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
+ svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
+
+ if (pollerOpen) {
+ IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted );
+ }
+
+ if (vars->consoleMapping) {
+ ProgressUpdate(gIOHibernateGraphicsInfo,
+ vars->consoleMapping, 0, kIOHibernateProgressCount);
+ }
+
+ HIBLOG("hibernate_write_image done(%x)\n", err);
+
+ // should we come back via regular wake, set the state in memory.
+ gIOHibernateState = kIOHibernateStateInactive;
+
+ KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, wiredPagesEncrypted,
+ wiredPagesClear, dirtyPagesEncrypted);
+
+ if (kIOReturnSuccess == err) {
+ if (kIOHibernateModeSleep & gIOHibernateMode) {
+ return kIOHibernatePostWriteSleep;
+ } else if (kIOHibernateModeRestart & gIOHibernateMode) {
+ return kIOHibernatePostWriteRestart;
+ } else {
+ /* by default, power down */
+ return kIOHibernatePostWriteHalt;
+ }
+ } else if (kIOReturnAborted == err) {
+ return kIOHibernatePostWriteWake;
+ } else {
+ /* on error, sleep */
+ return kIOHibernatePostWriteSleep;
+ }
projects / apple / xnu.git / blobdiff