- iterDone = !count;
-
- if (count && (kWired & pageType) && needEncrypt)
- {
- uint32_t checkIndex;
- for (checkIndex = 0;
- (checkIndex < count)
- && (((kEncrypt & pageType) == 0) == no_encrypt_page(ppnum + checkIndex));
- checkIndex++)
- {}
- if (!checkIndex)
- {
- ppnum++;
- continue;
- }
- count = checkIndex;
- }
-
- switch (pageType)
- {
- case kWiredEncrypt: wired_pages_encrypted += count; break;
- case kWiredClear: wired_pages_clear += count; break;
- case kUnwiredEncrypt: dirty_pages_encrypted += count; break;
- }
-
- if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */}
- else
- {
- pageAndCount[0] = ppnum;
- pageAndCount[1] = count;
- err = IOPolledFileWrite(vars->fileVars,
- (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
- cryptvars);
- if (kIOReturnSuccess != err)
- break;
- }
-
- for (page = ppnum; page < (ppnum + count); page++)
- {
- err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
- if (err)
- {
- HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
- break;
- }
-
- sum = hibernate_sum_page(src, page);
- if (kWired & pageType)
- sum1 += sum;
- else
- sum2 += sum;
-
- clock_get_uptime(&startTime);
-
- pageCompressedSize = WKdm_compress ((WK_word*) src, (WK_word*) (src + page_size), PAGE_SIZE_IN_WORDS);
-
- clock_get_uptime(&endTime);
- ADD_ABSOLUTETIME(&compTime, &endTime);
- SUB_ABSOLUTETIME(&compTime, &startTime);
- compBytes += page_size;
-
- if (kIOHibernateModeEncrypt & gIOHibernateMode)
- pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
-
- if (pageCompressedSize > page_size)
- {
-// HIBLOG("------------lose: %d\n", pageCompressedSize);
- pageCompressedSize = page_size;
- }
-
- if (pageCompressedSize != page_size)
- data = (src + page_size);
- else
- data = src;
-
- tag = pageCompressedSize | kIOHibernateTagSignature;
- err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
- if (kIOReturnSuccess != err)
- break;
-
- err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
- if (kIOReturnSuccess != err)
- break;
-
- compressedSize += pageCompressedSize;
- if (pageCompressedSize)
- uncompressedSize += page_size;
- pagesDone++;
-
- if (vars->consoleMapping && (0 == (1023 & pagesDone)))
- {
- blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
- if (blob != lastBlob)
- {
- ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
- lastBlob = blob;
- }
- }
- if (0 == (8191 & pagesDone))
- {
- clock_get_uptime(&endTime);
- SUB_ABSOLUTETIME(&endTime, &allTime);
- absolutetime_to_nanoseconds(endTime, &nsec);
- progressStamp = nsec / 750000000ULL;
- if (progressStamp != lastProgressStamp)
- {
- lastProgressStamp = progressStamp;
- HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
- }
- }
- }
- if (kIOReturnSuccess != err)
- break;
- ppnum = page;
- }
-
- if (kIOReturnSuccess != err)
- break;
-
- if ((kEncrypt & pageType))
- {
- vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
- HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
- }
-
- if (kWiredEncrypt != pageType)
- {
- // end of image1/2 - fill to next block
- err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
- if (kIOReturnSuccess != err)
- break;
- }
- if (kWiredClear == pageType)
- {
- // enlarge wired image for test
-// err = IOPolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
-
- // end wired image
- header->encryptStart = vars->fileVars->encryptStart;
- header->encryptEnd = vars->fileVars->encryptEnd;
- image1Size = vars->fileVars->position;
- HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
- image1Size, header->encryptStart, header->encryptEnd);
- }
- }
- if (kIOReturnSuccess != err)
- break;
-
- // Header:
-
- header->imageSize = vars->fileVars->position;
- header->image1Size = image1Size;
- header->bitmapSize = bitmap_size;
- header->pageCount = pageCount;
-
- header->restore1Sum = restore1Sum;
- header->image1Sum = sum1;
- header->image2Sum = sum2;
- header->sleepTime = gIOLastSleepTime.tv_sec;
-
- count = vars->fileExtents->getLength();
- if (count > sizeof(header->fileExtentMap))
- {
- header->fileExtentMapSize = count;
- count = sizeof(header->fileExtentMap);
- }
- else
- header->fileExtentMapSize = sizeof(header->fileExtentMap);
- bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
-
- header->deviceBase = vars->fileVars->block0;
-
- IOPolledFileSeek(vars->fileVars, 0);
- err = IOPolledFileWrite(vars->fileVars,
- (uint8_t *) header, sizeof(IOHibernateImageHeader),
- cryptvars);
- if (kIOReturnSuccess != err)
- break;
- err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
- if (kIOReturnSuccess != err)
- break;
- err = IOHibernatePollerIODone(vars->fileVars, true);
- if (kIOReturnSuccess != err)
- break;
- }
- while (false);
-
- clock_get_uptime(&endTime);
-
- IOService::getPMRootDomain()->pmStatsRecordEvent(
- kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
-
- SUB_ABSOLUTETIME(&endTime, &allTime);
- absolutetime_to_nanoseconds(endTime, &nsec);
- HIBLOG("all time: %qd ms, ",
- nsec / 1000000ULL);
-
- absolutetime_to_nanoseconds(compTime, &nsec);
- HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
- compBytes,
- nsec / 1000000ULL,
- nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
-
- absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
- HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
- vars->fileVars->cryptBytes,
- nsec / 1000000ULL,
- nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
-
- HIBLOG("\nimage %qd, uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
- header->imageSize,
- uncompressedSize, atop_32(uncompressedSize), compressedSize,
- uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
- sum1, sum2);
-
- HIBLOG("wired_pages_encrypted %d, wired_pages_clear %d, dirty_pages_encrypted %d\n",
- wired_pages_encrypted, wired_pages_clear, dirty_pages_encrypted);
-
- if (vars->fileVars->io)
- (void) IOHibernatePollerIODone(vars->fileVars, false);
-
- if (pollerOpen)
- IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
-
- if (vars->consoleMapping)
- ProgressUpdate(gIOHibernateGraphicsInfo,
- vars->consoleMapping, 0, kIOHibernateProgressCount);
-
- HIBLOG("hibernate_write_image done(%x)\n", err);
-
- // should we come back via regular wake, set the state in memory.
- gIOHibernateState = kIOHibernateStateInactive;
-
- KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END,
- wired_pages_encrypted, wired_pages_clear, dirty_pages_encrypted, 0, 0);
-
- if (kIOReturnSuccess == err)
- {
- if (kIOHibernateModeSleep & gIOHibernateMode)
- {
- return (kIOHibernatePostWriteSleep);
- }
- else if(kIOHibernateModeRestart & gIOHibernateMode)
- {
- return (kIOHibernatePostWriteRestart);
- }
- else
- {
- /* by default, power down */
- return (kIOHibernatePostWriteHalt);
- }
- }
- else if (kIOReturnAborted == err)
- {
- return (kIOHibernatePostWriteWake);
- }
- else
- {
- /* on error, sleep */
- return (kIOHibernatePostWriteSleep);
- }
+ iterDone = !count;
+
+ if (!cpuAES) {
+ if (count && (kWired & pageType) && needEncrypt) {
+ uint32_t checkIndex;
+ for (checkIndex = 0;
+ (checkIndex < count)
+ && (((kEncrypt & pageType) == 0) == pmap_is_noencrypt(ppnum + checkIndex));
+ checkIndex++) {
+ }
+ if (!checkIndex) {
+ ppnum++;
+ continue;
+ }
+ count = checkIndex;
+ }
+ }
+
+ switch (pageType) {
+ case kWiredEncrypt: wiredPagesEncrypted += count; break;
+ case kWiredClear: wiredPagesClear += count; break;
+ case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
+ }
+
+ if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */
+ } else {
+ pageAndCount[0] = ppnum;
+ pageAndCount[1] = count;
+ err = IOHibernatePolledFileWrite(vars->fileVars,
+ (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
+ cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ }
+
+ for (page = ppnum; page < (ppnum + count); page++) {
+ err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
+ if (err) {
+ HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
+ break;
+ }
+
+ sum = hibernate_sum_page(src, page);
+ if (kWired & pageType) {
+ sum1 += sum;
+ } else {
+ sum2 += sum;
+ }
+
+ clock_get_uptime(&startTime);
+ wkresult = WKdm_compress_new((const WK_word*) src,
+ (WK_word*) compressed,
+ (WK_word*) scratch,
+ page_size - 4);
+
+ clock_get_uptime(&endTime);
+ ADD_ABSOLUTETIME(&compTime, &endTime);
+ SUB_ABSOLUTETIME(&compTime, &startTime);
+
+ compBytes += page_size;
+ pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
+
+ if (pageCompressedSize == 0) {
+ pageCompressedSize = 4;
+ data = src;
+
+ if (*(uint32_t *)src) {
+ svPageCount++;
+ } else {
+ zvPageCount++;
+ }
+ } else {
+ if (pageCompressedSize != page_size) {
+ data = compressed;
+ } else {
+ data = src;
+ }
+ }
+
+ tag = pageCompressedSize | kIOHibernateTagSignature;
+ err = IOHibernatePolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+
+ err = IOHibernatePolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+
+ compressedSize += pageCompressedSize;
+ uncompressedSize += page_size;
+ pagesDone++;
+
+ if (vars->consoleMapping && (0 == (1023 & pagesDone))) {
+ blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
+ if (blob != lastBlob) {
+ ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
+ lastBlob = blob;
+ }
+ }
+ if (0 == (8191 & pagesDone)) {
+ clock_get_uptime(&endTime);
+ SUB_ABSOLUTETIME(&endTime, &allTime);
+ absolutetime_to_nanoseconds(endTime, &nsec);
+ progressStamp = nsec / 750000000ULL;
+ if (progressStamp != lastProgressStamp) {
+ lastProgressStamp = progressStamp;
+ HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
+ }
+ }
+ }
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ ppnum = page;
+ }
+
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+
+ if ((kEncrypt & pageType) && vars->fileVars->encryptStart) {
+ vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
+ HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
+ }
+
+ if (kWiredEncrypt != pageType) {
+ // end of image1/2 - fill to next block
+ err = IOHibernatePolledFileWrite(vars->fileVars, NULL, 0, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ }
+ if (kWiredClear == pageType) {
+ // enlarge wired image for test
+// err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
+
+ // end wired image
+ header->encryptStart = vars->fileVars->encryptStart;
+ header->encryptEnd = vars->fileVars->encryptEnd;
+ image1Size = vars->fileVars->position;
+ HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
+ image1Size, header->encryptStart, header->encryptEnd);
+ }
+ }
+ if (kIOReturnSuccess != err) {
+ if (kIOReturnOverrun == err) {
+ // update actual compression ratio on not enough space (for retry)
+ gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
+ }
+
+ // update partial amount written (for IOPolledFileClose cleanup/unmap)
+ header->imageSize = vars->fileVars->position;
+ break;
+ }
+
+ // Header:
+
+ header->imageSize = vars->fileVars->position;
+ header->image1Size = image1Size;
+ header->bitmapSize = bitmap_size;
+ header->pageCount = pageCount;
+
+ header->restore1Sum = restore1Sum;
+ header->image1Sum = sum1;
+ header->image2Sum = sum2;
+ header->sleepTime = gIOLastSleepTime.tv_sec;
+
+ header->compression = (compressedSize << 8) / uncompressedSize;
+ gIOHibernateCompression = header->compression;
+
+ count = vars->fileVars->fileExtents->getLength();
+ if (count > sizeof(header->fileExtentMap)) {
+ header->fileExtentMapSize = count;
+ count = sizeof(header->fileExtentMap);
+ } else {
+ header->fileExtentMapSize = sizeof(header->fileExtentMap);
+ }
+ bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
+
+ header->deviceBase = vars->fileVars->block0;
+ header->deviceBlockSize = vars->fileVars->blockSize;
+
+ IOPolledFileSeek(vars->fileVars, 0);
+ err = IOHibernatePolledFileWrite(vars->fileVars,
+ (uint8_t *) header, sizeof(IOHibernateImageHeader),
+ cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ err = IOHibernatePolledFileWrite(vars->fileVars, NULL, 0, cryptvars);
+ }while (false);
+
+ clock_get_uptime(&endTime);
+
+ IOService::getPMRootDomain()->pmStatsRecordEvent(
+ kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
+
+ SUB_ABSOLUTETIME(&endTime, &allTime);
+ absolutetime_to_nanoseconds(endTime, &nsec);
+ HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
+
+ absolutetime_to_nanoseconds(compTime, &nsec);
+ HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
+ compBytes,
+ nsec / 1000000ULL,
+ nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+ absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
+ HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
+ vars->fileVars->cryptBytes,
+ nsec / 1000000ULL,
+ nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+ HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
+ header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
+ uncompressedSize, atop_32(uncompressedSize), compressedSize,
+ uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
+ sum1, sum2);
+
+ HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
+ svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
+
+ if (pollerOpen) {
+ IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted );
+ }
+
+ if (vars->consoleMapping) {
+ ProgressUpdate(gIOHibernateGraphicsInfo,
+ vars->consoleMapping, 0, kIOHibernateProgressCount);
+ }
+
+ HIBLOG("hibernate_write_image done(%x)\n", err);
+
+ // should we come back via regular wake, set the state in memory.
+ gIOHibernateState = kIOHibernateStateInactive;
+
+ KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, wiredPagesEncrypted,
+ wiredPagesClear, dirtyPagesEncrypted);
+
+ if (kIOReturnSuccess == err) {
+ if (kIOHibernateModeSleep & gIOHibernateMode) {
+ return kIOHibernatePostWriteSleep;
+ } else if (kIOHibernateModeRestart & gIOHibernateMode) {
+ return kIOHibernatePostWriteRestart;
+ } else {
+ /* by default, power down */
+ return kIOHibernatePostWriteHalt;
+ }
+ } else if (kIOReturnAborted == err) {
+ return kIOHibernatePostWriteWake;
+ } else {
+ /* on error, sleep */
+ return kIOHibernatePostWriteSleep;
+ }
projects / apple / xnu.git / blobdiff