+ HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
+ bitmap_size, header->previewSize,
+ pageCount, vars->fileVars->position);
+
+ enum
+ // pageType
+ {
+ kWired = 0x02,
+ kEncrypt = 0x01,
+ kWiredEncrypt = kWired | kEncrypt,
+ kWiredClear = kWired,
+ kUnwiredEncrypt = kEncrypt
+ };
+
+ bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features()));
+
+ for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--) {
+ if (kUnwiredEncrypt == pageType) {
+ // start unwired image
+ if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) {
+ vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
+ vars->fileVars->encryptEnd = UINT64_MAX;
+ HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
+ }
+ bcopy(&cryptvars->aes_iv[0],
+ &gIOHibernateCryptWakeContext.aes_iv[0],
+ sizeof(cryptvars->aes_iv));
+ cryptvars = &gIOHibernateCryptWakeContext;
+ }
+ for (iterDone = false, ppnum = 0; !iterDone;) {
+ if (cpuAES && (pageType == kWiredClear)) {
+ count = 0;
+ } else {
+ count = hibernate_page_list_iterate((kWired & pageType) ? vars->page_list_wired : vars->page_list,
+ &ppnum);
+ }
+// kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
+ iterDone = !count;
+
+ if (!cpuAES) {
+ if (count && (kWired & pageType) && needEncrypt) {
+ uint32_t checkIndex;
+ for (checkIndex = 0;
+ (checkIndex < count)
+ && (((kEncrypt & pageType) == 0) == pmap_is_noencrypt(ppnum + checkIndex));
+ checkIndex++) {
+ }
+ if (!checkIndex) {
+ ppnum++;
+ continue;
+ }
+ count = checkIndex;
+ }
+ }
+
+ switch (pageType) {
+ case kWiredEncrypt: wiredPagesEncrypted += count; break;
+ case kWiredClear: wiredPagesClear += count; break;
+ case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
+ }
+
+ if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */
+ } else {
+ pageAndCount[0] = ppnum;
+ pageAndCount[1] = count;
+ err = IOHibernatePolledFileWrite(vars->fileVars,
+ (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
+ cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ }
+
+ for (page = ppnum; page < (ppnum + count); page++) {
+ err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
+ if (err) {
+ HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
+ break;
+ }
+
+ sum = hibernate_sum_page(src, page);
+ if (kWired & pageType) {
+ sum1 += sum;
+ } else {
+ sum2 += sum;
+ }
+
+ clock_get_uptime(&startTime);
+ wkresult = WKdm_compress_new((const WK_word*) src,
+ (WK_word*) compressed,
+ (WK_word*) scratch,
+ page_size - 4);
+
+ clock_get_uptime(&endTime);
+ ADD_ABSOLUTETIME(&compTime, &endTime);
+ SUB_ABSOLUTETIME(&compTime, &startTime);
+
+ compBytes += page_size;
+ pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
+
+ if (pageCompressedSize == 0) {
+ pageCompressedSize = 4;
+ data = src;
+
+ if (*(uint32_t *)src) {
+ svPageCount++;
+ } else {
+ zvPageCount++;
+ }
+ } else {
+ if (pageCompressedSize != page_size) {
+ data = compressed;
+ } else {
+ data = src;
+ }
+ }
+
+ tag = pageCompressedSize | kIOHibernateTagSignature;
+ err = IOHibernatePolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+
+ err = IOHibernatePolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+
+ compressedSize += pageCompressedSize;
+ uncompressedSize += page_size;
+ pagesDone++;
+
+ if (vars->consoleMapping && (0 == (1023 & pagesDone))) {
+ blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
+ if (blob != lastBlob) {
+ ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
+ lastBlob = blob;
+ }
+ }
+ if (0 == (8191 & pagesDone)) {
+ clock_get_uptime(&endTime);
+ SUB_ABSOLUTETIME(&endTime, &allTime);
+ absolutetime_to_nanoseconds(endTime, &nsec);
+ progressStamp = nsec / 750000000ULL;
+ if (progressStamp != lastProgressStamp) {
+ lastProgressStamp = progressStamp;
+ HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
+ }
+ }
+ }
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ ppnum = page;
+ }
+
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+
+ if ((kEncrypt & pageType) && vars->fileVars->encryptStart) {
+ vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
+ HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
+ }
+
+ if (kWiredEncrypt != pageType) {
+ // end of image1/2 - fill to next block
+ err = IOHibernatePolledFileWrite(vars->fileVars, NULL, 0, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ }
+ if (kWiredClear == pageType) {
+ // enlarge wired image for test
+// err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
+
+ // end wired image
+ header->encryptStart = vars->fileVars->encryptStart;
+ header->encryptEnd = vars->fileVars->encryptEnd;
+ image1Size = vars->fileVars->position;
+ HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
+ image1Size, header->encryptStart, header->encryptEnd);
+ }
+ }
+ if (kIOReturnSuccess != err) {
+ if (kIOReturnOverrun == err) {
+ // update actual compression ratio on not enough space (for retry)
+ gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
+ }
+
+ // update partial amount written (for IOPolledFileClose cleanup/unmap)
+ header->imageSize = vars->fileVars->position;
+ break;
+ }
+
+ // Header:
+
+ header->imageSize = vars->fileVars->position;
+ header->image1Size = image1Size;
+ header->bitmapSize = bitmap_size;
+ header->pageCount = pageCount;
+
+ header->restore1Sum = restore1Sum;
+ header->image1Sum = sum1;
+ header->image2Sum = sum2;
+ header->sleepTime = gIOLastSleepTime.tv_sec;
+
+ header->compression = (compressedSize << 8) / uncompressedSize;
+ gIOHibernateCompression = header->compression;
+
+ count = vars->fileVars->fileExtents->getLength();
+ if (count > sizeof(header->fileExtentMap)) {
+ header->fileExtentMapSize = count;
+ count = sizeof(header->fileExtentMap);
+ } else {
+ header->fileExtentMapSize = sizeof(header->fileExtentMap);
+ }
+ bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
+
+ header->deviceBase = vars->fileVars->block0;
+ header->deviceBlockSize = vars->fileVars->blockSize;
+
+ IOPolledFileSeek(vars->fileVars, 0);
+ err = IOHibernatePolledFileWrite(vars->fileVars,
+ (uint8_t *) header, sizeof(IOHibernateImageHeader),
+ cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ err = IOHibernatePolledFileWrite(vars->fileVars, NULL, 0, cryptvars);
+ }while (false);
+
+ clock_get_uptime(&endTime);
+
+ IOService::getPMRootDomain()->pmStatsRecordEvent(
+ kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
+
+ SUB_ABSOLUTETIME(&endTime, &allTime);
+ absolutetime_to_nanoseconds(endTime, &nsec);
+ HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
+
+ absolutetime_to_nanoseconds(compTime, &nsec);
+ HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
+ compBytes,
+ nsec / 1000000ULL,
+ nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+ absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
+ HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
+ vars->fileVars->cryptBytes,
+ nsec / 1000000ULL,
+ nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+ HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
+ header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
+ uncompressedSize, atop_32(uncompressedSize), compressedSize,
+ uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
+ sum1, sum2);
+
+ HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
+ svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
+
+ if (pollerOpen) {
+ IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted );
+ }
+
+ if (vars->consoleMapping) {
+ ProgressUpdate(gIOHibernateGraphicsInfo,
+ vars->consoleMapping, 0, kIOHibernateProgressCount);
+ }
+
+ HIBLOG("hibernate_write_image done(%x)\n", err);
+
+ // should we come back via regular wake, set the state in memory.
+ gIOHibernateState = kIOHibernateStateInactive;
+
+ KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, wiredPagesEncrypted,
+ wiredPagesClear, dirtyPagesEncrypted);
+
+ if (kIOReturnSuccess == err) {
+ if (kIOHibernateModeSleep & gIOHibernateMode) {
+ return kIOHibernatePostWriteSleep;
+ } else if (kIOHibernateModeRestart & gIOHibernateMode) {
+ return kIOHibernatePostWriteRestart;
+ } else {
+ /* by default, power down */
+ return kIOHibernatePostWriteHalt;
+ }
+ } else if (kIOReturnAborted == err) {
+ return kIOHibernatePostWriteWake;
+ } else {
+ /* on error, sleep */
+ return kIOHibernatePostWriteSleep;
+ }
projects / apple / xnu.git / blobdiff