- needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
- AbsoluteTime_to_scalar(&compTime) = 0;
- compBytes = 0;
-
- clock_get_uptime(&allTime);
- IOService::getPMRootDomain()->pmStatsRecordEvent(
- kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
-
- do
- {
- compressedSize = 0;
- uncompressedSize = 0;
-
- IOPolledFileSeek(vars->fileVars, sizeof(IOHibernateImageHeader));
-
- HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
- ml_get_interrupts_enabled());
- err = IOHibernatePollerOpen(vars->fileVars, kIOPolledBeforeSleepState, vars->ioBuffer);
- HIBLOG("IOHibernatePollerOpen(%x)\n", err);
- pollerOpen = (kIOReturnSuccess == err);
- if (!pollerOpen)
- break;
-
- // copy file block extent list if larger than header
-
- count = vars->fileExtents->getLength();
- if (count > sizeof(header->fileExtentMap))
- {
- count -= sizeof(header->fileExtentMap);
- err = IOPolledFileWrite(vars->fileVars,
- ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
- if (kIOReturnSuccess != err)
- break;
- }
-
- uintptr_t hibernateBase;
- uintptr_t hibernateEnd;
-
- hibernateBase = HIB_BASE; /* Defined in PAL headers */
-
- hibernateEnd = (sectHIBB + sectSizeHIB);
-
- // copy out restore1 code
-
- for (count = 0;
- (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
- count += segLen)
- {
- for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++)
- {
- gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
- }
- }
-
- page = atop_32(kvtophys(hibernateBase));
- count = atop_32(round_page(hibernateEnd) - hibernateBase);
- header->restore1CodePhysPage = page;
- header->restore1CodeVirt = hibernateBase;
- header->restore1PageCount = count;
- header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
- header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
-
- // sum __HIB sect, with zeros for the stack
- src = (uint8_t *) trunc_page(hibernateBase);
- for (page = 0; page < count; page++)
- {
- if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
- restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
- else
- restore1Sum += 0x00000000;
- src += page_size;
- }
- sum1 = restore1Sum;
-
- // write the __HIB sect, with zeros for the stack
-
- src = (uint8_t *) trunc_page(hibernateBase);
- count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
- if (count)
- {
- err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
- if (kIOReturnSuccess != err)
- break;
- }
- err = IOPolledFileWrite(vars->fileVars,
- (uint8_t *) 0,
- &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
- cryptvars);
- if (kIOReturnSuccess != err)
- break;
- src = &gIOHibernateRestoreStackEnd[0];
- count = round_page(hibernateEnd) - ((uintptr_t) src);
- if (count)
- {
- err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
- if (kIOReturnSuccess != err)
- break;
- }
-
- // write the preview buffer
-
- if (vars->previewBuffer)
- {
- ppnum = 0;
- count = 0;
- do
- {
- phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
- pageAndCount[0] = atop_64(phys64);
- pageAndCount[1] = atop_32(segLen);
- err = IOPolledFileWrite(vars->fileVars,
- (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
- cryptvars);
- if (kIOReturnSuccess != err)
- break;
- count += segLen;
- ppnum += sizeof(pageAndCount);
- }
- while (phys64);
- if (kIOReturnSuccess != err)
- break;
-
- src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
- count = vars->previewBuffer->getLength();
-
- header->previewPageListSize = ppnum;
- header->previewSize = count + ppnum;
-
- for (page = 0; page < count; page += page_size)
- {
- phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
- sum1 += hibernate_sum_page(src + page, atop_64(phys64));
- }
- err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
- if (kIOReturnSuccess != err)
- break;
- }
-
- // mark areas for no save
-
- for (count = 0;
- (phys64 = vars->ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
- count += segLen)
- {
- hibernate_set_page_state(vars->page_list, vars->page_list_wired,
- atop_64(phys64), atop_32(segLen),
- kIOHibernatePageStateFree);
- pageCount -= atop_32(segLen);
- }
-
- for (count = 0;
- (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
- count += segLen)
- {
- hibernate_set_page_state(vars->page_list, vars->page_list_wired,
- atop_64(phys64), atop_32(segLen),
- kIOHibernatePageStateFree);
- pageCount -= atop_32(segLen);
- }
-
- // copy out bitmap of pages available for trashing during restore
-
- bitmap_size = vars->page_list_wired->list_size;
- src = (uint8_t *) vars->page_list_wired;
- err = IOPolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
- if (kIOReturnSuccess != err)
- break;
-
- // mark more areas for no save, but these are not available
- // for trashing during restore
-
- hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
-
-
- page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
- count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
- hibernate_set_page_state(vars->page_list, vars->page_list_wired,
- page, count,
- kIOHibernatePageStateFree);
- pageCount -= count;
-
- if (vars->previewBuffer) for (count = 0;
- (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
- count += segLen)
- {
- hibernate_set_page_state(vars->page_list, vars->page_list_wired,
- atop_64(phys64), atop_32(segLen),
- kIOHibernatePageStateFree);
- pageCount -= atop_32(segLen);
- }
-
- for (count = 0;
- (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
- count += segLen)
- {
- hibernate_set_page_state(vars->page_list, vars->page_list_wired,
- atop_64(phys64), atop_32(segLen),
- kIOHibernatePageStateFree);
- pageCount -= atop_32(segLen);
- }
-
- (void)hibernate_pal_callback;
-
- src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
-
- pagesDone = 0;
- lastBlob = 0;
-
- HIBLOG("writing %d pages\n", pageCount);
-
- enum
- // pageType
- {
- kWired = 0x02,
- kEncrypt = 0x01,
- kWiredEncrypt = kWired | kEncrypt,
- kWiredClear = kWired,
- kUnwiredEncrypt = kEncrypt
- };
-
- for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)
- {
- if (needEncrypt && (kEncrypt & pageType))
- {
- vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
- vars->fileVars->encryptEnd = UINT64_MAX;
- HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
-
- if (kUnwiredEncrypt == pageType)
- {
- // start unwired image
- bcopy(&cryptvars->aes_iv[0],
- &gIOHibernateCryptWakeContext.aes_iv[0],
- sizeof(cryptvars->aes_iv));
- cryptvars = &gIOHibernateCryptWakeContext;
- }
- }
- for (iterDone = false, ppnum = 0; !iterDone; )
- {
- count = hibernate_page_list_iterate((kWired & pageType)
- ? vars->page_list_wired : vars->page_list,
- &ppnum);
+ needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
+ AbsoluteTime_to_scalar(&compTime) = 0;
+ compBytes = 0;
+
+ clock_get_uptime(&allTime);
+ IOService::getPMRootDomain()->pmStatsRecordEvent(
+ kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
+ do{
+ compressedSize = 0;
+ uncompressedSize = 0;
+ svPageCount = 0;
+ zvPageCount = 0;
+
+ IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
+
+ HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
+ ml_get_interrupts_enabled());
+ err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState,
+ // abortable if not low battery
+ !IOService::getPMRootDomain()->mustHibernate());
+ HIBLOG("IOHibernatePollerOpen(%x)\n", err);
+ pollerOpen = (kIOReturnSuccess == err);
+ if (!pollerOpen) {
+ break;
+ }
+
+ if (vars->volumeCryptKeySize) {
+ err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
+ HIBLOG("IOPolledFilePollersSetEncryptionKey(%x)\n", err);
+ vars->hwEncrypt = (kIOReturnSuccess == err);
+ bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
+ if (vars->hwEncrypt) {
+ header->options |= kIOHibernateOptionHWEncrypt;
+ }
+ }
+
+ // copy file block extent list if larger than header
+
+ count = vars->fileVars->fileExtents->getLength();
+ if (count > sizeof(header->fileExtentMap)) {
+ count -= sizeof(header->fileExtentMap);
+ err = IOHibernatePolledFileWrite(vars->fileVars,
+ ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ }
+
+ hibernateBase = HIB_BASE; /* Defined in PAL headers */
+ hibernateEnd = (segHIBB + segSizeHIB);
+
+ // copy out restore1 code
+
+ for (count = 0;
+ (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+ count += segLen) {
+ for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++) {
+ gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
+ }
+ }
+
+ page = atop_32(kvtophys(hibernateBase));
+ count = atop_32(round_page(hibernateEnd) - hibernateBase);
+ header->restore1CodePhysPage = page;
+ header->restore1CodeVirt = hibernateBase;
+ header->restore1PageCount = count;
+ header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
+ header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
+
+ if (uuid_parse(&gIOHibernateBridgeBootSessionUUIDString[0], &header->bridgeBootSessionUUID[0])) {
+ bzero(&header->bridgeBootSessionUUID[0], sizeof(header->bridgeBootSessionUUID));
+ }
+
+ // sum __HIB seg, with zeros for the stack
+ src = (uint8_t *) trunc_page(hibernateBase);
+ for (page = 0; page < count; page++) {
+ if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0])) {
+ restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
+ } else {
+ restore1Sum += 0x00000000;
+ }
+ src += page_size;
+ }
+ sum1 = restore1Sum;
+
+ // write the __HIB seg, with zeros for the stack
+
+ src = (uint8_t *) trunc_page(hibernateBase);
+ count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
+ if (count) {
+ err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ }
+ err = IOHibernatePolledFileWrite(vars->fileVars,
+ (uint8_t *) NULL,
+ &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
+ cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ src = &gIOHibernateRestoreStackEnd[0];
+ count = round_page(hibernateEnd) - ((uintptr_t) src);
+ if (count) {
+ err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ }
+
+ if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) {
+ vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
+ vars->fileVars->encryptEnd = UINT64_MAX;
+ HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
+ }
+
+ // write the preview buffer
+
+ if (vars->previewBuffer) {
+ ppnum = 0;
+ count = 0;
+ do{
+ phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
+ pageAndCount[0] = atop_64(phys64);
+ pageAndCount[1] = atop_32(segLen);
+ err = IOHibernatePolledFileWrite(vars->fileVars,
+ (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
+ cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ count += segLen;
+ ppnum += sizeof(pageAndCount);
+ }while (phys64);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+
+ src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
+
+ ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
+
+ count = vars->previewBuffer->getLength();
+
+ header->previewPageListSize = ppnum;
+ header->previewSize = count + ppnum;
+
+ for (page = 0; page < count; page += page_size) {
+ phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
+ sum1 += hibernate_sum_page(src + page, atop_64(phys64));
+ }
+ err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ }
+
+ // mark areas for no save
+ IOMemoryDescriptor * ioBuffer;
+ ioBuffer = IOPolledFileGetIOBuffer(vars->fileVars);
+ for (count = 0;
+ (phys64 = ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+ count += segLen) {
+ hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+ atop_64(phys64), atop_32(segLen),
+ kIOHibernatePageStateFree);
+ pageCount -= atop_32(segLen);
+ }
+
+ for (count = 0;
+ (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+ count += segLen) {
+ hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+ atop_64(phys64), atop_32(segLen),
+ kIOHibernatePageStateFree);
+ pageCount -= atop_32(segLen);
+ }
+
+ // copy out bitmap of pages available for trashing during restore
+
+ bitmap_size = vars->page_list_wired->list_size;
+ src = (uint8_t *) vars->page_list_wired;
+ err = IOHibernatePolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+
+ // mark more areas for no save, but these are not available
+ // for trashing during restore
+
+ hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
+
+
+ page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
+ count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
+ hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+ page, count,
+ kIOHibernatePageStateFree);
+ pageCount -= count;
+
+ if (vars->previewBuffer) {
+ for (count = 0;
+ (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+ count += segLen) {
+ hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+ atop_64(phys64), atop_32(segLen),
+ kIOHibernatePageStateFree);
+ pageCount -= atop_32(segLen);
+ }
+ }
+
+ for (count = 0;
+ (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+ count += segLen) {
+ hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+ atop_64(phys64), atop_32(segLen),
+ kIOHibernatePageStateFree);
+ pageCount -= atop_32(segLen);
+ }
+
+#if KASAN
+ vm_size_t shadow_pages_free = atop_64(shadow_ptop) - atop_64(shadow_pnext);
+
+ /* no need to save unused shadow pages */
+ hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+ atop_64(shadow_pnext),
+ shadow_pages_free,
+ kIOHibernatePageStateFree);
+#endif
+
+ src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
+ compressed = src + page_size;
+ scratch = compressed + page_size;
+
+ pagesDone = 0;
+ lastBlob = 0;
+
+ HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
+ bitmap_size, header->previewSize,
+ pageCount, vars->fileVars->position);
+
+ enum
+ // pageType
+ {
+ kWired = 0x02,
+ kEncrypt = 0x01,
+ kWiredEncrypt = kWired | kEncrypt,
+ kWiredClear = kWired,
+ kUnwiredEncrypt = kEncrypt
+ };
+
+ bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features()));
+
+ for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--) {
+ if (kUnwiredEncrypt == pageType) {
+ // start unwired image
+ if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) {
+ vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
+ vars->fileVars->encryptEnd = UINT64_MAX;
+ HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
+ }
+ bcopy(&cryptvars->aes_iv[0],
+ &gIOHibernateCryptWakeContext.aes_iv[0],
+ sizeof(cryptvars->aes_iv));
+ cryptvars = &gIOHibernateCryptWakeContext;
+ }
+ for (iterDone = false, ppnum = 0; !iterDone;) {
+ if (cpuAES && (pageType == kWiredClear)) {
+ count = 0;
+ } else {
+ count = hibernate_page_list_iterate((kWired & pageType) ? vars->page_list_wired : vars->page_list,
+ &ppnum);
+ }
projects / apple / xnu.git / blobdiff