+ // knock all the image pages to be used out of free map
+ for (ppnum = atop_64(headerPhys); ppnum <= lastImagePage; ppnum++) {
+ hibernate_page_bitset(map, FALSE, ppnum);
+ }
+ // knock all the handoff pages to be used out of free map
+ for (ppnum = handoffPages; ppnum < (handoffPages + handoffPageCount); ppnum++) {
+ hibernate_page_bitset(map, FALSE, ppnum);
+ }
+
+ nextFree = 0;
+ hibernate_page_list_grab(map, &nextFree);
+
+ sum = gIOHibernateCurrentHeader->actualRestore1Sum;
+ gIOHibernateCurrentHeader->diag[0] = atop_64(headerPhys);
+ gIOHibernateCurrentHeader->diag[1] = sum;
+ gIOHibernateCurrentHeader->trampolineTime = 0;
+
+ uncompressedPages = 0;
+ conflictCount = 0;
+ copyPageListHeadPage = 0;
+ copyPageList = 0;
+ copyPageIndex = PAGE_SIZE >> 2;
+
+ compressedSize = PAGE_SIZE;
+ stage = 2;
+ count = 0;
+ srcPhys = 0;
+
+ if (gIOHibernateCurrentHeader->previewSize) {
+ pageIndexPhys = headerPhys
+ + (offsetof(IOHibernateImageHeader, fileExtentMap)
+ + gIOHibernateCurrentHeader->fileExtentMapSize
+ + ptoa_32(gIOHibernateCurrentHeader->restore1PageCount));
+ imageReadPhys = (pageIndexPhys + gIOHibernateCurrentHeader->previewPageListSize);
+ lastPageIndexPage = atop_64(imageReadPhys);
+ pageIndexSource = (uint32_t *) pal_hib_map(IMAGE2_AREA, pageIndexPhys);
+ } else {
+ pageIndexPhys = 0;
+ lastPageIndexPage = 0;
+ imageReadPhys = (mapPhys + gIOHibernateCurrentHeader->bitmapSize);
+ }
+
+ debug_code(kIOHibernateRestoreCodePageIndexStart, pageIndexPhys);
+ debug_code(kIOHibernateRestoreCodePageIndexEnd, ptoa_64(lastPageIndexPage));
+
+ while (1) {
+ switch (stage) {
+ case 2:
+ // copy handoff data
+ count = srcPhys ? 0 : handoffPageCount;
+ if (!count) {
+ break;
+ }
+ if (count > gIOHibernateHandoffPageCount) {
+ count = gIOHibernateHandoffPageCount;
+ }
+ srcPhys = ptoa_64(handoffPages);
+ break;
+
+ case 1:
+ // copy pageIndexSource pages == preview image data
+ if (!srcPhys) {
+ if (!pageIndexPhys) {
+ break;
+ }
+ srcPhys = imageReadPhys;
+ }
+ ppnum = pageIndexSource[0];
+ count = pageIndexSource[1];
+ pageIndexSource += 2;
+ pageIndexPhys += 2 * sizeof(pageIndexSource[0]);
+ imageReadPhys = srcPhys;
+ break;
+
+ case 0:
+ // copy pages
+ if (!srcPhys) {
+ srcPhys = (mapPhys + gIOHibernateCurrentHeader->bitmapSize);
+ }
+ src = (uint32_t *) pal_hib_map(IMAGE_AREA, srcPhys);
+ ppnum = src[0];
+ count = src[1];
+ srcPhys += 2 * sizeof(*src);
+ imageReadPhys = srcPhys;
+ break;
+ }
+
+
+ if (!count) {
+ if (!stage) {
+ break;
+ }
+ stage--;
+ srcPhys = 0;
+ continue;
+ }
+
+ for (page = 0; page < count; page++, ppnum++) {
+ uint32_t tag;
+ int conflicts;
+
+ src = (uint32_t *) pal_hib_map(IMAGE_AREA, srcPhys);
+
+ if (2 == stage) {
+ ppnum = gIOHibernateHandoffPages[page];
+ } else if (!stage) {
+ tag = *src++;
+// debug_code(kIOHibernateRestoreCodeTag, (uintptr_t) tag);
+ srcPhys += sizeof(*src);
+ compressedSize = kIOHibernateTagLength & tag;
+ }
+
+ conflicts = (ppnum >= atop_64(mapPhys)) && (ppnum <= lastMapPage);
+
+ conflicts |= ((ppnum >= atop_64(imageReadPhys)) && (ppnum <= lastImagePage));
+
+ if (stage >= 2) {
+ conflicts |= ((ppnum >= atop_64(srcPhys)) && (ppnum <= (handoffPages + handoffPageCount - 1)));
+ }
+
+ if (stage >= 1) {
+ conflicts |= ((ppnum >= atop_64(pageIndexPhys)) && (ppnum <= lastPageIndexPage));
+ }
+
+ if (!conflicts) {
+ pageSum = store_one_page(gIOHibernateCurrentHeader->processorFlags,
+ src, compressedSize, 0, ppnum);
+ if (stage != 2) {
+ sum += pageSum;
+ }
+ uncompressedPages++;
+ } else {
+ uint32_t bufferPage = 0;
+ uint32_t * dst;
+
+// debug_code(kIOHibernateRestoreCodeConflictPage, ppnum);
+// debug_code(kIOHibernateRestoreCodeConflictSource, (uintptr_t) src);
+ conflictCount++;
+ if (compressedSize) {
+ // alloc new buffer page
+ bufferPage = hibernate_page_list_grab(map, &nextFree);
+ dst = (uint32_t *)pal_hib_map(DEST_COPY_AREA, ptoa_64(bufferPage));
+ __nosan_memcpy(dst, src, compressedSize);
+ }
+ if (copyPageIndex > ((PAGE_SIZE >> 2) - 3)) {
+ // alloc new copy list page
+ pageListPage = hibernate_page_list_grab(map, &nextFree);
+ // link to current
+ if (copyPageList) {
+ copyPageList[1] = pageListPage;
+ } else {
+ copyPageListHeadPage = pageListPage;
+ }
+ copyPageList = (uint32_t *)pal_hib_map(SRC_COPY_AREA,
+ ptoa_64(pageListPage));
+ copyPageList[1] = 0;
+ copyPageIndex = 2;
+ }
+ copyPageList[copyPageIndex++] = ppnum;
+ copyPageList[copyPageIndex++] = bufferPage;
+ copyPageList[copyPageIndex++] = (compressedSize | (stage << 24));
+ copyPageList[0] = copyPageIndex;
+ }
+ srcPhys += ((compressedSize + 3) & ~3);
+ src += ((compressedSize + 3) >> 2);
+ }
+ }
+
+ /* src points to the last page restored, so we need to skip over that */
+ hibernateRestorePALState(src);
+
+ // -- copy back conflicts
+
+ pageListPage = copyPageListHeadPage;
+ while (pageListPage) {
+ copyPageList = (uint32_t *)pal_hib_map(COPY_PAGE_AREA, ptoa_64(pageListPage));
+ for (copyPageIndex = 2; copyPageIndex < copyPageList[0]; copyPageIndex += 3) {
+ ppnum = copyPageList[copyPageIndex + 0];
+ srcPhys = ptoa_64(copyPageList[copyPageIndex + 1]);
+ src = (uint32_t *) pal_hib_map(SRC_COPY_AREA, srcPhys);
+ compressedSize = copyPageList[copyPageIndex + 2];
+ stage = compressedSize >> 24;
+ compressedSize &= 0x1FFF;
+ pageSum = store_one_page(gIOHibernateCurrentHeader->processorFlags,
+ src, compressedSize, 0, ppnum);
+ if (stage != 2) {
+ sum += pageSum;
+ }
+ uncompressedPages++;
+ }
+ pageListPage = copyPageList[1];
+ }
+
+ pal_hib_patchup();
+
+ // -- image has been destroyed...
+
+ gIOHibernateCurrentHeader->actualImage1Sum = sum;
+ gIOHibernateCurrentHeader->actualUncompressedPages = uncompressedPages;
+ gIOHibernateCurrentHeader->conflictCount = conflictCount;
+ gIOHibernateCurrentHeader->nextFree = nextFree;
+
+ gIOHibernateState = kIOHibernateStateWakingFromHibernate;
+
+ gIOHibernateCurrentHeader->trampolineTime = (((rdtsc64() - timeStart)) >> 8);
+
+// debug_code('done', 0);
+
+#if CONFIG_SLEEP
+#if defined(__i386__) || defined(__x86_64__)
+ typedef void (*ResetProc)(void);
+ ResetProc proc;
+ proc = HIB_ENTRYPOINT;
+ // flush caches
+ __asm__("wbinvd");
+ proc();
+#else
+// implement me
+#endif
+#endif
+
+ return -1;
projects / apple / xnu.git / blobdiff