X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/6d2010ae8f7a6078e10b361c6962983bab233e0f..008676633c2ad2c325837c2b64915f7ded690a8f:/iokit/Kernel/IOHibernateRestoreKernel.c diff --git a/iokit/Kernel/IOHibernateRestoreKernel.c b/iokit/Kernel/IOHibernateRestoreKernel.c index 7259ab3ec..017d4d4f8 100644 --- a/iokit/Kernel/IOHibernateRestoreKernel.c +++ b/iokit/Kernel/IOHibernateRestoreKernel.c @@ -32,15 +32,12 @@ #include #include #include -#include #include -#include +#include #include "IOHibernateInternal.h" -#if defined(__i386__) || defined(__x86_64__) -#include -#endif +#include /* This code is linked into the kernel but part of the "__HIB" section, which means @@ -77,6 +74,27 @@ extern void acpi_wake_prot_entry(void); #if defined(__i386__) || defined(__x86_64__) +#define rdtsc(lo,hi) \ + __asm__ volatile("lfence; rdtsc; lfence" : "=a" (lo), "=d" (hi)) + +static inline uint64_t rdtsc64(void) +{ + uint64_t lo, hi; + rdtsc(lo, hi); + return ((hi) << 32) | (lo); +} + +#else + +static inline uint64_t rdtsc64(void) +{ + return (0); +} + +#endif /* defined(__i386__) || defined(__x86_64__) */ + +#if defined(__i386__) || defined(__x86_64__) + #define DBGLOG 1 #include @@ -201,7 +219,12 @@ enum kIOHibernateRestoreCodeWakeMapSize = 'wkms', kIOHibernateRestoreCodeConflictPage = 'cfpg', kIOHibernateRestoreCodeConflictSource = 'cfsr', - kIOHibernateRestoreCodeNoMemory = 'nomm' + kIOHibernateRestoreCodeNoMemory = 'nomm', + kIOHibernateRestoreCodeTag = 'tag ', + kIOHibernateRestoreCodeSignature = 'sign', + kIOHibernateRestoreCodeMapVirt = 'mapV', + kIOHibernateRestoreCodeHandoffPages = 'hand', + kIOHibernateRestoreCodeHandoffCount = 'hndc', }; /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -335,6 +358,8 @@ hibernate_page_bitmap_count(hibernate_bitmap_t * bitmap, uint32_t set, uint32_t } } + if ((page + count) > (bitmap->last_page + 1)) count = (bitmap->last_page + 1) - page; + return (count); } @@ -371,11 +396,22 @@ store_one_page(uint32_t procFlags, uint32_t * src, uint32_t compressedSize, uint32_t * buffer, uint32_t ppnum) { uint64_t dst = ptoa_64(ppnum); + uint8_t scratch[WKdm_SCRATCH_BUF_SIZE_INTERNAL] __attribute__ ((aligned (16))); if (compressedSize != PAGE_SIZE) { dst = pal_hib_map(DEST_COPY_AREA, dst); - WKdm_decompress((WK_word*) src, (WK_word*)(uintptr_t)dst, PAGE_SIZE >> 2); + if (compressedSize != 4) WKdm_decompress_new((WK_word*) src, (WK_word*)(uintptr_t)dst, (WK_word*) &scratch[0], compressedSize); + else + { + size_t i; + uint32_t s, *d; + + s = *src; + d = (uint32_t *)(uintptr_t)dst; + if (!s) bzero((void *) dst, PAGE_SIZE); + else for (i = 0; i < (PAGE_SIZE / sizeof(int32_t)); i++) *d++ = s; + } } else { @@ -385,30 +421,15 @@ store_one_page(uint32_t procFlags, uint32_t * src, uint32_t compressedSize, return hibernate_sum_page((uint8_t *)(uintptr_t)dst, ppnum); } -// used only for small struct copies -static void -bcopy_internal(const void *src, void *dst, uint32_t len) -{ - const char *s = src; - char *d = dst; - uint32_t idx = 0; - - while (idx < len) - { - d[idx] = s[idx]; - idx++; - } -} - -#define C_ASSERT(e) typedef char __C_ASSERT__[(e) ? 1 : -1] - long -hibernate_kernel_entrypoint(IOHibernateImageHeader * header, - void * p2, void * p3, void * p4) +hibernate_kernel_entrypoint(uint32_t p1, + uint32_t p2, uint32_t p3, uint32_t p4) { - uint32_t idx; - uint32_t * src; - uint32_t * imageReadPos; + uint64_t headerPhys; + uint64_t mapPhys; + uint64_t srcPhys; + uint64_t imageReadPhys; + uint64_t pageIndexPhys; uint32_t * pageIndexSource; hibernate_page_list_t * map; uint32_t stage; @@ -418,8 +439,10 @@ hibernate_kernel_entrypoint(IOHibernateImageHeader * header, uint32_t conflictCount; uint32_t compressedSize; uint32_t uncompressedPages; - uint32_t copyPageListHead; + uint32_t copyPageListHeadPage; + uint32_t pageListPage; uint32_t * copyPageList; + uint32_t * src; uint32_t copyPageIndex; uint32_t sum; uint32_t pageSum; @@ -430,39 +453,48 @@ hibernate_kernel_entrypoint(IOHibernateImageHeader * header, uint32_t handoffPages; uint32_t handoffPageCount; - C_ASSERT(sizeof(IOHibernateImageHeader) == 512); + uint64_t timeStart; + timeStart = rdtsc64(); + + static_assert(sizeof(IOHibernateImageHeader) == 512); + + headerPhys = ptoa_64(p1); if ((kIOHibernateDebugRestoreLogs & gIOHibernateDebugFlags) && !debug_probe()) gIOHibernateDebugFlags &= ~kIOHibernateDebugRestoreLogs; - debug_code(kIOHibernateRestoreCodeImageStart, (uintptr_t) header); + debug_code(kIOHibernateRestoreCodeImageStart, headerPhys); + + memcpy(gIOHibernateCurrentHeader, + (void *) pal_hib_map(IMAGE_AREA, headerPhys), + sizeof(IOHibernateImageHeader)); - bcopy_internal(header, - gIOHibernateCurrentHeader, - sizeof(IOHibernateImageHeader)); + debug_code(kIOHibernateRestoreCodeSignature, gIOHibernateCurrentHeader->signature); - map = (hibernate_page_list_t *) - (((uintptr_t) &header->fileExtentMap[0]) - + header->fileExtentMapSize - + ptoa_32(header->restore1PageCount) - + header->previewSize); + mapPhys = headerPhys + + (offsetof(IOHibernateImageHeader, fileExtentMap) + + gIOHibernateCurrentHeader->fileExtentMapSize + + ptoa_32(gIOHibernateCurrentHeader->restore1PageCount) + + gIOHibernateCurrentHeader->previewSize); - lastImagePage = atop_32(((uintptr_t) header) + header->image1Size); + map = (hibernate_page_list_t *) pal_hib_map(BITMAP_AREA, mapPhys); - lastMapPage = atop_32(((uintptr_t) map) + header->bitmapSize); + lastImagePage = atop_64(headerPhys + gIOHibernateCurrentHeader->image1Size); + lastMapPage = atop_64(mapPhys + gIOHibernateCurrentHeader->bitmapSize); - handoffPages = header->handoffPages; - handoffPageCount = header->handoffPageCount; + handoffPages = gIOHibernateCurrentHeader->handoffPages; + handoffPageCount = gIOHibernateCurrentHeader->handoffPageCount; debug_code(kIOHibernateRestoreCodeImageEnd, ptoa_64(lastImagePage)); - debug_code(kIOHibernateRestoreCodeMapStart, (uintptr_t) map); + debug_code(kIOHibernateRestoreCodeMapStart, mapPhys); debug_code(kIOHibernateRestoreCodeMapEnd, ptoa_64(lastMapPage)); - debug_code('hand', ptoa_64(handoffPages)); - debug_code('hnde', ptoa_64(handoffPageCount)); + debug_code(kIOHibernateRestoreCodeMapVirt, (uintptr_t) map); + debug_code(kIOHibernateRestoreCodeHandoffPages, ptoa_64(handoffPages)); + debug_code(kIOHibernateRestoreCodeHandoffCount, handoffPageCount); // knock all the image pages to be used out of free map - for (ppnum = atop_32((uintptr_t) header); ppnum <= lastImagePage; ppnum++) + for (ppnum = atop_64(headerPhys); ppnum <= lastImagePage; ppnum++) { hibernate_page_bitset(map, FALSE, ppnum); } @@ -475,40 +507,40 @@ hibernate_kernel_entrypoint(IOHibernateImageHeader * header, nextFree = 0; hibernate_page_list_grab(map, &nextFree); - pal_hib_window_setup(hibernate_page_list_grab(map, &nextFree)); - - sum = header->actualRestore1Sum; - gIOHibernateCurrentHeader->diag[0] = (uint32_t)(uintptr_t) header; + sum = gIOHibernateCurrentHeader->actualRestore1Sum; + gIOHibernateCurrentHeader->diag[0] = atop_64(headerPhys); gIOHibernateCurrentHeader->diag[1] = sum; + gIOHibernateCurrentHeader->trampolineTime = 0; - uncompressedPages = 0; - conflictCount = 0; - copyPageListHead = 0; - copyPageList = 0; - copyPageIndex = PAGE_SIZE >> 2; + uncompressedPages = 0; + conflictCount = 0; + copyPageListHeadPage = 0; + copyPageList = 0; + copyPageIndex = PAGE_SIZE >> 2; - compressedSize = PAGE_SIZE; - stage = 2; - count = 0; - src = NULL; + compressedSize = PAGE_SIZE; + stage = 2; + count = 0; + srcPhys = 0; if (gIOHibernateCurrentHeader->previewSize) { - pageIndexSource = (uint32_t *) - (((uintptr_t) &header->fileExtentMap[0]) - + gIOHibernateCurrentHeader->fileExtentMapSize - + ptoa_32(gIOHibernateCurrentHeader->restore1PageCount)); - imageReadPos = (uint32_t *) (((uintptr_t) pageIndexSource) + gIOHibernateCurrentHeader->previewPageListSize); - lastPageIndexPage = atop_32((uintptr_t) imageReadPos); + pageIndexPhys = headerPhys + + (offsetof(IOHibernateImageHeader, fileExtentMap) + + gIOHibernateCurrentHeader->fileExtentMapSize + + ptoa_32(gIOHibernateCurrentHeader->restore1PageCount)); + imageReadPhys = (pageIndexPhys + gIOHibernateCurrentHeader->previewPageListSize); + lastPageIndexPage = atop_64(imageReadPhys); + pageIndexSource = (uint32_t *) pal_hib_map(IMAGE2_AREA, pageIndexPhys); } else { - pageIndexSource = NULL; + pageIndexPhys = 0; lastPageIndexPage = 0; - imageReadPos = (uint32_t *) (((uintptr_t) map) + gIOHibernateCurrentHeader->bitmapSize); + imageReadPhys = (mapPhys + gIOHibernateCurrentHeader->bitmapSize); } - debug_code(kIOHibernateRestoreCodePageIndexStart, (uintptr_t) pageIndexSource); + debug_code(kIOHibernateRestoreCodePageIndexStart, pageIndexPhys); debug_code(kIOHibernateRestoreCodePageIndexEnd, ptoa_64(lastPageIndexPage)); while (1) @@ -517,38 +549,35 @@ hibernate_kernel_entrypoint(IOHibernateImageHeader * header, { case 2: // copy handoff data - count = src ? 0 : handoffPageCount; + count = srcPhys ? 0 : handoffPageCount; if (!count) break; - if (count > gIOHibernateHandoffPageCount) - count = gIOHibernateHandoffPageCount; - src = (uint32_t *) (uintptr_t) ptoa_64(handoffPages); + if (count > gIOHibernateHandoffPageCount) count = gIOHibernateHandoffPageCount; + srcPhys = ptoa_64(handoffPages); break; case 1: // copy pageIndexSource pages == preview image data - if (!src) + if (!srcPhys) { - if (!pageIndexSource) - break; - src = imageReadPos; + if (!pageIndexPhys) break; + srcPhys = imageReadPhys; } ppnum = pageIndexSource[0]; count = pageIndexSource[1]; pageIndexSource += 2; - imageReadPos = src; + pageIndexPhys += 2 * sizeof(pageIndexSource[0]); + imageReadPhys = srcPhys; break; case 0: // copy pages - if (!src) - { - src = (uint32_t *) (((uintptr_t) map) + gIOHibernateCurrentHeader->bitmapSize); - } + if (!srcPhys) srcPhys = (mapPhys + gIOHibernateCurrentHeader->bitmapSize); + src = (uint32_t *) pal_hib_map(IMAGE_AREA, srcPhys); ppnum = src[0]; count = src[1]; - src += 2; - imageReadPos = src; + srcPhys += 2 * sizeof(*src); + imageReadPhys = srcPhys; break; } @@ -558,7 +587,7 @@ hibernate_kernel_entrypoint(IOHibernateImageHeader * header, if (!stage) break; stage--; - src = NULL; + srcPhys = 0; continue; } @@ -567,27 +596,29 @@ hibernate_kernel_entrypoint(IOHibernateImageHeader * header, uint32_t tag; int conflicts; - if (2 == stage) - ppnum = gIOHibernateHandoffPages[page]; + src = (uint32_t *) pal_hib_map(IMAGE_AREA, srcPhys); + + if (2 == stage) ppnum = gIOHibernateHandoffPages[page]; else if (!stage) { tag = *src++; +// debug_code(kIOHibernateRestoreCodeTag, (uintptr_t) tag); + srcPhys += sizeof(*src); compressedSize = kIOHibernateTagLength & tag; } - conflicts = (ppnum >= atop_32((uintptr_t) map)) && (ppnum <= lastMapPage); + conflicts = (ppnum >= atop_64(mapPhys)) && (ppnum <= lastMapPage); - conflicts |= ((ppnum >= atop_32((uintptr_t) imageReadPos)) && (ppnum <= lastImagePage)); + conflicts |= ((ppnum >= atop_64(imageReadPhys)) && (ppnum <= lastImagePage)); if (stage >= 2) - conflicts |= ((ppnum >= atop_32((uintptr_t) src)) && (ppnum <= (handoffPages + handoffPageCount - 1))); + conflicts |= ((ppnum >= atop_64(srcPhys)) && (ppnum <= (handoffPages + handoffPageCount - 1))); if (stage >= 1) - conflicts |= ((ppnum >= atop_32((uintptr_t) pageIndexSource)) && (ppnum <= lastPageIndexPage)); + conflicts |= ((ppnum >= atop_64(pageIndexPhys)) && (ppnum <= lastPageIndexPage)); if (!conflicts) { -// if (compressedSize) pageSum = store_one_page(gIOHibernateCurrentHeader->processorFlags, src, compressedSize, 0, ppnum); if (stage != 2) @@ -596,43 +627,41 @@ hibernate_kernel_entrypoint(IOHibernateImageHeader * header, } else { - uint32_t bufferPage; + uint32_t bufferPage = 0; uint32_t * dst; // debug_code(kIOHibernateRestoreCodeConflictPage, ppnum); // debug_code(kIOHibernateRestoreCodeConflictSource, (uintptr_t) src); - conflictCount++; - - // alloc new buffer page - bufferPage = hibernate_page_list_grab(map, &nextFree); - + if (compressedSize) + { + // alloc new buffer page + bufferPage = hibernate_page_list_grab(map, &nextFree); + dst = (uint32_t *)pal_hib_map(DEST_COPY_AREA, ptoa_64(bufferPage)); + memcpy(dst, src, compressedSize); + } if (copyPageIndex > ((PAGE_SIZE >> 2) - 3)) { // alloc new copy list page - uint32_t pageListPage = hibernate_page_list_grab(map, &nextFree); + pageListPage = hibernate_page_list_grab(map, &nextFree); // link to current if (copyPageList) { copyPageList[1] = pageListPage; } else { - copyPageListHead = pageListPage; + copyPageListHeadPage = pageListPage; } copyPageList = (uint32_t *)pal_hib_map(SRC_COPY_AREA, - ptoa_32(pageListPage)); + ptoa_64(pageListPage)); copyPageList[1] = 0; copyPageIndex = 2; } - copyPageList[copyPageIndex++] = ppnum; copyPageList[copyPageIndex++] = bufferPage; copyPageList[copyPageIndex++] = (compressedSize | (stage << 24)); copyPageList[0] = copyPageIndex; - - dst = (uint32_t *)pal_hib_map(DEST_COPY_AREA, ptoa_32(bufferPage)); - for (idx = 0; idx < ((compressedSize + 3) >> 2); idx++) - dst[idx] = src[idx]; } - src += ((compressedSize + 3) >> 2); + srcPhys += ((compressedSize + 3) & ~3); + src += ((compressedSize + 3) >> 2); } } @@ -641,16 +670,15 @@ hibernate_kernel_entrypoint(IOHibernateImageHeader * header, // -- copy back conflicts - copyPageList = (uint32_t *)(uintptr_t) ptoa_32(copyPageListHead); - - while (copyPageList) + pageListPage = copyPageListHeadPage; + while (pageListPage) { - copyPageList = (uint32_t *)pal_hib_map(COPY_PAGE_AREA, (uintptr_t)copyPageList); + copyPageList = (uint32_t *)pal_hib_map(COPY_PAGE_AREA, ptoa_64(pageListPage)); for (copyPageIndex = 2; copyPageIndex < copyPageList[0]; copyPageIndex += 3) { ppnum = copyPageList[copyPageIndex + 0]; - src = (uint32_t *) (uintptr_t) ptoa_32(copyPageList[copyPageIndex + 1]); - src = (uint32_t *)pal_hib_map(SRC_COPY_AREA, (uintptr_t)src); + srcPhys = ptoa_64(copyPageList[copyPageIndex + 1]); + src = (uint32_t *) pal_hib_map(SRC_COPY_AREA, srcPhys); compressedSize = copyPageList[copyPageIndex + 2]; stage = compressedSize >> 24; compressedSize &= 0x1FFF; @@ -660,7 +688,7 @@ hibernate_kernel_entrypoint(IOHibernateImageHeader * header, sum += pageSum; uncompressedPages++; } - copyPageList = (uint32_t *) (uintptr_t) ptoa_32(copyPageList[1]); + pageListPage = copyPageList[1]; } pal_hib_patchup(); @@ -674,6 +702,10 @@ hibernate_kernel_entrypoint(IOHibernateImageHeader * header, gIOHibernateState = kIOHibernateStateWakingFromHibernate; + gIOHibernateCurrentHeader->trampolineTime = (((rdtsc64() - timeStart)) >> 8); + +// debug_code('done', 0); + #if CONFIG_SLEEP #if defined(__i386__) || defined(__x86_64__) typedef void (*ResetProc)(void);