xnu-2782.20.48.tar.gz

[apple/xnu.git] / iokit / Kernel / IOHibernateRestoreKernel.c

diff --git a/iokit/Kernel/IOHibernateRestoreKernel.c b/iokit/Kernel/IOHibernateRestoreKernel.c
index 79410326b5521cd67088b64f43e4a582e47977e6..daf5d2804b2405acee3cda41ab3f9c7b48a6a06a 100644 (file)
--- a/iokit/Kernel/IOHibernateRestoreKernel.c
+++ b/iokit/Kernel/IOHibernateRestoreKernel.c
@@ -32,15 +32,12 @@
 #include <IOKit/IOHibernatePrivate.h>
 #include <IOKit/IOLib.h>
 #include <pexpert/boot.h>
 #include <IOKit/IOHibernatePrivate.h>
 #include <IOKit/IOLib.h>
 #include <pexpert/boot.h>

-#include <crypto/aes.h>

 #include <libkern/libkern.h>
 
 #include <libkern/libkern.h>
 

-#include <libkern/WKdm.h>
+#include <vm/WKdm_new.h>

 #include "IOHibernateInternal.h"
 
 #include "IOHibernateInternal.h"
 

-#if defined(__i386__) || defined(__x86_64__)
-#include <i386/pal_hibernate.h>
-#endif
+#include <machine/pal_hibernate.h>

 
 /*
 This code is linked into the kernel but part of the "__HIB" section, which means
 
 /*
 This code is linked into the kernel but part of the "__HIB" section, which means


@@ -77,6 +74,27 @@ extern void acpi_wake_prot_entry(void);
 
 #if defined(__i386__) || defined(__x86_64__)
 
 
 #if defined(__i386__) || defined(__x86_64__)
 

+#define rdtsc(lo,hi) \
+    __asm__ volatile("lfence; rdtsc; lfence" : "=a" (lo), "=d" (hi))
+
+static inline uint64_t rdtsc64(void)
+{
+    uint64_t lo, hi;
+    rdtsc(lo, hi);
+    return ((hi) << 32) | (lo);
+}
+
+#else
+
+static inline uint64_t rdtsc64(void)
+{
+    return (0);
+}
+
+#endif /* defined(__i386__) || defined(__x86_64__) */
+
+#if defined(__i386__) || defined(__x86_64__)
+

 #define DBGLOG 1
 
 #include <architecture/i386/pio.h>
 #define DBGLOG 1
 
 #include <architecture/i386/pio.h>


@@ -378,11 +396,13 @@ store_one_page(uint32_t procFlags, uint32_t * src, uint32_t compressedSize,
                uint32_t * buffer, uint32_t ppnum)
 {
        uint64_t dst = ptoa_64(ppnum);
                uint32_t * buffer, uint32_t ppnum)
 {
        uint64_t dst = ptoa_64(ppnum);

+       uint8_t scratch[WKdm_SCRATCH_BUF_SIZE] __attribute__ ((aligned (16)));

 
        if (compressedSize != PAGE_SIZE)
        {
                dst = pal_hib_map(DEST_COPY_AREA, dst);
 
        if (compressedSize != PAGE_SIZE)
        {
                dst = pal_hib_map(DEST_COPY_AREA, dst);

-               WKdm_decompress((WK_word*) src, (WK_word*)(uintptr_t)dst, PAGE_SIZE >> 2);
+               if (compressedSize) WKdm_decompress_new((WK_word*) src, (WK_word*)(uintptr_t)dst, (WK_word*) &scratch[0], PAGE_SIZE);
+               else bzero((void *) dst, PAGE_SIZE);

        }
        else
        {
        }
        else
        {


@@ -392,21 +412,6 @@ store_one_page(uint32_t procFlags, uint32_t * src, uint32_t compressedSize,
        return hibernate_sum_page((uint8_t *)(uintptr_t)dst, ppnum);
 }
 
        return hibernate_sum_page((uint8_t *)(uintptr_t)dst, ppnum);
 }
 

-// used only for small struct copies
-static void 
-bcopy_internal(const void *src, void *dst, uint32_t len)
-{
-    const char *s = src;
-    char       *d = dst;
-    uint32_t   idx = 0;
-
-    while (idx < len)
-    {
-        d[idx] = s[idx];
-        idx++;
-    }
-}
-

 #define C_ASSERT(e) typedef char    __C_ASSERT__[(e) ? 1 : -1]
 
 long 
 #define C_ASSERT(e) typedef char    __C_ASSERT__[(e) ? 1 : -1]
 
 long 


@@ -418,7 +423,6 @@ hibernate_kernel_entrypoint(uint32_t p1,
     uint64_t srcPhys;
     uint64_t imageReadPhys;
     uint64_t pageIndexPhys;
     uint64_t srcPhys;
     uint64_t imageReadPhys;
     uint64_t pageIndexPhys;

-    uint32_t idx;

     uint32_t * pageIndexSource;
     hibernate_page_list_t * map;
     uint32_t stage;
     uint32_t * pageIndexSource;
     hibernate_page_list_t * map;
     uint32_t stage;


@@ -442,6 +446,9 @@ hibernate_kernel_entrypoint(uint32_t p1,
     uint32_t handoffPages;
     uint32_t handoffPageCount;
 
     uint32_t handoffPages;
     uint32_t handoffPageCount;
 

+    uint64_t timeStart;
+    timeStart = rdtsc64();
+

     C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
 
     headerPhys = ptoa_64(p1);
     C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
 
     headerPhys = ptoa_64(p1);


@@ -451,9 +458,9 @@ hibernate_kernel_entrypoint(uint32_t p1,
 
     debug_code(kIOHibernateRestoreCodeImageStart, headerPhys);
 
 
     debug_code(kIOHibernateRestoreCodeImageStart, headerPhys);
 

-    bcopy_internal((void *) pal_hib_map(IMAGE_AREA, headerPhys), 
-                   gIOHibernateCurrentHeader, 
-                   sizeof(IOHibernateImageHeader));
+    memcpy(gIOHibernateCurrentHeader,
+          (void *) pal_hib_map(IMAGE_AREA, headerPhys), 
+          sizeof(IOHibernateImageHeader));

 
     debug_code(kIOHibernateRestoreCodeSignature, gIOHibernateCurrentHeader->signature);
 
 
     debug_code(kIOHibernateRestoreCodeSignature, gIOHibernateCurrentHeader->signature);
 


@@ -496,6 +503,7 @@ hibernate_kernel_entrypoint(uint32_t p1,
     sum = gIOHibernateCurrentHeader->actualRestore1Sum;
     gIOHibernateCurrentHeader->diag[0] = atop_64(headerPhys);
     gIOHibernateCurrentHeader->diag[1] = sum;
     sum = gIOHibernateCurrentHeader->actualRestore1Sum;
     gIOHibernateCurrentHeader->diag[0] = atop_64(headerPhys);
     gIOHibernateCurrentHeader->diag[1] = sum;

+    gIOHibernateCurrentHeader->trampolineTime = 0;

 
     uncompressedPages    = 0;
     conflictCount        = 0;
 
     uncompressedPages    = 0;
     conflictCount        = 0;


@@ -604,7 +612,6 @@ hibernate_kernel_entrypoint(uint32_t p1,
 
            if (!conflicts)
            {
 
            if (!conflicts)
            {

-//              if (compressedSize)

                pageSum = store_one_page(gIOHibernateCurrentHeader->processorFlags,
                                         src, compressedSize, 0, ppnum);
                if (stage != 2)
                pageSum = store_one_page(gIOHibernateCurrentHeader->processorFlags,
                                         src, compressedSize, 0, ppnum);
                if (stage != 2)


@@ -613,17 +620,19 @@ hibernate_kernel_entrypoint(uint32_t p1,
            }
            else
            {
            }
            else
            {

-               uint32_t   bufferPage;
+               uint32_t   bufferPage = 0;

                uint32_t * dst;
 
 //             debug_code(kIOHibernateRestoreCodeConflictPage,   ppnum);
 //             debug_code(kIOHibernateRestoreCodeConflictSource, (uintptr_t) src);
                uint32_t * dst;
 
 //             debug_code(kIOHibernateRestoreCodeConflictPage,   ppnum);
 //             debug_code(kIOHibernateRestoreCodeConflictSource, (uintptr_t) src);

-

                conflictCount++;
                conflictCount++;

-
-               // alloc new buffer page
-               bufferPage = hibernate_page_list_grab(map, &nextFree);
-
+               if (compressedSize)
+               {
+                   // alloc new buffer page
+                   bufferPage = hibernate_page_list_grab(map, &nextFree);
+                   dst = (uint32_t *)pal_hib_map(DEST_COPY_AREA, ptoa_64(bufferPage));
+                   memcpy(dst, src, compressedSize);
+               }

                if (copyPageIndex > ((PAGE_SIZE >> 2) - 3))
                {
                    // alloc new copy list page
                if (copyPageIndex > ((PAGE_SIZE >> 2) - 3))
                {
                    // alloc new copy list page


@@ -639,15 +648,10 @@ hibernate_kernel_entrypoint(uint32_t p1,
                    copyPageList[1] = 0;
                    copyPageIndex = 2;
                }
                    copyPageList[1] = 0;
                    copyPageIndex = 2;
                }

-

                copyPageList[copyPageIndex++] = ppnum;
                copyPageList[copyPageIndex++] = bufferPage;
                copyPageList[copyPageIndex++] = (compressedSize | (stage << 24));
                copyPageList[0] = copyPageIndex;
                copyPageList[copyPageIndex++] = ppnum;
                copyPageList[copyPageIndex++] = bufferPage;
                copyPageList[copyPageIndex++] = (compressedSize | (stage << 24));
                copyPageList[0] = copyPageIndex;

-
-               dst = (uint32_t *)pal_hib_map(DEST_COPY_AREA, ptoa_64(bufferPage));
-               for (idx = 0; idx < ((compressedSize + 3) >> 2); idx++)
-                       dst[idx] = src[idx];

            }
            srcPhys += ((compressedSize + 3) & ~3);
            src     += ((compressedSize + 3) >> 2);
            }
            srcPhys += ((compressedSize + 3) & ~3);
            src     += ((compressedSize + 3) >> 2);


@@ -691,6 +695,8 @@ hibernate_kernel_entrypoint(uint32_t p1,
 
     gIOHibernateState = kIOHibernateStateWakingFromHibernate;
 
 
     gIOHibernateState = kIOHibernateStateWakingFromHibernate;
 

+    gIOHibernateCurrentHeader->trampolineTime = (((rdtsc64() - timeStart)) >> 8);
+

 #if CONFIG_SLEEP
 #if defined(__i386__) || defined(__x86_64__)
     typedef void (*ResetProc)(void);
 #if CONFIG_SLEEP
 #if defined(__i386__) || defined(__x86_64__)
     typedef void (*ResetProc)(void);