xnu-792.10.96.tar.gz

[apple/xnu.git] / iokit / Kernel / IOHibernateRestoreKernel.c

diff --git a/iokit/Kernel/IOHibernateRestoreKernel.c b/iokit/Kernel/IOHibernateRestoreKernel.c
index 61adce7b78986635b30222b35425ec6baa6f9a2e..a614211901c9006d81f2022e25c6ac243ea58bad 100644 (file)
--- a/iokit/Kernel/IOHibernateRestoreKernel.c
+++ b/iokit/Kernel/IOHibernateRestoreKernel.c
@@ -3,20 +3,19 @@
  *
  * @APPLE_LICENSE_HEADER_START@
  * 
  *
  * @APPLE_LICENSE_HEADER_START@
  * 

- * This file contains Original Code and/or Modifications of Original Code
- * as defined in and that are subject to the Apple Public Source License
- * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this
- * file.
+ * The contents of this file constitute Original Code as defined in and
+ * are subject to the Apple Public Source License Version 1.1 (the
+ * "License").  You may not use this file except in compliance with the
+ * License.  Please obtain a copy of the License at
+ * http://www.apple.com/publicsource and read it before using this file.

  * 
  * 

- * The Original Code and all software distributed under the License are
- * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * This Original Code and all software distributed under the License are
+ * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER

  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,

- * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
- * Please see the License for the specific language governing rights and
- * limitations under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
+ * License for the specific language governing rights and limitations
+ * under the License.

  * 
  * @APPLE_LICENSE_HEADER_END@
  */
  * 
  * @APPLE_LICENSE_HEADER_END@
  */


@@ -25,6 +24,7 @@
 #include <mach/mach_types.h>
 #include <mach/vm_param.h>
 #include <IOKit/IOHibernatePrivate.h>
 #include <mach/mach_types.h>
 #include <mach/vm_param.h>
 #include <IOKit/IOHibernatePrivate.h>

+#include <IOKit/IOLib.h>

 #include <pexpert/boot.h>
 #include <crypto/aes.h>
 
 #include <pexpert/boot.h>
 #include <crypto/aes.h>
 


@@ -49,11 +49,13 @@ hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
 static hibernate_cryptwakevars_t _cryptWakeVars;
 hibernate_cryptwakevars_t * gIOHibernateCryptWakeVars = &_cryptWakeVars;
 
 static hibernate_cryptwakevars_t _cryptWakeVars;
 hibernate_cryptwakevars_t * gIOHibernateCryptWakeVars = &_cryptWakeVars;
 

+vm_offset_t gIOHibernateWakeMap;           // ppnum
+vm_size_t   gIOHibernateWakeMapSize;
+

 #if __i386__
 extern void   acpi_wake_prot_entry(void);
 #endif
 
 #if __i386__
 extern void   acpi_wake_prot_entry(void);
 #endif
 

-

 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 #define BASE 65521L /* largest prime smaller than 65536 */
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 #define BASE 65521L /* largest prime smaller than 65536 */


@@ -93,29 +95,8 @@ hibernate_sum(uint8_t *buf, int32_t len)
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 

-#if __ppc__
-static __inline__ unsigned int cntlzw(unsigned int num)
-{
-  unsigned int result;
-  __asm__ volatile("cntlzw %0, %1" : "=r" (result) : "r" (num));
-  return result;
-}
-#elif __i386__
-static __inline__ unsigned int cntlzw(unsigned int num)
-{
-    unsigned int result;
-    __asm__ volatile(  "bsrl   %1, %0\n\t"
-                       "cmovel %2, %0"
-                     : "=r" (result)
-                     : "rm" (num), "r" (63));
-    return 31 ^ result;
-}
-#else
-#error arch
-#endif
-
-void 
-hibernate_page_bitset(hibernate_page_list_t * list, boolean_t set, uint32_t page)
+static hibernate_bitmap_t *
+hibernate_page_bitmap(hibernate_page_list_t * list, uint32_t page)

 {
     uint32_t             bank;
     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 {
     uint32_t             bank;
     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];


@@ -123,117 +104,129 @@ hibernate_page_bitset(hibernate_page_list_t * list, boolean_t set, uint32_t page
     for (bank = 0; bank < list->bank_count; bank++)
     {
        if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
     for (bank = 0; bank < list->bank_count; bank++)
     {
        if ((page >= bitmap->first_page) && (page <= bitmap->last_page))

-       {
-           page -= bitmap->first_page;
-           if (set)
-               bitmap->bitmap[page >> 5] |= (0x80000000 >> (page & 31));
-               //setbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
-           else
-               bitmap->bitmap[page >> 5] &= ~(0x80000000 >> (page & 31));
-               //clrbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);

            break;
            break;

-       }

        bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
     }
        bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
     }

+    if (bank == list->bank_count)
+       bitmap = 0;
+       
+    return (bitmap);

 }
 
 }
 

-boolean_t 
-hibernate_page_bittst(hibernate_page_list_t * list, uint32_t page)
+hibernate_bitmap_t *
+hibernate_page_bitmap_pin(hibernate_page_list_t * list, uint32_t * pPage)

 {
 {

-    boolean_t           result = TRUE;
-    uint32_t             bank;
+    uint32_t             bank, page = *pPage;

     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 
     for (bank = 0; bank < list->bank_count; bank++)
     {
     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 
     for (bank = 0; bank < list->bank_count; bank++)
     {

-       if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
+       if (page <= bitmap->first_page)

        {
        {

-           page -= bitmap->first_page;
-            result = (0 != (bitmap->bitmap[page >> 5] & (0x80000000 >> (page & 31))));
+           *pPage = bitmap->first_page;

            break;
        }
            break;
        }

+       if (page <= bitmap->last_page)
+           break;

        bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
     }
        bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
     }

+    if (bank == list->bank_count)
+       bitmap = 0;
+       
+    return (bitmap);
+}
+
+void 
+hibernate_page_bitset(hibernate_page_list_t * list, boolean_t set, uint32_t page)
+{
+    hibernate_bitmap_t * bitmap;
+
+    bitmap = hibernate_page_bitmap(list, page);
+    if (bitmap)
+    {
+       page -= bitmap->first_page;
+       if (set)
+           bitmap->bitmap[page >> 5] |= (0x80000000 >> (page & 31));
+           //setbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
+       else
+           bitmap->bitmap[page >> 5] &= ~(0x80000000 >> (page & 31));
+           //clrbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
+    }
+}
+
+boolean_t 
+hibernate_page_bittst(hibernate_page_list_t * list, uint32_t page)
+{
+    boolean_t           result = TRUE;
+    hibernate_bitmap_t * bitmap;
+
+    bitmap = hibernate_page_bitmap(list, page);
+    if (bitmap)
+    {
+       page -= bitmap->first_page;
+       result = (0 != (bitmap->bitmap[page >> 5] & (0x80000000 >> (page & 31))));
+    }

     return (result);
 }
 
     return (result);
 }
 

-// count bits clear or set (set == TRUE) starting at index page.
+// count bits clear or set (set == TRUE) starting at page.

 uint32_t
 uint32_t

-hibernate_page_list_count(hibernate_page_list_t * list, uint32_t set, uint32_t page)
+hibernate_page_bitmap_count(hibernate_bitmap_t * bitmap, uint32_t set, uint32_t page)

 {
 {

-    uint32_t                   bank, count;
-    hibernate_bitmap_t *       bitmap;
+    uint32_t index, bit, bits;
+    uint32_t count;

 
 

-    bitmap = &list->bank_bitmap[0];
-    count  = 0;
+    count = 0;

 
 

-    for (bank = 0; bank < list->bank_count; bank++)
+    index = (page - bitmap->first_page) >> 5;
+    bit = (page - bitmap->first_page) & 31;
+
+    bits = bitmap->bitmap[index];
+    if (set)
+       bits = ~bits;
+    bits = (bits << bit);
+    if (bits)
+       count += __builtin_clz(bits);
+    else

     {
     {

-       // bits between banks are "set"
-       if (set && (page < bitmap->first_page))
+       count += 32 - bit;
+       while (++index < bitmap->bitmapwords)

        {
        {

-           count += bitmap->first_page - page;
-           page  = bitmap->first_page;
-       }
-       if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
-       {
-           uint32_t index, bit, bits;
-       
-           index = (page - bitmap->first_page) >> 5;
-           bit = (page - bitmap->first_page) & 31;
-       
-           while (TRUE)
+           bits = bitmap->bitmap[index];
+           if (set)
+               bits = ~bits;
+           if (bits)

            {
            {

-               bits = bitmap->bitmap[index];
-               if (set)
-                   bits = ~bits;
-               bits = (bits << bit);
-               count += cntlzw(bits);
-               if (bits)
-                   break;
-               count -= bit;
-           
-               while (++index < bitmap->bitmapwords)
-               {
-                   bits = bitmap->bitmap[index];
-                   if (set)
-                       bits = ~bits;
-                   count += cntlzw(bits);
-                   if (bits)
-                       break;
-               }
-               if (bits)
-                   break;
-               if (!set)
-                   break;
-               // bits between banks are "set"
-               bank++;
-               if (bank >= list->bank_count)
-                   break;
-               count -= (bitmap->last_page + 1);
-               bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
-               count += bitmap->first_page;
-               index = 0;
-               bit = 0;                            
+               count += __builtin_clz(bits);
+               break;

            }
            }

-           break;
+           count += 32;

        }
        }

-       bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];

     }
 
     return (count);
 }
 
     }
 
     return (count);
 }
 

-
-static uint32_t
-hibernate_page_list_grab(hibernate_page_list_t * map, uint32_t * _nextFree)
+static vm_offset_t
+hibernate_page_list_grab(hibernate_page_list_t * list, uint32_t * pNextFree)

 {
 {

-    uint32_t nextFree = *_nextFree;
+    uint32_t            nextFree = *pNextFree;
+    uint32_t            nextFreeInBank;
+    hibernate_bitmap_t * bitmap;

 
 

-    if (!nextFree)
-       nextFree = hibernate_page_list_count(map, 0, 0);
+    nextFreeInBank = nextFree + 1;
+    while ((bitmap = hibernate_page_bitmap_pin(list, &nextFreeInBank)))
+    {
+       nextFreeInBank += hibernate_page_bitmap_count(bitmap, FALSE, nextFreeInBank);
+       if (nextFreeInBank <= bitmap->last_page)
+       {
+           *pNextFree = nextFreeInBank;
+           break;
+       }
+    }

 
 

-    *_nextFree = nextFree + 1 + hibernate_page_list_count(map, 0, nextFree + 1);
+    if (!bitmap)
+       IOPanic(__FUNCTION__);

 
     return (nextFree);
 }
 
     return (nextFree);
 }


@@ -246,14 +239,8 @@ store_one_page(uint32_t procFlags, uint32_t * src, uint32_t compressedSize,
     uint32_t sum;
 
     dst = ptoa_64(ppnum);
     uint32_t sum;
 
     dst = ptoa_64(ppnum);

-#if __ppc__

     if (ppnum < 0x00100000)
        buffer = (uint32_t *) (uint32_t) dst;
     if (ppnum < 0x00100000)
        buffer = (uint32_t *) (uint32_t) dst;

-#elif __i386__
-    if (ppnum < atop_32(0xC0000000)) {
-        buffer = (uint32_t *) (uint32_t) dst;
-    }
-#endif

 
     if (compressedSize != PAGE_SIZE)
     {
 
     if (compressedSize != PAGE_SIZE)
     {


@@ -285,9 +272,11 @@ bcopy_internal(const void *src, void *dst, uint32_t len)
     }
 }
 
     }
 }
 

+#define C_ASSERT(e) typedef char    __C_ASSERT__[(e) ? 1 : -1]
+

 long 
 hibernate_kernel_entrypoint(IOHibernateImageHeader * header, 
 long 
 hibernate_kernel_entrypoint(IOHibernateImageHeader * header, 

-                            void * p2, void * p3, __unused void * p4)
+                            void * p2, void * p3, void * p4)

 {
     typedef void (*ResetProc)(void);
     uint32_t idx;
 {
     typedef void (*ResetProc)(void);
     uint32_t idx;


@@ -310,11 +299,18 @@ hibernate_kernel_entrypoint(IOHibernateImageHeader * header,
     uint32_t lastMapPage;
     uint32_t lastPageIndexPage;
 
     uint32_t lastMapPage;
     uint32_t lastPageIndexPage;
 

+    C_ASSERT(sizeof(IOHibernateImageHeader) == 512);

 
     bcopy_internal(header, 
                 gIOHibernateCurrentHeader, 
                 sizeof(IOHibernateImageHeader));
 
 
     bcopy_internal(header, 
                 gIOHibernateCurrentHeader, 
                 sizeof(IOHibernateImageHeader));
 

+    if (!p2)
+    {
+       count = header->graphicsInfoOffset;
+       if (count)
+           p2 = (void *)(((uintptr_t) header) - count);
+    }

     if (p2) 
         bcopy_internal(p2, 
                 gIOHibernateGraphicsInfo, 
     if (p2) 
         bcopy_internal(p2, 
                 gIOHibernateGraphicsInfo, 


@@ -322,6 +318,12 @@ hibernate_kernel_entrypoint(IOHibernateImageHeader * header,
     else
         gIOHibernateGraphicsInfo->physicalAddress = gIOHibernateGraphicsInfo->depth = 0;
 
     else
         gIOHibernateGraphicsInfo->physicalAddress = gIOHibernateGraphicsInfo->depth = 0;
 

+    if (!p3)
+    {
+       count = header->cryptVarsOffset;
+       if (count)
+           p3 = (void *)(((uintptr_t) header) - count);
+    }

     if (p3)
         bcopy_internal(p3, 
                 gIOHibernateCryptWakeVars, 
     if (p3)
         bcopy_internal(p3, 
                 gIOHibernateCryptWakeVars, 


@@ -358,8 +360,19 @@ hibernate_kernel_entrypoint(IOHibernateImageHeader * header,
     }
 
     nextFree = 0;
     }
 
     nextFree = 0;

+    hibernate_page_list_grab(map, &nextFree);

     buffer = (uint32_t *) ptoa_32(hibernate_page_list_grab(map, &nextFree));
 
     buffer = (uint32_t *) ptoa_32(hibernate_page_list_grab(map, &nextFree));
 

+    if (header->memoryMapSize && (count = header->memoryMapOffset))
+    {
+       p4 = (void *)(((uintptr_t) header) - count);
+       gIOHibernateWakeMap     = hibernate_page_list_grab(map, &nextFree);
+       gIOHibernateWakeMapSize = header->memoryMapSize;
+       bcopy_internal(p4, (void  *) ptoa_32(gIOHibernateWakeMap), gIOHibernateWakeMapSize);
+    }
+    else
+       gIOHibernateWakeMapSize = 0;
+

     sum = gIOHibernateCurrentHeader->actualRestore1Sum;
     gIOHibernateCurrentHeader->diag[0] = (uint32_t) header;
     gIOHibernateCurrentHeader->diag[1] = sum;
     sum = gIOHibernateCurrentHeader->actualRestore1Sum;
     gIOHibernateCurrentHeader->diag[0] = (uint32_t) header;
     gIOHibernateCurrentHeader->diag[1] = sum;


@@ -408,6 +421,8 @@ hibernate_kernel_entrypoint(IOHibernateImageHeader * header,
                 compressedSize = kIOHibernateTagLength & tag;
             }
 
                 compressedSize = kIOHibernateTagLength & tag;
             }
 

+//    SINT(ppnum);
+

            conflicts = (((ppnum >= atop_32(map)) && (ppnum <= lastMapPage))
                      || ((ppnum >= atop_32(src)) && (ppnum <= lastImagePage)));
 
            conflicts = (((ppnum >= atop_32(map)) && (ppnum <= lastMapPage))
                      || ((ppnum >= atop_32(src)) && (ppnum <= lastImagePage)));
 


@@ -493,7 +508,8 @@ hibernate_kernel_entrypoint(IOHibernateImageHeader * header,
 #elif __i386__
     ResetProc proc;
     proc = (ResetProc) acpi_wake_prot_entry;
 #elif __i386__
     ResetProc proc;
     proc = (ResetProc) acpi_wake_prot_entry;

-
+    // flush caches
+    __asm__("wbinvd");

     proc();
 #endif
   
     proc();
 #endif