iokit/Kernel/IOHibernateRestoreKernel.c

   1 /*
   2  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * The contents of this file constitute Original Code as defined in and
   7  * are subject to the Apple Public Source License Version 1.1 (the
   8  * "License").  You may not use this file except in compliance with the
   9  * License.  Please obtain a copy of the License at
  10  * http://www.apple.com/publicsource and read it before using this file.
  11  *
  12  * This Original Code and all software distributed under the License are
  13  * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  14  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  15  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
  17  * License for the specific language governing rights and limitations
  18  * under the License.
  19  *
  20  * @APPLE_LICENSE_HEADER_END@
  21  */
  22
  23 #include <stdint.h>
  24 #include <mach/mach_types.h>
  25 #include <mach/vm_param.h>
  26 #include <IOKit/IOHibernatePrivate.h>
  27 #include <IOKit/IOLib.h>
  28 #include <pexpert/boot.h>
  29 #include <crypto/aes.h>
  30
  31 #include "WKdm.h"
  32 #include "IOHibernateInternal.h"
  33
  34 /*
  35 This code is linked into the kernel but part of the "__HIB" section, which means
  36 its used by code running in the special context of restoring the kernel text and data
  37 from the hibernation image read by the booter. hibernate_kernel_entrypoint() and everything
  38 it calls or references needs to be careful to only touch memory also in the "__HIB" section.
  39 */
  40
  41 uint32_t gIOHibernateState;
  42
  43 static IOHibernateImageHeader _hibernateHeader;
  44 IOHibernateImageHeader * gIOHibernateCurrentHeader = &_hibernateHeader;
  45
  46 static hibernate_graphics_t _hibernateGraphics;
  47 hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
  48
  49 static hibernate_cryptwakevars_t _cryptWakeVars;
  50 hibernate_cryptwakevars_t * gIOHibernateCryptWakeVars = &_cryptWakeVars;
  51
  52 vm_offset_t gIOHibernateWakeMap;            // ppnum
  53 vm_size_t   gIOHibernateWakeMapSize;
  54
  55 #if __i386__
  56 extern void   acpi_wake_prot_entry(void);
  57 #endif
  58
  59 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  60
  61 #define BASE 65521L /* largest prime smaller than 65536 */
  62 #define NMAX 5000
  63 // NMAX (was 5521) the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1
  64
  65 #define DO1(buf,i)  {s1 += buf[i]; s2 += s1;}
  66 #define DO2(buf,i)  DO1(buf,i); DO1(buf,i+1);
  67 #define DO4(buf,i)  DO2(buf,i); DO2(buf,i+2);
  68 #define DO8(buf,i)  DO4(buf,i); DO4(buf,i+4);
  69 #define DO16(buf)   DO8(buf,0); DO8(buf,8);
  70
  71 uint32_t
  72 hibernate_sum(uint8_t *buf, int32_t len)
  73 {
  74     unsigned long s1 = 1; // adler & 0xffff;
  75     unsigned long s2 = 0; // (adler >> 16) & 0xffff;
  76     int k;
  77
  78     while (len > 0) {
  79         k = len < NMAX ? len : NMAX;
  80         len -= k;
  81         while (k >= 16) {
  82             DO16(buf);
  83             buf += 16;
  84             k -= 16;
  85         }
  86         if (k != 0) do {
  87             s1 += *buf++;
  88             s2 += s1;
  89         } while (--k);
  90         s1 %= BASE;
  91         s2 %= BASE;
  92     }
  93     return (s2 << 16) | s1;
  94 }
  95
  96 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  97
  98 static hibernate_bitmap_t *
  99 hibernate_page_bitmap(hibernate_page_list_t * list, uint32_t page)
 100 {
 101     uint32_t             bank;
 102     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 103
 104     for (bank = 0; bank < list->bank_count; bank++)
 105     {
 106         if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
 107             break;
 108         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 109     }
 110     if (bank == list->bank_count)
 111         bitmap = 0;
 112
 113     return (bitmap);
 114 }
 115
 116 hibernate_bitmap_t *
 117 hibernate_page_bitmap_pin(hibernate_page_list_t * list, uint32_t * pPage)
 118 {
 119     uint32_t             bank, page = *pPage;
 120     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 121
 122     for (bank = 0; bank < list->bank_count; bank++)
 123     {
 124         if (page <= bitmap->first_page)
 125         {
 126             *pPage = bitmap->first_page;
 127             break;
 128         }
 129         if (page <= bitmap->last_page)
 130             break;
 131         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 132     }
 133     if (bank == list->bank_count)
 134         bitmap = 0;
 135
 136     return (bitmap);
 137 }
 138
 139 void
 140 hibernate_page_bitset(hibernate_page_list_t * list, boolean_t set, uint32_t page)
 141 {
 142     hibernate_bitmap_t * bitmap;
 143
 144     bitmap = hibernate_page_bitmap(list, page);
 145     if (bitmap)
 146     {
 147         page -= bitmap->first_page;
 148         if (set)
 149             bitmap->bitmap[page >> 5] |= (0x80000000 >> (page & 31));
 150             //setbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
 151         else
 152             bitmap->bitmap[page >> 5] &= ~(0x80000000 >> (page & 31));
 153             //clrbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
 154     }
 155 }
 156
 157 boolean_t
 158 hibernate_page_bittst(hibernate_page_list_t * list, uint32_t page)
 159 {
 160     boolean_t            result = TRUE;
 161     hibernate_bitmap_t * bitmap;
 162
 163     bitmap = hibernate_page_bitmap(list, page);
 164     if (bitmap)
 165     {
 166         page -= bitmap->first_page;
 167         result = (0 != (bitmap->bitmap[page >> 5] & (0x80000000 >> (page & 31))));
 168     }
 169     return (result);
 170 }
 171
 172 // count bits clear or set (set == TRUE) starting at page.
 173 uint32_t
 174 hibernate_page_bitmap_count(hibernate_bitmap_t * bitmap, uint32_t set, uint32_t page)
 175 {
 176     uint32_t index, bit, bits;
 177     uint32_t count;
 178
 179     count = 0;
 180
 181     index = (page - bitmap->first_page) >> 5;
 182     bit = (page - bitmap->first_page) & 31;
 183
 184     bits = bitmap->bitmap[index];
 185     if (set)
 186         bits = ~bits;
 187     bits = (bits << bit);
 188     if (bits)
 189         count += __builtin_clz(bits);
 190     else
 191     {
 192         count += 32 - bit;
 193         while (++index < bitmap->bitmapwords)
 194         {
 195             bits = bitmap->bitmap[index];
 196             if (set)
 197                 bits = ~bits;
 198             if (bits)
 199             {
 200                 count += __builtin_clz(bits);
 201                 break;
 202             }
 203             count += 32;
 204         }
 205     }
 206
 207     return (count);
 208 }
 209
 210 static vm_offset_t
 211 hibernate_page_list_grab(hibernate_page_list_t * list, uint32_t * pNextFree)
 212 {
 213     uint32_t             nextFree = *pNextFree;
 214     uint32_t             nextFreeInBank;
 215     hibernate_bitmap_t * bitmap;
 216
 217     nextFreeInBank = nextFree + 1;
 218     while ((bitmap = hibernate_page_bitmap_pin(list, &nextFreeInBank)))
 219     {
 220         nextFreeInBank += hibernate_page_bitmap_count(bitmap, FALSE, nextFreeInBank);
 221         if (nextFreeInBank <= bitmap->last_page)
 222         {
 223             *pNextFree = nextFreeInBank;
 224             break;
 225         }
 226     }
 227
 228     if (!bitmap)
 229         IOPanic(__FUNCTION__);
 230
 231     return (nextFree);
 232 }
 233
 234 static uint32_t
 235 store_one_page(uint32_t procFlags, uint32_t * src, uint32_t compressedSize,
 236                 uint32_t * buffer, uint32_t ppnum)
 237 {
 238     uint64_t dst;
 239     uint32_t sum;
 240
 241     dst = ptoa_64(ppnum);
 242     if (ppnum < 0x00100000)
 243         buffer = (uint32_t *) (uint32_t) dst;
 244
 245     if (compressedSize != PAGE_SIZE)
 246     {
 247         WKdm_decompress((WK_word*) src, (WK_word*) buffer, PAGE_SIZE >> 2);
 248         src = buffer;
 249     }
 250
 251     sum = hibernate_sum((uint8_t *) src, PAGE_SIZE);
 252
 253     if (((uint64_t) (uint32_t) src) == dst)
 254         src = 0;
 255
 256     hibernate_restore_phys_page((uint64_t) (uint32_t) src, dst, PAGE_SIZE, procFlags);
 257
 258     return (sum);
 259 }
 260
 261 static void
 262 bcopy_internal(const void *src, void *dst, uint32_t len)
 263 {
 264     const char *s = src;
 265     char       *d = dst;
 266     uint32_t   idx = 0;
 267
 268     while (idx < len)
 269     {
 270         d[idx] = s[idx];
 271         idx++;
 272     }
 273 }
 274
 275 #define C_ASSERT(e) typedef char    __C_ASSERT__[(e) ? 1 : -1]
 276
 277 long
 278 hibernate_kernel_entrypoint(IOHibernateImageHeader * header,
 279                             void * p2, void * p3, void * p4)
 280 {
 281     typedef void (*ResetProc)(void);
 282     uint32_t idx;
 283     uint32_t * src;
 284     uint32_t * buffer;
 285     uint32_t * pageIndexSource;
 286     hibernate_page_list_t * map;
 287     uint32_t count;
 288     uint32_t ppnum;
 289     uint32_t page;
 290     uint32_t conflictCount;
 291     uint32_t compressedSize;
 292     uint32_t uncompressedPages;
 293     uint32_t copyPageListHead;
 294     uint32_t * copyPageList;
 295     uint32_t copyPageIndex;
 296     uint32_t sum;
 297     uint32_t nextFree;
 298     uint32_t lastImagePage;
 299     uint32_t lastMapPage;
 300     uint32_t lastPageIndexPage;
 301
 302     C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
 303
 304     bcopy_internal(header,
 305                 gIOHibernateCurrentHeader,
 306                 sizeof(IOHibernateImageHeader));
 307
 308     if (!p2)
 309     {
 310         count = header->graphicsInfoOffset;
 311         if (count)
 312             p2 = (void *)(((uintptr_t) header) - count);
 313     }
 314     if (p2)
 315         bcopy_internal(p2,
 316                 gIOHibernateGraphicsInfo,
 317                 sizeof(hibernate_graphics_t));
 318     else
 319         gIOHibernateGraphicsInfo->physicalAddress = gIOHibernateGraphicsInfo->depth = 0;
 320
 321     if (!p3)
 322     {
 323         count = header->cryptVarsOffset;
 324         if (count)
 325             p3 = (void *)(((uintptr_t) header) - count);
 326     }
 327     if (p3)
 328         bcopy_internal(p3,
 329                 gIOHibernateCryptWakeVars,
 330                 sizeof(hibernate_cryptvars_t));
 331
 332     src = (uint32_t *)
 333                 (((uint32_t) &header->fileExtentMap[0])
 334                             + header->fileExtentMapSize
 335                             + ptoa_32(header->restore1PageCount));
 336
 337     if (header->previewSize)
 338     {
 339         pageIndexSource = src;
 340         map = (hibernate_page_list_t *)(((uint32_t) pageIndexSource) + header->previewSize);
 341         src = (uint32_t *) (((uint32_t) pageIndexSource) + header->previewPageListSize);
 342     }
 343     else
 344     {
 345         pageIndexSource = 0;
 346         map = (hibernate_page_list_t *) src;
 347         src = (uint32_t *) (((uint32_t) map) + header->bitmapSize);
 348     }
 349
 350     lastPageIndexPage = atop_32(src);
 351
 352     lastImagePage = atop_32(((uint32_t) header) + header->image1Size);
 353
 354     lastMapPage = atop_32(((uint32_t) map) + header->bitmapSize);
 355
 356     // knock all the image pages to be used out of free map
 357     for (ppnum = atop_32(header); ppnum <= lastImagePage; ppnum++)
 358     {
 359         hibernate_page_bitset(map, FALSE, ppnum);
 360     }
 361
 362     nextFree = 0;
 363     hibernate_page_list_grab(map, &nextFree);
 364     buffer = (uint32_t *) ptoa_32(hibernate_page_list_grab(map, &nextFree));
 365
 366     if (header->memoryMapSize && (count = header->memoryMapOffset))
 367     {
 368         p4 = (void *)(((uintptr_t) header) - count);
 369         gIOHibernateWakeMap     = hibernate_page_list_grab(map, &nextFree);
 370         gIOHibernateWakeMapSize = header->memoryMapSize;
 371         bcopy_internal(p4, (void  *) ptoa_32(gIOHibernateWakeMap), gIOHibernateWakeMapSize);
 372     }
 373     else
 374         gIOHibernateWakeMapSize = 0;
 375
 376     sum = gIOHibernateCurrentHeader->actualRestore1Sum;
 377     gIOHibernateCurrentHeader->diag[0] = (uint32_t) header;
 378     gIOHibernateCurrentHeader->diag[1] = sum;
 379
 380     uncompressedPages = 0;
 381     conflictCount     = 0;
 382     copyPageListHead  = 0;
 383     copyPageList      = 0;
 384     copyPageIndex     = PAGE_SIZE >> 2;
 385
 386     compressedSize    = PAGE_SIZE;
 387
 388     while (1)
 389     {
 390         if (pageIndexSource)
 391         {
 392             ppnum = pageIndexSource[0];
 393             count = pageIndexSource[1];
 394             pageIndexSource += 2;
 395             if (!count)
 396             {
 397                 pageIndexSource = 0;
 398                 src =  (uint32_t *) (((uint32_t) map) + gIOHibernateCurrentHeader->bitmapSize);
 399                 ppnum = src[0];
 400                 count = src[1];
 401                 src += 2;
 402             }
 403         }
 404         else
 405         {
 406             ppnum = src[0];
 407             count = src[1];
 408             if (!count)
 409                 break;
 410             src += 2;
 411         }
 412
 413         for (page = 0; page < count; page++, ppnum++)
 414         {
 415             uint32_t tag;
 416             int conflicts;
 417
 418             if (!pageIndexSource)
 419             {
 420                 tag = *src++;
 421                 compressedSize = kIOHibernateTagLength & tag;
 422             }
 423
 424 //    SINT(ppnum);
 425
 426             conflicts = (((ppnum >= atop_32(map)) && (ppnum <= lastMapPage))
 427                       || ((ppnum >= atop_32(src)) && (ppnum <= lastImagePage)));
 428
 429             if (pageIndexSource)
 430                 conflicts |= ((ppnum >= atop_32(pageIndexSource)) && (ppnum <= lastPageIndexPage));
 431
 432             if (!conflicts)
 433             {
 434                 if (compressedSize)
 435                     sum += store_one_page(gIOHibernateCurrentHeader->processorFlags,
 436                                             src, compressedSize, buffer, ppnum);
 437                 uncompressedPages++;
 438             }
 439             else
 440             {
 441                 uint32_t   bufferPage;
 442                 uint32_t * dst;
 443
 444                 conflictCount++;
 445
 446                 // alloc new buffer page
 447                 bufferPage = hibernate_page_list_grab(map, &nextFree);
 448
 449                 if (copyPageIndex > ((PAGE_SIZE >> 2) - 3))
 450                 {
 451                     // alloc new copy list page
 452                     uint32_t pageListPage = hibernate_page_list_grab(map, &nextFree);
 453                     // link to current
 454                     if (copyPageList)
 455                         copyPageList[1] = pageListPage;
 456                     else
 457                         copyPageListHead = pageListPage;
 458                     copyPageList = (uint32_t *) ptoa_32(pageListPage);
 459                     copyPageList[1] = 0;
 460                     copyPageIndex = 2;
 461                 }
 462
 463                 copyPageList[copyPageIndex++] = ppnum;
 464                 copyPageList[copyPageIndex++] = bufferPage;
 465                 copyPageList[copyPageIndex++] = compressedSize;
 466                 copyPageList[0] = copyPageIndex;
 467
 468                 dst = (uint32_t *) ptoa_32(bufferPage);
 469                 for (idx = 0; idx < ((compressedSize + 3) >> 2); idx++)
 470                     dst[idx] = src[idx];
 471             }
 472             src += ((compressedSize + 3) >> 2);
 473         }
 474     }
 475
 476     // -- copy back conflicts
 477
 478     copyPageList = (uint32_t *) ptoa_32(copyPageListHead);
 479     while (copyPageList)
 480     {
 481         for (copyPageIndex = 2; copyPageIndex < copyPageList[0]; copyPageIndex += 3)
 482         {
 483             ppnum          =              copyPageList[copyPageIndex + 0];
 484             src            = (uint32_t *) ptoa_32(copyPageList[copyPageIndex + 1]);
 485             compressedSize =              copyPageList[copyPageIndex + 2];
 486
 487             sum += store_one_page(gIOHibernateCurrentHeader->processorFlags,
 488                                     src, compressedSize, buffer, ppnum);
 489             uncompressedPages++;
 490         }
 491         copyPageList = (uint32_t *) ptoa_32(copyPageList[1]);
 492     }
 493
 494     // -- image has been destroyed...
 495
 496     gIOHibernateCurrentHeader->actualImage1Sum         = sum;
 497     gIOHibernateCurrentHeader->actualUncompressedPages = uncompressedPages;
 498     gIOHibernateCurrentHeader->conflictCount           = conflictCount;
 499     gIOHibernateCurrentHeader->nextFree                = nextFree;
 500
 501     gIOHibernateState = kIOHibernateStateWakingFromHibernate;
 502
 503 #if __ppc__
 504     ResetProc proc;
 505     proc = (ResetProc) 0x100;
 506     __asm__ volatile("ori 0, 0, 0" : : );
 507     proc();
 508 #elif __i386__
 509     ResetProc proc;
 510     proc = (ResetProc) acpi_wake_prot_entry;
 511     // flush caches
 512     __asm__("wbinvd");
 513     proc();
 514 #endif
 515
 516     return -1;
 517 }
 518