iokit/Kernel/IOHibernateRestoreKernel.c

   1 /*
   2  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License.  The rights granted to you under the
  10  * License may not be used to create, or enable the creation or
  11  * redistribution of, unlawful or unlicensed copies of an Apple operating
  12  * system, or to circumvent, violate, or enable the circumvention or
  13  * violation of, any terms of an Apple operating system software license
  14  * agreement.
  15  *
  16  * Please obtain a copy of the License at
  17  * http://www.opensource.apple.com/apsl/ and read it before using this
  18  * file.
  19  *
  20  * The Original Code and all software distributed under the License are
  21  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  22  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  23  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  24  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  25  * Please see the License for the specific language governing rights and
  26  * limitations under the License.
  27  *
  28  * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
  29  */
  30
  31 #include <stdint.h>
  32 #include <mach/mach_types.h>
  33 #include <mach/vm_param.h>
  34 #include <IOKit/IOHibernatePrivate.h>
  35 #include <pexpert/boot.h>
  36 #include <crypto/aes.h>
  37
  38 #include "WKdm.h"
  39 #include "IOHibernateInternal.h"
  40
  41 /*
  42 This code is linked into the kernel but part of the "__HIB" section, which means
  43 its used by code running in the special context of restoring the kernel text and data
  44 from the hibernation image read by the booter. hibernate_kernel_entrypoint() and everything
  45 it calls or references needs to be careful to only touch memory also in the "__HIB" section.
  46 */
  47
  48 uint32_t gIOHibernateState;
  49
  50 static IOHibernateImageHeader _hibernateHeader;
  51 IOHibernateImageHeader * gIOHibernateCurrentHeader = &_hibernateHeader;
  52
  53 static hibernate_graphics_t _hibernateGraphics;
  54 hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
  55
  56 static hibernate_cryptwakevars_t _cryptWakeVars;
  57 hibernate_cryptwakevars_t * gIOHibernateCryptWakeVars = &_cryptWakeVars;
  58
  59 #if __i386__
  60 extern void   acpi_wake_prot_entry(void);
  61 #endif
  62
  63
  64 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  65
  66 #define BASE 65521L /* largest prime smaller than 65536 */
  67 #define NMAX 5000
  68 // NMAX (was 5521) the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1
  69
  70 #define DO1(buf,i)  {s1 += buf[i]; s2 += s1;}
  71 #define DO2(buf,i)  DO1(buf,i); DO1(buf,i+1);
  72 #define DO4(buf,i)  DO2(buf,i); DO2(buf,i+2);
  73 #define DO8(buf,i)  DO4(buf,i); DO4(buf,i+4);
  74 #define DO16(buf)   DO8(buf,0); DO8(buf,8);
  75
  76 uint32_t
  77 hibernate_sum(uint8_t *buf, int32_t len)
  78 {
  79     unsigned long s1 = 1; // adler & 0xffff;
  80     unsigned long s2 = 0; // (adler >> 16) & 0xffff;
  81     int k;
  82
  83     while (len > 0) {
  84         k = len < NMAX ? len : NMAX;
  85         len -= k;
  86         while (k >= 16) {
  87             DO16(buf);
  88             buf += 16;
  89             k -= 16;
  90         }
  91         if (k != 0) do {
  92             s1 += *buf++;
  93             s2 += s1;
  94         } while (--k);
  95         s1 %= BASE;
  96         s2 %= BASE;
  97     }
  98     return (s2 << 16) | s1;
  99 }
 100
 101 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 102
 103 #if __ppc__
 104 static __inline__ unsigned int cntlzw(unsigned int num)
 105 {
 106   unsigned int result;
 107   __asm__ volatile("cntlzw %0, %1" : "=r" (result) : "r" (num));
 108   return result;
 109 }
 110 #elif __i386__
 111 static __inline__ unsigned int cntlzw(unsigned int num)
 112 {
 113     unsigned int result;
 114     __asm__ volatile(   "bsrl   %1, %0\n\t"
 115                         "cmovel %2, %0"
 116                      : "=r" (result)
 117                      : "rm" (num), "r" (63));
 118     return 31 ^ result;
 119 }
 120 #else
 121 #error arch
 122 #endif
 123
 124 void
 125 hibernate_page_bitset(hibernate_page_list_t * list, boolean_t set, uint32_t page)
 126 {
 127     uint32_t             bank;
 128     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 129
 130     for (bank = 0; bank < list->bank_count; bank++)
 131     {
 132         if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
 133         {
 134             page -= bitmap->first_page;
 135             if (set)
 136                 bitmap->bitmap[page >> 5] |= (0x80000000 >> (page & 31));
 137                 //setbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
 138             else
 139                 bitmap->bitmap[page >> 5] &= ~(0x80000000 >> (page & 31));
 140                 //clrbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
 141             break;
 142         }
 143         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 144     }
 145 }
 146
 147 boolean_t
 148 hibernate_page_bittst(hibernate_page_list_t * list, uint32_t page)
 149 {
 150     boolean_t            result = TRUE;
 151     uint32_t             bank;
 152     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 153
 154     for (bank = 0; bank < list->bank_count; bank++)
 155     {
 156         if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
 157         {
 158             page -= bitmap->first_page;
 159             result = (0 != (bitmap->bitmap[page >> 5] & (0x80000000 >> (page & 31))));
 160             break;
 161         }
 162         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 163     }
 164     return (result);
 165 }
 166
 167 // count bits clear or set (set == TRUE) starting at index page.
 168 uint32_t
 169 hibernate_page_list_count(hibernate_page_list_t * list, uint32_t set, uint32_t page)
 170 {
 171     uint32_t                    bank, count;
 172     hibernate_bitmap_t *        bitmap;
 173
 174     bitmap = &list->bank_bitmap[0];
 175     count  = 0;
 176
 177     for (bank = 0; bank < list->bank_count; bank++)
 178     {
 179         // bits between banks are "set"
 180         if (set && (page < bitmap->first_page))
 181         {
 182             count += bitmap->first_page - page;
 183             page  = bitmap->first_page;
 184         }
 185         if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
 186         {
 187             uint32_t index, bit, bits;
 188
 189             index = (page - bitmap->first_page) >> 5;
 190             bit = (page - bitmap->first_page) & 31;
 191
 192             while (TRUE)
 193             {
 194                 bits = bitmap->bitmap[index];
 195                 if (set)
 196                     bits = ~bits;
 197                 bits = (bits << bit);
 198                 count += cntlzw(bits);
 199                 if (bits)
 200                     break;
 201                 count -= bit;
 202
 203                 while (++index < bitmap->bitmapwords)
 204                 {
 205                     bits = bitmap->bitmap[index];
 206                     if (set)
 207                         bits = ~bits;
 208                     count += cntlzw(bits);
 209                     if (bits)
 210                         break;
 211                 }
 212                 if (bits)
 213                     break;
 214                 if (!set)
 215                     break;
 216                 // bits between banks are "set"
 217                 bank++;
 218                 if (bank >= list->bank_count)
 219                     break;
 220                 count -= (bitmap->last_page + 1);
 221                 bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 222                 count += bitmap->first_page;
 223                 index = 0;
 224                 bit = 0;
 225             }
 226             break;
 227         }
 228         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 229     }
 230
 231     return (count);
 232 }
 233
 234
 235 static uint32_t
 236 hibernate_page_list_grab(hibernate_page_list_t * map, uint32_t * _nextFree)
 237 {
 238     uint32_t nextFree = *_nextFree;
 239
 240     if (!nextFree)
 241         nextFree = hibernate_page_list_count(map, 0, 0);
 242
 243     *_nextFree = nextFree + 1 + hibernate_page_list_count(map, 0, nextFree + 1);
 244
 245     return (nextFree);
 246 }
 247
 248 static uint32_t
 249 store_one_page(uint32_t procFlags, uint32_t * src, uint32_t compressedSize,
 250                 uint32_t * buffer, uint32_t ppnum)
 251 {
 252     uint64_t dst;
 253     uint32_t sum;
 254
 255     dst = ptoa_64(ppnum);
 256 #if __ppc__
 257     if (ppnum < 0x00100000)
 258         buffer = (uint32_t *) (uint32_t) dst;
 259 #elif __i386__
 260     if (ppnum < atop_32(0xC0000000)) {
 261         buffer = (uint32_t *) (uint32_t) dst;
 262     }
 263 #endif
 264
 265     if (compressedSize != PAGE_SIZE)
 266     {
 267         WKdm_decompress((WK_word*) src, (WK_word*) buffer, PAGE_SIZE >> 2);
 268         src = buffer;
 269     }
 270
 271     sum = hibernate_sum((uint8_t *) src, PAGE_SIZE);
 272
 273     if (((uint64_t) (uint32_t) src) == dst)
 274         src = 0;
 275
 276     hibernate_restore_phys_page((uint64_t) (uint32_t) src, dst, PAGE_SIZE, procFlags);
 277
 278     return (sum);
 279 }
 280
 281 static void
 282 bcopy_internal(const void *src, void *dst, uint32_t len)
 283 {
 284     const char *s = src;
 285     char       *d = dst;
 286     uint32_t   idx = 0;
 287
 288     while (idx < len)
 289     {
 290         d[idx] = s[idx];
 291         idx++;
 292     }
 293 }
 294
 295 long
 296 hibernate_kernel_entrypoint(IOHibernateImageHeader * header,
 297                             void * p2, void * p3, __unused void * p4)
 298 {
 299     typedef void (*ResetProc)(void);
 300     uint32_t idx;
 301     uint32_t * src;
 302     uint32_t * buffer;
 303     uint32_t * pageIndexSource;
 304     hibernate_page_list_t * map;
 305     uint32_t count;
 306     uint32_t ppnum;
 307     uint32_t page;
 308     uint32_t conflictCount;
 309     uint32_t compressedSize;
 310     uint32_t uncompressedPages;
 311     uint32_t copyPageListHead;
 312     uint32_t * copyPageList;
 313     uint32_t copyPageIndex;
 314     uint32_t sum;
 315     uint32_t nextFree;
 316     uint32_t lastImagePage;
 317     uint32_t lastMapPage;
 318     uint32_t lastPageIndexPage;
 319
 320
 321     bcopy_internal(header,
 322                 gIOHibernateCurrentHeader,
 323                 sizeof(IOHibernateImageHeader));
 324
 325     if (p2)
 326         bcopy_internal(p2,
 327                 gIOHibernateGraphicsInfo,
 328                 sizeof(hibernate_graphics_t));
 329     else
 330         gIOHibernateGraphicsInfo->physicalAddress = gIOHibernateGraphicsInfo->depth = 0;
 331
 332     if (p3)
 333         bcopy_internal(p3,
 334                 gIOHibernateCryptWakeVars,
 335                 sizeof(hibernate_cryptvars_t));
 336
 337     src = (uint32_t *)
 338                 (((uint32_t) &header->fileExtentMap[0])
 339                             + header->fileExtentMapSize
 340                             + ptoa_32(header->restore1PageCount));
 341
 342     if (header->previewSize)
 343     {
 344         pageIndexSource = src;
 345         map = (hibernate_page_list_t *)(((uint32_t) pageIndexSource) + header->previewSize);
 346         src = (uint32_t *) (((uint32_t) pageIndexSource) + header->previewPageListSize);
 347     }
 348     else
 349     {
 350         pageIndexSource = 0;
 351         map = (hibernate_page_list_t *) src;
 352         src = (uint32_t *) (((uint32_t) map) + header->bitmapSize);
 353     }
 354
 355     lastPageIndexPage = atop_32(src);
 356
 357     lastImagePage = atop_32(((uint32_t) header) + header->image1Size);
 358
 359     lastMapPage = atop_32(((uint32_t) map) + header->bitmapSize);
 360
 361     // knock all the image pages to be used out of free map
 362     for (ppnum = atop_32(header); ppnum <= lastImagePage; ppnum++)
 363     {
 364         hibernate_page_bitset(map, FALSE, ppnum);
 365     }
 366
 367     nextFree = 0;
 368     buffer = (uint32_t *) ptoa_32(hibernate_page_list_grab(map, &nextFree));
 369
 370     sum = gIOHibernateCurrentHeader->actualRestore1Sum;
 371     gIOHibernateCurrentHeader->diag[0] = (uint32_t) header;
 372     gIOHibernateCurrentHeader->diag[1] = sum;
 373
 374     uncompressedPages = 0;
 375     conflictCount     = 0;
 376     copyPageListHead  = 0;
 377     copyPageList      = 0;
 378     copyPageIndex     = PAGE_SIZE >> 2;
 379
 380     compressedSize    = PAGE_SIZE;
 381
 382     while (1)
 383     {
 384         if (pageIndexSource)
 385         {
 386             ppnum = pageIndexSource[0];
 387             count = pageIndexSource[1];
 388             pageIndexSource += 2;
 389             if (!count)
 390             {
 391                 pageIndexSource = 0;
 392                 src =  (uint32_t *) (((uint32_t) map) + gIOHibernateCurrentHeader->bitmapSize);
 393                 ppnum = src[0];
 394                 count = src[1];
 395                 src += 2;
 396             }
 397         }
 398         else
 399         {
 400             ppnum = src[0];
 401             count = src[1];
 402             if (!count)
 403                 break;
 404             src += 2;
 405         }
 406
 407         for (page = 0; page < count; page++, ppnum++)
 408         {
 409             uint32_t tag;
 410             int conflicts;
 411
 412             if (!pageIndexSource)
 413             {
 414                 tag = *src++;
 415                 compressedSize = kIOHibernateTagLength & tag;
 416             }
 417
 418             conflicts = (((ppnum >= atop_32(map)) && (ppnum <= lastMapPage))
 419                       || ((ppnum >= atop_32(src)) && (ppnum <= lastImagePage)));
 420
 421             if (pageIndexSource)
 422                 conflicts |= ((ppnum >= atop_32(pageIndexSource)) && (ppnum <= lastPageIndexPage));
 423
 424             if (!conflicts)
 425             {
 426                 if (compressedSize)
 427                     sum += store_one_page(gIOHibernateCurrentHeader->processorFlags,
 428                                             src, compressedSize, buffer, ppnum);
 429                 uncompressedPages++;
 430             }
 431             else
 432             {
 433                 uint32_t   bufferPage;
 434                 uint32_t * dst;
 435
 436                 conflictCount++;
 437
 438                 // alloc new buffer page
 439                 bufferPage = hibernate_page_list_grab(map, &nextFree);
 440
 441                 if (copyPageIndex > ((PAGE_SIZE >> 2) - 3))
 442                 {
 443                     // alloc new copy list page
 444                     uint32_t pageListPage = hibernate_page_list_grab(map, &nextFree);
 445                     // link to current
 446                     if (copyPageList)
 447                         copyPageList[1] = pageListPage;
 448                     else
 449                         copyPageListHead = pageListPage;
 450                     copyPageList = (uint32_t *) ptoa_32(pageListPage);
 451                     copyPageList[1] = 0;
 452                     copyPageIndex = 2;
 453                 }
 454
 455                 copyPageList[copyPageIndex++] = ppnum;
 456                 copyPageList[copyPageIndex++] = bufferPage;
 457                 copyPageList[copyPageIndex++] = compressedSize;
 458                 copyPageList[0] = copyPageIndex;
 459
 460                 dst = (uint32_t *) ptoa_32(bufferPage);
 461                 for (idx = 0; idx < ((compressedSize + 3) >> 2); idx++)
 462                     dst[idx] = src[idx];
 463             }
 464             src += ((compressedSize + 3) >> 2);
 465         }
 466     }
 467
 468     // -- copy back conflicts
 469
 470     copyPageList = (uint32_t *) ptoa_32(copyPageListHead);
 471     while (copyPageList)
 472     {
 473         for (copyPageIndex = 2; copyPageIndex < copyPageList[0]; copyPageIndex += 3)
 474         {
 475             ppnum          =              copyPageList[copyPageIndex + 0];
 476             src            = (uint32_t *) ptoa_32(copyPageList[copyPageIndex + 1]);
 477             compressedSize =              copyPageList[copyPageIndex + 2];
 478
 479             sum += store_one_page(gIOHibernateCurrentHeader->processorFlags,
 480                                     src, compressedSize, buffer, ppnum);
 481             uncompressedPages++;
 482         }
 483         copyPageList = (uint32_t *) ptoa_32(copyPageList[1]);
 484     }
 485
 486     // -- image has been destroyed...
 487
 488     gIOHibernateCurrentHeader->actualImage1Sum         = sum;
 489     gIOHibernateCurrentHeader->actualUncompressedPages = uncompressedPages;
 490     gIOHibernateCurrentHeader->conflictCount           = conflictCount;
 491     gIOHibernateCurrentHeader->nextFree                = nextFree;
 492
 493     gIOHibernateState = kIOHibernateStateWakingFromHibernate;
 494
 495 #if __ppc__
 496     ResetProc proc;
 497     proc = (ResetProc) 0x100;
 498     __asm__ volatile("ori 0, 0, 0" : : );
 499     proc();
 500 #elif __i386__
 501     ResetProc proc;
 502     proc = (ResetProc) acpi_wake_prot_entry;
 503
 504     proc();
 505 #endif
 506
 507     return -1;
 508 }
 509