iokit/Kernel/IOHibernateRestoreKernel.c

   1 /*
   2  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 #include <stdint.h>
  25 #include <mach/mach_types.h>
  26 #include <mach/vm_param.h>
  27 #include <IOKit/IOHibernatePrivate.h>
  28 #include <pexpert/boot.h>
  29 #include <crypto/aes.h>
  30
  31 #include "WKdm.h"
  32 #include "IOHibernateInternal.h"
  33
  34 /*
  35 This code is linked into the kernel but part of the "__HIB" section, which means
  36 its used by code running in the special context of restoring the kernel text and data
  37 from the hibernation image read by the booter. hibernate_kernel_entrypoint() and everything
  38 it calls or references needs to be careful to only touch memory also in the "__HIB" section.
  39 */
  40
  41 uint32_t gIOHibernateState;
  42
  43 static IOHibernateImageHeader _hibernateHeader;
  44 IOHibernateImageHeader * gIOHibernateCurrentHeader = &_hibernateHeader;
  45
  46 static hibernate_graphics_t _hibernateGraphics;
  47 hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
  48
  49 static hibernate_cryptwakevars_t _cryptWakeVars;
  50 hibernate_cryptwakevars_t * gIOHibernateCryptWakeVars = &_cryptWakeVars;
  51
  52 #if __i386__
  53 extern void   acpi_wake_prot_entry(void);
  54 #endif
  55
  56
  57 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  58
  59 #define BASE 65521L /* largest prime smaller than 65536 */
  60 #define NMAX 5000
  61 // NMAX (was 5521) the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1
  62
  63 #define DO1(buf,i)  {s1 += buf[i]; s2 += s1;}
  64 #define DO2(buf,i)  DO1(buf,i); DO1(buf,i+1);
  65 #define DO4(buf,i)  DO2(buf,i); DO2(buf,i+2);
  66 #define DO8(buf,i)  DO4(buf,i); DO4(buf,i+4);
  67 #define DO16(buf)   DO8(buf,0); DO8(buf,8);
  68
  69 uint32_t
  70 hibernate_sum(uint8_t *buf, int32_t len)
  71 {
  72     unsigned long s1 = 1; // adler & 0xffff;
  73     unsigned long s2 = 0; // (adler >> 16) & 0xffff;
  74     int k;
  75
  76     while (len > 0) {
  77         k = len < NMAX ? len : NMAX;
  78         len -= k;
  79         while (k >= 16) {
  80             DO16(buf);
  81             buf += 16;
  82             k -= 16;
  83         }
  84         if (k != 0) do {
  85             s1 += *buf++;
  86             s2 += s1;
  87         } while (--k);
  88         s1 %= BASE;
  89         s2 %= BASE;
  90     }
  91     return (s2 << 16) | s1;
  92 }
  93
  94 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  95
  96 #if __ppc__
  97 static __inline__ unsigned int cntlzw(unsigned int num)
  98 {
  99   unsigned int result;
 100   __asm__ volatile("cntlzw %0, %1" : "=r" (result) : "r" (num));
 101   return result;
 102 }
 103 #elif __i386__
 104 static __inline__ unsigned int cntlzw(unsigned int num)
 105 {
 106     unsigned int result;
 107     __asm__ volatile(   "bsrl   %1, %0\n\t"
 108                         "cmovel %2, %0"
 109                      : "=r" (result)
 110                      : "rm" (num), "r" (63));
 111     return 31 ^ result;
 112 }
 113 #else
 114 #error arch
 115 #endif
 116
 117 void
 118 hibernate_page_bitset(hibernate_page_list_t * list, boolean_t set, uint32_t page)
 119 {
 120     uint32_t             bank;
 121     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 122
 123     for (bank = 0; bank < list->bank_count; bank++)
 124     {
 125         if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
 126         {
 127             page -= bitmap->first_page;
 128             if (set)
 129                 bitmap->bitmap[page >> 5] |= (0x80000000 >> (page & 31));
 130                 //setbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
 131             else
 132                 bitmap->bitmap[page >> 5] &= ~(0x80000000 >> (page & 31));
 133                 //clrbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
 134             break;
 135         }
 136         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 137     }
 138 }
 139
 140 boolean_t
 141 hibernate_page_bittst(hibernate_page_list_t * list, uint32_t page)
 142 {
 143     boolean_t            result = TRUE;
 144     uint32_t             bank;
 145     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 146
 147     for (bank = 0; bank < list->bank_count; bank++)
 148     {
 149         if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
 150         {
 151             page -= bitmap->first_page;
 152             result = (0 != (bitmap->bitmap[page >> 5] & (0x80000000 >> (page & 31))));
 153             break;
 154         }
 155         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 156     }
 157     return (result);
 158 }
 159
 160 // count bits clear or set (set == TRUE) starting at index page.
 161 uint32_t
 162 hibernate_page_list_count(hibernate_page_list_t * list, uint32_t set, uint32_t page)
 163 {
 164     uint32_t                    bank, count;
 165     hibernate_bitmap_t *        bitmap;
 166
 167     bitmap = &list->bank_bitmap[0];
 168     count  = 0;
 169
 170     for (bank = 0; bank < list->bank_count; bank++)
 171     {
 172         // bits between banks are "set"
 173         if (set && (page < bitmap->first_page))
 174         {
 175             count += bitmap->first_page - page;
 176             page  = bitmap->first_page;
 177         }
 178         if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
 179         {
 180             uint32_t index, bit, bits;
 181
 182             index = (page - bitmap->first_page) >> 5;
 183             bit = (page - bitmap->first_page) & 31;
 184
 185             while (TRUE)
 186             {
 187                 bits = bitmap->bitmap[index];
 188                 if (set)
 189                     bits = ~bits;
 190                 bits = (bits << bit);
 191                 count += cntlzw(bits);
 192                 if (bits)
 193                     break;
 194                 count -= bit;
 195
 196                 while (++index < bitmap->bitmapwords)
 197                 {
 198                     bits = bitmap->bitmap[index];
 199                     if (set)
 200                         bits = ~bits;
 201                     count += cntlzw(bits);
 202                     if (bits)
 203                         break;
 204                 }
 205                 if (bits)
 206                     break;
 207                 if (!set)
 208                     break;
 209                 // bits between banks are "set"
 210                 bank++;
 211                 if (bank >= list->bank_count)
 212                     break;
 213                 count -= (bitmap->last_page + 1);
 214                 bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 215                 count += bitmap->first_page;
 216                 index = 0;
 217                 bit = 0;
 218             }
 219             break;
 220         }
 221         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 222     }
 223
 224     return (count);
 225 }
 226
 227
 228 static uint32_t
 229 hibernate_page_list_grab(hibernate_page_list_t * map, uint32_t * _nextFree)
 230 {
 231     uint32_t nextFree = *_nextFree;
 232
 233     if (!nextFree)
 234         nextFree = hibernate_page_list_count(map, 0, 0);
 235
 236     *_nextFree = nextFree + 1 + hibernate_page_list_count(map, 0, nextFree + 1);
 237
 238     return (nextFree);
 239 }
 240
 241 static uint32_t
 242 store_one_page(uint32_t procFlags, uint32_t * src, uint32_t compressedSize,
 243                 uint32_t * buffer, uint32_t ppnum)
 244 {
 245     uint64_t dst;
 246     uint32_t sum;
 247
 248     dst = ptoa_64(ppnum);
 249 #if __ppc__
 250     if (ppnum < 0x00100000)
 251         buffer = (uint32_t *) (uint32_t) dst;
 252 #elif __i386__
 253     if (ppnum < atop_32(0xC0000000)) {
 254         buffer = (uint32_t *) (uint32_t) dst;
 255     }
 256 #endif
 257
 258     if (compressedSize != PAGE_SIZE)
 259     {
 260         WKdm_decompress((WK_word*) src, (WK_word*) buffer, PAGE_SIZE >> 2);
 261         src = buffer;
 262     }
 263
 264     sum = hibernate_sum((uint8_t *) src, PAGE_SIZE);
 265
 266     if (((uint64_t) (uint32_t) src) == dst)
 267         src = 0;
 268
 269     hibernate_restore_phys_page((uint64_t) (uint32_t) src, dst, PAGE_SIZE, procFlags);
 270
 271     return (sum);
 272 }
 273
 274 static void
 275 bcopy_internal(const void *src, void *dst, uint32_t len)
 276 {
 277     const char *s = src;
 278     char       *d = dst;
 279     uint32_t   idx = 0;
 280
 281     while (idx < len)
 282     {
 283         d[idx] = s[idx];
 284         idx++;
 285     }
 286 }
 287
 288 long
 289 hibernate_kernel_entrypoint(IOHibernateImageHeader * header,
 290                             void * p2, void * p3, __unused void * p4)
 291 {
 292     typedef void (*ResetProc)(void);
 293     uint32_t idx;
 294     uint32_t * src;
 295     uint32_t * buffer;
 296     uint32_t * pageIndexSource;
 297     hibernate_page_list_t * map;
 298     uint32_t count;
 299     uint32_t ppnum;
 300     uint32_t page;
 301     uint32_t conflictCount;
 302     uint32_t compressedSize;
 303     uint32_t uncompressedPages;
 304     uint32_t copyPageListHead;
 305     uint32_t * copyPageList;
 306     uint32_t copyPageIndex;
 307     uint32_t sum;
 308     uint32_t nextFree;
 309     uint32_t lastImagePage;
 310     uint32_t lastMapPage;
 311     uint32_t lastPageIndexPage;
 312
 313
 314     bcopy_internal(header,
 315                 gIOHibernateCurrentHeader,
 316                 sizeof(IOHibernateImageHeader));
 317
 318     if (p2)
 319         bcopy_internal(p2,
 320                 gIOHibernateGraphicsInfo,
 321                 sizeof(hibernate_graphics_t));
 322     else
 323         gIOHibernateGraphicsInfo->physicalAddress = gIOHibernateGraphicsInfo->depth = 0;
 324
 325     if (p3)
 326         bcopy_internal(p3,
 327                 gIOHibernateCryptWakeVars,
 328                 sizeof(hibernate_cryptvars_t));
 329
 330     src = (uint32_t *)
 331                 (((uint32_t) &header->fileExtentMap[0])
 332                             + header->fileExtentMapSize
 333                             + ptoa_32(header->restore1PageCount));
 334
 335     if (header->previewSize)
 336     {
 337         pageIndexSource = src;
 338         map = (hibernate_page_list_t *)(((uint32_t) pageIndexSource) + header->previewSize);
 339         src = (uint32_t *) (((uint32_t) pageIndexSource) + header->previewPageListSize);
 340     }
 341     else
 342     {
 343         pageIndexSource = 0;
 344         map = (hibernate_page_list_t *) src;
 345         src = (uint32_t *) (((uint32_t) map) + header->bitmapSize);
 346     }
 347
 348     lastPageIndexPage = atop_32(src);
 349
 350     lastImagePage = atop_32(((uint32_t) header) + header->image1Size);
 351
 352     lastMapPage = atop_32(((uint32_t) map) + header->bitmapSize);
 353
 354     // knock all the image pages to be used out of free map
 355     for (ppnum = atop_32(header); ppnum <= lastImagePage; ppnum++)
 356     {
 357         hibernate_page_bitset(map, FALSE, ppnum);
 358     }
 359
 360     nextFree = 0;
 361     buffer = (uint32_t *) ptoa_32(hibernate_page_list_grab(map, &nextFree));
 362
 363     sum = gIOHibernateCurrentHeader->actualRestore1Sum;
 364     gIOHibernateCurrentHeader->diag[0] = (uint32_t) header;
 365     gIOHibernateCurrentHeader->diag[1] = sum;
 366
 367     uncompressedPages = 0;
 368     conflictCount     = 0;
 369     copyPageListHead  = 0;
 370     copyPageList      = 0;
 371     copyPageIndex     = PAGE_SIZE >> 2;
 372
 373     compressedSize    = PAGE_SIZE;
 374
 375     while (1)
 376     {
 377         if (pageIndexSource)
 378         {
 379             ppnum = pageIndexSource[0];
 380             count = pageIndexSource[1];
 381             pageIndexSource += 2;
 382             if (!count)
 383             {
 384                 pageIndexSource = 0;
 385                 src =  (uint32_t *) (((uint32_t) map) + gIOHibernateCurrentHeader->bitmapSize);
 386                 ppnum = src[0];
 387                 count = src[1];
 388                 src += 2;
 389             }
 390         }
 391         else
 392         {
 393             ppnum = src[0];
 394             count = src[1];
 395             if (!count)
 396                 break;
 397             src += 2;
 398         }
 399
 400         for (page = 0; page < count; page++, ppnum++)
 401         {
 402             uint32_t tag;
 403             int conflicts;
 404
 405             if (!pageIndexSource)
 406             {
 407                 tag = *src++;
 408                 compressedSize = kIOHibernateTagLength & tag;
 409             }
 410
 411             conflicts = (((ppnum >= atop_32(map)) && (ppnum <= lastMapPage))
 412                       || ((ppnum >= atop_32(src)) && (ppnum <= lastImagePage)));
 413
 414             if (pageIndexSource)
 415                 conflicts |= ((ppnum >= atop_32(pageIndexSource)) && (ppnum <= lastPageIndexPage));
 416
 417             if (!conflicts)
 418             {
 419                 if (compressedSize)
 420                     sum += store_one_page(gIOHibernateCurrentHeader->processorFlags,
 421                                             src, compressedSize, buffer, ppnum);
 422                 uncompressedPages++;
 423             }
 424             else
 425             {
 426                 uint32_t   bufferPage;
 427                 uint32_t * dst;
 428
 429                 conflictCount++;
 430
 431                 // alloc new buffer page
 432                 bufferPage = hibernate_page_list_grab(map, &nextFree);
 433
 434                 if (copyPageIndex > ((PAGE_SIZE >> 2) - 3))
 435                 {
 436                     // alloc new copy list page
 437                     uint32_t pageListPage = hibernate_page_list_grab(map, &nextFree);
 438                     // link to current
 439                     if (copyPageList)
 440                         copyPageList[1] = pageListPage;
 441                     else
 442                         copyPageListHead = pageListPage;
 443                     copyPageList = (uint32_t *) ptoa_32(pageListPage);
 444                     copyPageList[1] = 0;
 445                     copyPageIndex = 2;
 446                 }
 447
 448                 copyPageList[copyPageIndex++] = ppnum;
 449                 copyPageList[copyPageIndex++] = bufferPage;
 450                 copyPageList[copyPageIndex++] = compressedSize;
 451                 copyPageList[0] = copyPageIndex;
 452
 453                 dst = (uint32_t *) ptoa_32(bufferPage);
 454                 for (idx = 0; idx < ((compressedSize + 3) >> 2); idx++)
 455                     dst[idx] = src[idx];
 456             }
 457             src += ((compressedSize + 3) >> 2);
 458         }
 459     }
 460
 461     // -- copy back conflicts
 462
 463     copyPageList = (uint32_t *) ptoa_32(copyPageListHead);
 464     while (copyPageList)
 465     {
 466         for (copyPageIndex = 2; copyPageIndex < copyPageList[0]; copyPageIndex += 3)
 467         {
 468             ppnum          =              copyPageList[copyPageIndex + 0];
 469             src            = (uint32_t *) ptoa_32(copyPageList[copyPageIndex + 1]);
 470             compressedSize =              copyPageList[copyPageIndex + 2];
 471
 472             sum += store_one_page(gIOHibernateCurrentHeader->processorFlags,
 473                                     src, compressedSize, buffer, ppnum);
 474             uncompressedPages++;
 475         }
 476         copyPageList = (uint32_t *) ptoa_32(copyPageList[1]);
 477     }
 478
 479     // -- image has been destroyed...
 480
 481     gIOHibernateCurrentHeader->actualImage1Sum         = sum;
 482     gIOHibernateCurrentHeader->actualUncompressedPages = uncompressedPages;
 483     gIOHibernateCurrentHeader->conflictCount           = conflictCount;
 484     gIOHibernateCurrentHeader->nextFree                = nextFree;
 485
 486     gIOHibernateState = kIOHibernateStateWakingFromHibernate;
 487
 488 #if __ppc__
 489     ResetProc proc;
 490     proc = (ResetProc) 0x100;
 491     __asm__ volatile("ori 0, 0, 0" : : );
 492     proc();
 493 #elif __i386__
 494     ResetProc proc;
 495     proc = (ResetProc) acpi_wake_prot_entry;
 496
 497     proc();
 498 #endif
 499
 500     return -1;
 501 }
 502