iokit/Kernel/IOHibernateRestoreKernel.c

   1 /*
   2  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * The contents of this file constitute Original Code as defined in and
   7  * are subject to the Apple Public Source License Version 1.1 (the
   8  * "License").  You may not use this file except in compliance with the
   9  * License.  Please obtain a copy of the License at
  10  * http://www.apple.com/publicsource and read it before using this file.
  11  *
  12  * This Original Code and all software distributed under the License are
  13  * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  14  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  15  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
  17  * License for the specific language governing rights and limitations
  18  * under the License.
  19  *
  20  * @APPLE_LICENSE_HEADER_END@
  21  */
  22
  23 #include <stdint.h>
  24 #include <mach/mach_types.h>
  25 #include <mach/vm_param.h>
  26 #include <IOKit/IOHibernatePrivate.h>
  27 #include <pexpert/boot.h>
  28 #include <crypto/aes.h>
  29
  30 #include "WKdm.h"
  31 #include "IOHibernateInternal.h"
  32
  33 /*
  34 This code is linked into the kernel but part of the "__HIB" section, which means
  35 its used by code running in the special context of restoring the kernel text and data
  36 from the hibernation image read by the booter. hibernate_kernel_entrypoint() and everything
  37 it calls or references needs to be careful to only touch memory also in the "__HIB" section.
  38 */
  39
  40 uint32_t gIOHibernateState;
  41
  42 static IOHibernateImageHeader _hibernateHeader;
  43 IOHibernateImageHeader * gIOHibernateCurrentHeader = &_hibernateHeader;
  44
  45 static hibernate_graphics_t _hibernateGraphics;
  46 hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
  47
  48 static hibernate_cryptwakevars_t _cryptWakeVars;
  49 hibernate_cryptwakevars_t * gIOHibernateCryptWakeVars = &_cryptWakeVars;
  50
  51 #if __i386__
  52 extern void   acpi_wake_prot_entry(void);
  53 #endif
  54
  55
  56 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  57
  58 #define BASE 65521L /* largest prime smaller than 65536 */
  59 #define NMAX 5000
  60 // NMAX (was 5521) the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1
  61
  62 #define DO1(buf,i)  {s1 += buf[i]; s2 += s1;}
  63 #define DO2(buf,i)  DO1(buf,i); DO1(buf,i+1);
  64 #define DO4(buf,i)  DO2(buf,i); DO2(buf,i+2);
  65 #define DO8(buf,i)  DO4(buf,i); DO4(buf,i+4);
  66 #define DO16(buf)   DO8(buf,0); DO8(buf,8);
  67
  68 uint32_t
  69 hibernate_sum(uint8_t *buf, int32_t len)
  70 {
  71     unsigned long s1 = 1; // adler & 0xffff;
  72     unsigned long s2 = 0; // (adler >> 16) & 0xffff;
  73     int k;
  74
  75     while (len > 0) {
  76         k = len < NMAX ? len : NMAX;
  77         len -= k;
  78         while (k >= 16) {
  79             DO16(buf);
  80             buf += 16;
  81             k -= 16;
  82         }
  83         if (k != 0) do {
  84             s1 += *buf++;
  85             s2 += s1;
  86         } while (--k);
  87         s1 %= BASE;
  88         s2 %= BASE;
  89     }
  90     return (s2 << 16) | s1;
  91 }
  92
  93 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  94
  95 #if __ppc__
  96 static __inline__ unsigned int cntlzw(unsigned int num)
  97 {
  98   unsigned int result;
  99   __asm__ volatile("cntlzw %0, %1" : "=r" (result) : "r" (num));
 100   return result;
 101 }
 102 #elif __i386__
 103 static __inline__ unsigned int cntlzw(unsigned int num)
 104 {
 105     unsigned int result;
 106     __asm__ volatile(   "bsrl   %1, %0\n\t"
 107                         "cmovel %2, %0"
 108                      : "=r" (result)
 109                      : "rm" (num), "r" (63));
 110     return 31 ^ result;
 111 }
 112 #else
 113 #error arch
 114 #endif
 115
 116 void
 117 hibernate_page_bitset(hibernate_page_list_t * list, boolean_t set, uint32_t page)
 118 {
 119     uint32_t             bank;
 120     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 121
 122     for (bank = 0; bank < list->bank_count; bank++)
 123     {
 124         if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
 125         {
 126             page -= bitmap->first_page;
 127             if (set)
 128                 bitmap->bitmap[page >> 5] |= (0x80000000 >> (page & 31));
 129                 //setbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
 130             else
 131                 bitmap->bitmap[page >> 5] &= ~(0x80000000 >> (page & 31));
 132                 //clrbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
 133             break;
 134         }
 135         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 136     }
 137 }
 138
 139 boolean_t
 140 hibernate_page_bittst(hibernate_page_list_t * list, uint32_t page)
 141 {
 142     boolean_t            result = TRUE;
 143     uint32_t             bank;
 144     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 145
 146     for (bank = 0; bank < list->bank_count; bank++)
 147     {
 148         if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
 149         {
 150             page -= bitmap->first_page;
 151             result = (0 != (bitmap->bitmap[page >> 5] & (0x80000000 >> (page & 31))));
 152             break;
 153         }
 154         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 155     }
 156     return (result);
 157 }
 158
 159 // count bits clear or set (set == TRUE) starting at index page.
 160 uint32_t
 161 hibernate_page_list_count(hibernate_page_list_t * list, uint32_t set, uint32_t page)
 162 {
 163     uint32_t                    bank, count;
 164     hibernate_bitmap_t *        bitmap;
 165
 166     bitmap = &list->bank_bitmap[0];
 167     count  = 0;
 168
 169     for (bank = 0; bank < list->bank_count; bank++)
 170     {
 171         // bits between banks are "set"
 172         if (set && (page < bitmap->first_page))
 173         {
 174             count += bitmap->first_page - page;
 175             page  = bitmap->first_page;
 176         }
 177         if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
 178         {
 179             uint32_t index, bit, bits;
 180
 181             index = (page - bitmap->first_page) >> 5;
 182             bit = (page - bitmap->first_page) & 31;
 183
 184             while (TRUE)
 185             {
 186                 bits = bitmap->bitmap[index];
 187                 if (set)
 188                     bits = ~bits;
 189                 bits = (bits << bit);
 190                 count += cntlzw(bits);
 191                 if (bits)
 192                     break;
 193                 count -= bit;
 194
 195                 while (++index < bitmap->bitmapwords)
 196                 {
 197                     bits = bitmap->bitmap[index];
 198                     if (set)
 199                         bits = ~bits;
 200                     count += cntlzw(bits);
 201                     if (bits)
 202                         break;
 203                 }
 204                 if (bits)
 205                     break;
 206                 if (!set)
 207                     break;
 208                 // bits between banks are "set"
 209                 bank++;
 210                 if (bank >= list->bank_count)
 211                     break;
 212                 count -= (bitmap->last_page + 1);
 213                 bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 214                 count += bitmap->first_page;
 215                 index = 0;
 216                 bit = 0;
 217             }
 218             break;
 219         }
 220         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 221     }
 222
 223     return (count);
 224 }
 225
 226
 227 static uint32_t
 228 hibernate_page_list_grab(hibernate_page_list_t * map, uint32_t * _nextFree)
 229 {
 230     uint32_t nextFree = *_nextFree;
 231
 232     if (!nextFree)
 233         nextFree = hibernate_page_list_count(map, 0, 0);
 234
 235     *_nextFree = nextFree + 1 + hibernate_page_list_count(map, 0, nextFree + 1);
 236
 237     return (nextFree);
 238 }
 239
 240 static uint32_t
 241 store_one_page(uint32_t procFlags, uint32_t * src, uint32_t compressedSize,
 242                 uint32_t * buffer, uint32_t ppnum)
 243 {
 244     uint64_t dst;
 245     uint32_t sum;
 246
 247     dst = ptoa_64(ppnum);
 248 #if __ppc__
 249     if (ppnum < 0x00100000)
 250         buffer = (uint32_t *) (uint32_t) dst;
 251 #elif __i386__
 252     if (ppnum < atop_32(0xC0000000)) {
 253         buffer = (uint32_t *) (uint32_t) dst;
 254     }
 255 #endif
 256
 257     if (compressedSize != PAGE_SIZE)
 258     {
 259         WKdm_decompress((WK_word*) src, (WK_word*) buffer, PAGE_SIZE >> 2);
 260         src = buffer;
 261     }
 262
 263     sum = hibernate_sum((uint8_t *) src, PAGE_SIZE);
 264
 265     if (((uint64_t) (uint32_t) src) == dst)
 266         src = 0;
 267
 268     hibernate_restore_phys_page((uint64_t) (uint32_t) src, dst, PAGE_SIZE, procFlags);
 269
 270     return (sum);
 271 }
 272
 273 static void
 274 bcopy_internal(const void *src, void *dst, uint32_t len)
 275 {
 276     const char *s = src;
 277     char       *d = dst;
 278     uint32_t   idx = 0;
 279
 280     while (idx < len)
 281     {
 282         d[idx] = s[idx];
 283         idx++;
 284     }
 285 }
 286
 287 long
 288 hibernate_kernel_entrypoint(IOHibernateImageHeader * header,
 289                             void * p2, void * p3, __unused void * p4)
 290 {
 291     typedef void (*ResetProc)(void);
 292     uint32_t idx;
 293     uint32_t * src;
 294     uint32_t * buffer;
 295     uint32_t * pageIndexSource;
 296     hibernate_page_list_t * map;
 297     uint32_t count;
 298     uint32_t ppnum;
 299     uint32_t page;
 300     uint32_t conflictCount;
 301     uint32_t compressedSize;
 302     uint32_t uncompressedPages;
 303     uint32_t copyPageListHead;
 304     uint32_t * copyPageList;
 305     uint32_t copyPageIndex;
 306     uint32_t sum;
 307     uint32_t nextFree;
 308     uint32_t lastImagePage;
 309     uint32_t lastMapPage;
 310     uint32_t lastPageIndexPage;
 311
 312
 313     bcopy_internal(header,
 314                 gIOHibernateCurrentHeader,
 315                 sizeof(IOHibernateImageHeader));
 316
 317     if (p2)
 318         bcopy_internal(p2,
 319                 gIOHibernateGraphicsInfo,
 320                 sizeof(hibernate_graphics_t));
 321     else
 322         gIOHibernateGraphicsInfo->physicalAddress = gIOHibernateGraphicsInfo->depth = 0;
 323
 324     if (p3)
 325         bcopy_internal(p3,
 326                 gIOHibernateCryptWakeVars,
 327                 sizeof(hibernate_cryptvars_t));
 328
 329     src = (uint32_t *)
 330                 (((uint32_t) &header->fileExtentMap[0])
 331                             + header->fileExtentMapSize
 332                             + ptoa_32(header->restore1PageCount));
 333
 334     if (header->previewSize)
 335     {
 336         pageIndexSource = src;
 337         map = (hibernate_page_list_t *)(((uint32_t) pageIndexSource) + header->previewSize);
 338         src = (uint32_t *) (((uint32_t) pageIndexSource) + header->previewPageListSize);
 339     }
 340     else
 341     {
 342         pageIndexSource = 0;
 343         map = (hibernate_page_list_t *) src;
 344         src = (uint32_t *) (((uint32_t) map) + header->bitmapSize);
 345     }
 346
 347     lastPageIndexPage = atop_32(src);
 348
 349     lastImagePage = atop_32(((uint32_t) header) + header->image1Size);
 350
 351     lastMapPage = atop_32(((uint32_t) map) + header->bitmapSize);
 352
 353     // knock all the image pages to be used out of free map
 354     for (ppnum = atop_32(header); ppnum <= lastImagePage; ppnum++)
 355     {
 356         hibernate_page_bitset(map, FALSE, ppnum);
 357     }
 358
 359     nextFree = 0;
 360     buffer = (uint32_t *) ptoa_32(hibernate_page_list_grab(map, &nextFree));
 361
 362     sum = gIOHibernateCurrentHeader->actualRestore1Sum;
 363     gIOHibernateCurrentHeader->diag[0] = (uint32_t) header;
 364     gIOHibernateCurrentHeader->diag[1] = sum;
 365
 366     uncompressedPages = 0;
 367     conflictCount     = 0;
 368     copyPageListHead  = 0;
 369     copyPageList      = 0;
 370     copyPageIndex     = PAGE_SIZE >> 2;
 371
 372     compressedSize    = PAGE_SIZE;
 373
 374     while (1)
 375     {
 376         if (pageIndexSource)
 377         {
 378             ppnum = pageIndexSource[0];
 379             count = pageIndexSource[1];
 380             pageIndexSource += 2;
 381             if (!count)
 382             {
 383                 pageIndexSource = 0;
 384                 src =  (uint32_t *) (((uint32_t) map) + gIOHibernateCurrentHeader->bitmapSize);
 385                 ppnum = src[0];
 386                 count = src[1];
 387                 src += 2;
 388             }
 389         }
 390         else
 391         {
 392             ppnum = src[0];
 393             count = src[1];
 394             if (!count)
 395                 break;
 396             src += 2;
 397         }
 398
 399         for (page = 0; page < count; page++, ppnum++)
 400         {
 401             uint32_t tag;
 402             int conflicts;
 403
 404             if (!pageIndexSource)
 405             {
 406                 tag = *src++;
 407                 compressedSize = kIOHibernateTagLength & tag;
 408             }
 409
 410             conflicts = (((ppnum >= atop_32(map)) && (ppnum <= lastMapPage))
 411                       || ((ppnum >= atop_32(src)) && (ppnum <= lastImagePage)));
 412
 413             if (pageIndexSource)
 414                 conflicts |= ((ppnum >= atop_32(pageIndexSource)) && (ppnum <= lastPageIndexPage));
 415
 416             if (!conflicts)
 417             {
 418                 if (compressedSize)
 419                     sum += store_one_page(gIOHibernateCurrentHeader->processorFlags,
 420                                             src, compressedSize, buffer, ppnum);
 421                 uncompressedPages++;
 422             }
 423             else
 424             {
 425                 uint32_t   bufferPage;
 426                 uint32_t * dst;
 427
 428                 conflictCount++;
 429
 430                 // alloc new buffer page
 431                 bufferPage = hibernate_page_list_grab(map, &nextFree);
 432
 433                 if (copyPageIndex > ((PAGE_SIZE >> 2) - 3))
 434                 {
 435                     // alloc new copy list page
 436                     uint32_t pageListPage = hibernate_page_list_grab(map, &nextFree);
 437                     // link to current
 438                     if (copyPageList)
 439                         copyPageList[1] = pageListPage;
 440                     else
 441                         copyPageListHead = pageListPage;
 442                     copyPageList = (uint32_t *) ptoa_32(pageListPage);
 443                     copyPageList[1] = 0;
 444                     copyPageIndex = 2;
 445                 }
 446
 447                 copyPageList[copyPageIndex++] = ppnum;
 448                 copyPageList[copyPageIndex++] = bufferPage;
 449                 copyPageList[copyPageIndex++] = compressedSize;
 450                 copyPageList[0] = copyPageIndex;
 451
 452                 dst = (uint32_t *) ptoa_32(bufferPage);
 453                 for (idx = 0; idx < ((compressedSize + 3) >> 2); idx++)
 454                     dst[idx] = src[idx];
 455             }
 456             src += ((compressedSize + 3) >> 2);
 457         }
 458     }
 459
 460     // -- copy back conflicts
 461
 462     copyPageList = (uint32_t *) ptoa_32(copyPageListHead);
 463     while (copyPageList)
 464     {
 465         for (copyPageIndex = 2; copyPageIndex < copyPageList[0]; copyPageIndex += 3)
 466         {
 467             ppnum          =              copyPageList[copyPageIndex + 0];
 468             src            = (uint32_t *) ptoa_32(copyPageList[copyPageIndex + 1]);
 469             compressedSize =              copyPageList[copyPageIndex + 2];
 470
 471             sum += store_one_page(gIOHibernateCurrentHeader->processorFlags,
 472                                     src, compressedSize, buffer, ppnum);
 473             uncompressedPages++;
 474         }
 475         copyPageList = (uint32_t *) ptoa_32(copyPageList[1]);
 476     }
 477
 478     // -- image has been destroyed...
 479
 480     gIOHibernateCurrentHeader->actualImage1Sum         = sum;
 481     gIOHibernateCurrentHeader->actualUncompressedPages = uncompressedPages;
 482     gIOHibernateCurrentHeader->conflictCount           = conflictCount;
 483     gIOHibernateCurrentHeader->nextFree                = nextFree;
 484
 485     gIOHibernateState = kIOHibernateStateWakingFromHibernate;
 486
 487 #if __ppc__
 488     ResetProc proc;
 489     proc = (ResetProc) 0x100;
 490     __asm__ volatile("ori 0, 0, 0" : : );
 491     proc();
 492 #elif __i386__
 493     ResetProc proc;
 494     proc = (ResetProc) acpi_wake_prot_entry;
 495
 496     proc();
 497 #endif
 498
 499     return -1;
 500 }
 501