iokit/Kernel/IOHibernateRestoreKernel.c

   1 /*
   2  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License.  The rights granted to you under the
  10  * License may not be used to create, or enable the creation or
  11  * redistribution of, unlawful or unlicensed copies of an Apple operating
  12  * system, or to circumvent, violate, or enable the circumvention or
  13  * violation of, any terms of an Apple operating system software license
  14  * agreement.
  15  *
  16  * Please obtain a copy of the License at
  17  * http://www.opensource.apple.com/apsl/ and read it before using this
  18  * file.
  19  *
  20  * The Original Code and all software distributed under the License are
  21  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  22  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  23  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  24  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  25  * Please see the License for the specific language governing rights and
  26  * limitations under the License.
  27  *
  28  * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
  29  */
  30
  31 #include <stdint.h>
  32 #include <mach/mach_types.h>
  33 #include <mach/vm_param.h>
  34 #include <IOKit/IOHibernatePrivate.h>
  35 #include <IOKit/IOLib.h>
  36 #include <pexpert/boot.h>
  37 #include <crypto/aes.h>
  38
  39 #include "WKdm.h"
  40 #include "IOHibernateInternal.h"
  41
  42 /*
  43 This code is linked into the kernel but part of the "__HIB" section, which means
  44 its used by code running in the special context of restoring the kernel text and data
  45 from the hibernation image read by the booter. hibernate_kernel_entrypoint() and everything
  46 it calls or references needs to be careful to only touch memory also in the "__HIB" section.
  47 */
  48
  49 uint32_t gIOHibernateState;
  50
  51 static IOHibernateImageHeader _hibernateHeader;
  52 IOHibernateImageHeader * gIOHibernateCurrentHeader = &_hibernateHeader;
  53
  54 static hibernate_graphics_t _hibernateGraphics;
  55 hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
  56
  57 static hibernate_cryptwakevars_t _cryptWakeVars;
  58 hibernate_cryptwakevars_t * gIOHibernateCryptWakeVars = &_cryptWakeVars;
  59
  60 vm_offset_t gIOHibernateWakeMap;            // ppnum
  61 vm_size_t   gIOHibernateWakeMapSize;
  62
  63 #if __i386__
  64 extern void   acpi_wake_prot_entry(void);
  65 #endif
  66
  67 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  68
  69 #define BASE 65521L /* largest prime smaller than 65536 */
  70 #define NMAX 5000
  71 // NMAX (was 5521) the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1
  72
  73 #define DO1(buf,i)  {s1 += buf[i]; s2 += s1;}
  74 #define DO2(buf,i)  DO1(buf,i); DO1(buf,i+1);
  75 #define DO4(buf,i)  DO2(buf,i); DO2(buf,i+2);
  76 #define DO8(buf,i)  DO4(buf,i); DO4(buf,i+4);
  77 #define DO16(buf)   DO8(buf,0); DO8(buf,8);
  78
  79 uint32_t
  80 hibernate_sum(uint8_t *buf, int32_t len)
  81 {
  82     unsigned long s1 = 1; // adler & 0xffff;
  83     unsigned long s2 = 0; // (adler >> 16) & 0xffff;
  84     int k;
  85
  86     while (len > 0) {
  87         k = len < NMAX ? len : NMAX;
  88         len -= k;
  89         while (k >= 16) {
  90             DO16(buf);
  91             buf += 16;
  92             k -= 16;
  93         }
  94         if (k != 0) do {
  95             s1 += *buf++;
  96             s2 += s1;
  97         } while (--k);
  98         s1 %= BASE;
  99         s2 %= BASE;
 100     }
 101     return (s2 << 16) | s1;
 102 }
 103
 104 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 105
 106 static hibernate_bitmap_t *
 107 hibernate_page_bitmap(hibernate_page_list_t * list, uint32_t page)
 108 {
 109     uint32_t             bank;
 110     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 111
 112     for (bank = 0; bank < list->bank_count; bank++)
 113     {
 114         if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
 115             break;
 116         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 117     }
 118     if (bank == list->bank_count)
 119         bitmap = 0;
 120
 121     return (bitmap);
 122 }
 123
 124 hibernate_bitmap_t *
 125 hibernate_page_bitmap_pin(hibernate_page_list_t * list, uint32_t * pPage)
 126 {
 127     uint32_t             bank, page = *pPage;
 128     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 129
 130     for (bank = 0; bank < list->bank_count; bank++)
 131     {
 132         if (page <= bitmap->first_page)
 133         {
 134             *pPage = bitmap->first_page;
 135             break;
 136         }
 137         if (page <= bitmap->last_page)
 138             break;
 139         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 140     }
 141     if (bank == list->bank_count)
 142         bitmap = 0;
 143
 144     return (bitmap);
 145 }
 146
 147 void
 148 hibernate_page_bitset(hibernate_page_list_t * list, boolean_t set, uint32_t page)
 149 {
 150     hibernate_bitmap_t * bitmap;
 151
 152     bitmap = hibernate_page_bitmap(list, page);
 153     if (bitmap)
 154     {
 155         page -= bitmap->first_page;
 156         if (set)
 157             bitmap->bitmap[page >> 5] |= (0x80000000 >> (page & 31));
 158             //setbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
 159         else
 160             bitmap->bitmap[page >> 5] &= ~(0x80000000 >> (page & 31));
 161             //clrbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
 162     }
 163 }
 164
 165 boolean_t
 166 hibernate_page_bittst(hibernate_page_list_t * list, uint32_t page)
 167 {
 168     boolean_t            result = TRUE;
 169     hibernate_bitmap_t * bitmap;
 170
 171     bitmap = hibernate_page_bitmap(list, page);
 172     if (bitmap)
 173     {
 174         page -= bitmap->first_page;
 175         result = (0 != (bitmap->bitmap[page >> 5] & (0x80000000 >> (page & 31))));
 176     }
 177     return (result);
 178 }
 179
 180 // count bits clear or set (set == TRUE) starting at page.
 181 uint32_t
 182 hibernate_page_bitmap_count(hibernate_bitmap_t * bitmap, uint32_t set, uint32_t page)
 183 {
 184     uint32_t index, bit, bits;
 185     uint32_t count;
 186
 187     count = 0;
 188
 189     index = (page - bitmap->first_page) >> 5;
 190     bit = (page - bitmap->first_page) & 31;
 191
 192     bits = bitmap->bitmap[index];
 193     if (set)
 194         bits = ~bits;
 195     bits = (bits << bit);
 196     if (bits)
 197         count += __builtin_clz(bits);
 198     else
 199     {
 200         count += 32 - bit;
 201         while (++index < bitmap->bitmapwords)
 202         {
 203             bits = bitmap->bitmap[index];
 204             if (set)
 205                 bits = ~bits;
 206             if (bits)
 207             {
 208                 count += __builtin_clz(bits);
 209                 break;
 210             }
 211             count += 32;
 212         }
 213     }
 214
 215     return (count);
 216 }
 217
 218 static vm_offset_t
 219 hibernate_page_list_grab(hibernate_page_list_t * list, uint32_t * pNextFree)
 220 {
 221     uint32_t             nextFree = *pNextFree;
 222     uint32_t             nextFreeInBank;
 223     hibernate_bitmap_t * bitmap;
 224
 225     nextFreeInBank = nextFree + 1;
 226     while ((bitmap = hibernate_page_bitmap_pin(list, &nextFreeInBank)))
 227     {
 228         nextFreeInBank += hibernate_page_bitmap_count(bitmap, FALSE, nextFreeInBank);
 229         if (nextFreeInBank <= bitmap->last_page)
 230         {
 231             *pNextFree = nextFreeInBank;
 232             break;
 233         }
 234     }
 235
 236     if (!bitmap)
 237         IOPanic(__FUNCTION__);
 238
 239     return (nextFree);
 240 }
 241
 242 static uint32_t
 243 store_one_page(uint32_t procFlags, uint32_t * src, uint32_t compressedSize,
 244                 uint32_t * buffer, uint32_t ppnum)
 245 {
 246     uint64_t dst;
 247     uint32_t sum;
 248
 249     dst = ptoa_64(ppnum);
 250     if (ppnum < 0x00100000)
 251         buffer = (uint32_t *) (uint32_t) dst;
 252
 253     if (compressedSize != PAGE_SIZE)
 254     {
 255         WKdm_decompress((WK_word*) src, (WK_word*) buffer, PAGE_SIZE >> 2);
 256         src = buffer;
 257     }
 258
 259     sum = hibernate_sum((uint8_t *) src, PAGE_SIZE);
 260
 261     if (((uint64_t) (uint32_t) src) == dst)
 262         src = 0;
 263
 264     hibernate_restore_phys_page((uint64_t) (uint32_t) src, dst, PAGE_SIZE, procFlags);
 265
 266     return (sum);
 267 }
 268
 269 static void
 270 bcopy_internal(const void *src, void *dst, uint32_t len)
 271 {
 272     const char *s = src;
 273     char       *d = dst;
 274     uint32_t   idx = 0;
 275
 276     while (idx < len)
 277     {
 278         d[idx] = s[idx];
 279         idx++;
 280     }
 281 }
 282
 283 #define C_ASSERT(e) typedef char    __C_ASSERT__[(e) ? 1 : -1]
 284
 285 long
 286 hibernate_kernel_entrypoint(IOHibernateImageHeader * header,
 287                             void * p2, void * p3, void * p4)
 288 {
 289     typedef void (*ResetProc)(void);
 290     uint32_t idx;
 291     uint32_t * src;
 292     uint32_t * buffer;
 293     uint32_t * pageIndexSource;
 294     hibernate_page_list_t * map;
 295     uint32_t count;
 296     uint32_t ppnum;
 297     uint32_t page;
 298     uint32_t conflictCount;
 299     uint32_t compressedSize;
 300     uint32_t uncompressedPages;
 301     uint32_t copyPageListHead;
 302     uint32_t * copyPageList;
 303     uint32_t copyPageIndex;
 304     uint32_t sum;
 305     uint32_t nextFree;
 306     uint32_t lastImagePage;
 307     uint32_t lastMapPage;
 308     uint32_t lastPageIndexPage;
 309
 310     C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
 311
 312     bcopy_internal(header,
 313                 gIOHibernateCurrentHeader,
 314                 sizeof(IOHibernateImageHeader));
 315
 316     if (!p2)
 317     {
 318         count = header->graphicsInfoOffset;
 319         if (count)
 320             p2 = (void *)(((uintptr_t) header) - count);
 321     }
 322     if (p2)
 323         bcopy_internal(p2,
 324                 gIOHibernateGraphicsInfo,
 325                 sizeof(hibernate_graphics_t));
 326     else
 327         gIOHibernateGraphicsInfo->physicalAddress = gIOHibernateGraphicsInfo->depth = 0;
 328
 329     if (!p3)
 330     {
 331         count = header->cryptVarsOffset;
 332         if (count)
 333             p3 = (void *)(((uintptr_t) header) - count);
 334     }
 335     if (p3)
 336         bcopy_internal(p3,
 337                 gIOHibernateCryptWakeVars,
 338                 sizeof(hibernate_cryptvars_t));
 339
 340     src = (uint32_t *)
 341                 (((uint32_t) &header->fileExtentMap[0])
 342                             + header->fileExtentMapSize
 343                             + ptoa_32(header->restore1PageCount));
 344
 345     if (header->previewSize)
 346     {
 347         pageIndexSource = src;
 348         map = (hibernate_page_list_t *)(((uint32_t) pageIndexSource) + header->previewSize);
 349         src = (uint32_t *) (((uint32_t) pageIndexSource) + header->previewPageListSize);
 350     }
 351     else
 352     {
 353         pageIndexSource = 0;
 354         map = (hibernate_page_list_t *) src;
 355         src = (uint32_t *) (((uint32_t) map) + header->bitmapSize);
 356     }
 357
 358     lastPageIndexPage = atop_32(src);
 359
 360     lastImagePage = atop_32(((uint32_t) header) + header->image1Size);
 361
 362     lastMapPage = atop_32(((uint32_t) map) + header->bitmapSize);
 363
 364     // knock all the image pages to be used out of free map
 365     for (ppnum = atop_32(header); ppnum <= lastImagePage; ppnum++)
 366     {
 367         hibernate_page_bitset(map, FALSE, ppnum);
 368     }
 369
 370     nextFree = 0;
 371     hibernate_page_list_grab(map, &nextFree);
 372     buffer = (uint32_t *) ptoa_32(hibernate_page_list_grab(map, &nextFree));
 373
 374     if (header->memoryMapSize && (count = header->memoryMapOffset))
 375     {
 376         p4 = (void *)(((uintptr_t) header) - count);
 377         gIOHibernateWakeMap     = hibernate_page_list_grab(map, &nextFree);
 378         gIOHibernateWakeMapSize = header->memoryMapSize;
 379         bcopy_internal(p4, (void  *) ptoa_32(gIOHibernateWakeMap), gIOHibernateWakeMapSize);
 380     }
 381     else
 382         gIOHibernateWakeMapSize = 0;
 383
 384     sum = gIOHibernateCurrentHeader->actualRestore1Sum;
 385     gIOHibernateCurrentHeader->diag[0] = (uint32_t) header;
 386     gIOHibernateCurrentHeader->diag[1] = sum;
 387
 388     uncompressedPages = 0;
 389     conflictCount     = 0;
 390     copyPageListHead  = 0;
 391     copyPageList      = 0;
 392     copyPageIndex     = PAGE_SIZE >> 2;
 393
 394     compressedSize    = PAGE_SIZE;
 395
 396     while (1)
 397     {
 398         if (pageIndexSource)
 399         {
 400             ppnum = pageIndexSource[0];
 401             count = pageIndexSource[1];
 402             pageIndexSource += 2;
 403             if (!count)
 404             {
 405                 pageIndexSource = 0;
 406                 src =  (uint32_t *) (((uint32_t) map) + gIOHibernateCurrentHeader->bitmapSize);
 407                 ppnum = src[0];
 408                 count = src[1];
 409                 src += 2;
 410             }
 411         }
 412         else
 413         {
 414             ppnum = src[0];
 415             count = src[1];
 416             if (!count)
 417                 break;
 418             src += 2;
 419         }
 420
 421         for (page = 0; page < count; page++, ppnum++)
 422         {
 423             uint32_t tag;
 424             int conflicts;
 425
 426             if (!pageIndexSource)
 427             {
 428                 tag = *src++;
 429                 compressedSize = kIOHibernateTagLength & tag;
 430             }
 431
 432 //    SINT(ppnum);
 433
 434             conflicts = (((ppnum >= atop_32(map)) && (ppnum <= lastMapPage))
 435                       || ((ppnum >= atop_32(src)) && (ppnum <= lastImagePage)));
 436
 437             if (pageIndexSource)
 438                 conflicts |= ((ppnum >= atop_32(pageIndexSource)) && (ppnum <= lastPageIndexPage));
 439
 440             if (!conflicts)
 441             {
 442                 if (compressedSize)
 443                     sum += store_one_page(gIOHibernateCurrentHeader->processorFlags,
 444                                             src, compressedSize, buffer, ppnum);
 445                 uncompressedPages++;
 446             }
 447             else
 448             {
 449                 uint32_t   bufferPage;
 450                 uint32_t * dst;
 451
 452                 conflictCount++;
 453
 454                 // alloc new buffer page
 455                 bufferPage = hibernate_page_list_grab(map, &nextFree);
 456
 457                 if (copyPageIndex > ((PAGE_SIZE >> 2) - 3))
 458                 {
 459                     // alloc new copy list page
 460                     uint32_t pageListPage = hibernate_page_list_grab(map, &nextFree);
 461                     // link to current
 462                     if (copyPageList)
 463                         copyPageList[1] = pageListPage;
 464                     else
 465                         copyPageListHead = pageListPage;
 466                     copyPageList = (uint32_t *) ptoa_32(pageListPage);
 467                     copyPageList[1] = 0;
 468                     copyPageIndex = 2;
 469                 }
 470
 471                 copyPageList[copyPageIndex++] = ppnum;
 472                 copyPageList[copyPageIndex++] = bufferPage;
 473                 copyPageList[copyPageIndex++] = compressedSize;
 474                 copyPageList[0] = copyPageIndex;
 475
 476                 dst = (uint32_t *) ptoa_32(bufferPage);
 477                 for (idx = 0; idx < ((compressedSize + 3) >> 2); idx++)
 478                     dst[idx] = src[idx];
 479             }
 480             src += ((compressedSize + 3) >> 2);
 481         }
 482     }
 483
 484     // -- copy back conflicts
 485
 486     copyPageList = (uint32_t *) ptoa_32(copyPageListHead);
 487     while (copyPageList)
 488     {
 489         for (copyPageIndex = 2; copyPageIndex < copyPageList[0]; copyPageIndex += 3)
 490         {
 491             ppnum          =              copyPageList[copyPageIndex + 0];
 492             src            = (uint32_t *) ptoa_32(copyPageList[copyPageIndex + 1]);
 493             compressedSize =              copyPageList[copyPageIndex + 2];
 494
 495             sum += store_one_page(gIOHibernateCurrentHeader->processorFlags,
 496                                     src, compressedSize, buffer, ppnum);
 497             uncompressedPages++;
 498         }
 499         copyPageList = (uint32_t *) ptoa_32(copyPageList[1]);
 500     }
 501
 502     // -- image has been destroyed...
 503
 504     gIOHibernateCurrentHeader->actualImage1Sum         = sum;
 505     gIOHibernateCurrentHeader->actualUncompressedPages = uncompressedPages;
 506     gIOHibernateCurrentHeader->conflictCount           = conflictCount;
 507     gIOHibernateCurrentHeader->nextFree                = nextFree;
 508
 509     gIOHibernateState = kIOHibernateStateWakingFromHibernate;
 510
 511 #if __ppc__
 512     ResetProc proc;
 513     proc = (ResetProc) 0x100;
 514     __asm__ volatile("ori 0, 0, 0" : : );
 515     proc();
 516 #elif __i386__
 517     ResetProc proc;
 518     proc = (ResetProc) acpi_wake_prot_entry;
 519     // flush caches
 520     __asm__("wbinvd");
 521     proc();
 522 #endif
 523
 524     return -1;
 525 }
 526