iokit/Kernel/IOHibernateRestoreKernel.c

   1 /*
   2  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <stdint.h>
  30 #include <mach/mach_types.h>
  31 #include <mach/vm_param.h>
  32 #include <IOKit/IOHibernatePrivate.h>
  33 #include <IOKit/IOLib.h>
  34 #include <pexpert/boot.h>
  35 #include <crypto/aes.h>
  36
  37 #include "WKdm.h"
  38 #include "IOHibernateInternal.h"
  39
  40 /*
  41 This code is linked into the kernel but part of the "__HIB" section, which means
  42 its used by code running in the special context of restoring the kernel text and data
  43 from the hibernation image read by the booter. hibernate_kernel_entrypoint() and everything
  44 it calls or references needs to be careful to only touch memory also in the "__HIB" section.
  45 */
  46
  47 uint32_t gIOHibernateState;
  48
  49 static IOHibernateImageHeader _hibernateHeader;
  50 IOHibernateImageHeader * gIOHibernateCurrentHeader = &_hibernateHeader;
  51
  52 static hibernate_graphics_t _hibernateGraphics;
  53 hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
  54
  55 static hibernate_cryptwakevars_t _cryptWakeVars;
  56 hibernate_cryptwakevars_t * gIOHibernateCryptWakeVars = &_cryptWakeVars;
  57
  58 vm_offset_t gIOHibernateWakeMap;            // ppnum
  59 vm_size_t   gIOHibernateWakeMapSize;
  60
  61 #if __i386__
  62 extern void   acpi_wake_prot_entry(void);
  63 #endif
  64
  65 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  66
  67 #define BASE 65521L /* largest prime smaller than 65536 */
  68 #define NMAX 5000
  69 // NMAX (was 5521) the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1
  70
  71 #define DO1(buf,i)  {s1 += buf[i]; s2 += s1;}
  72 #define DO2(buf,i)  DO1(buf,i); DO1(buf,i+1);
  73 #define DO4(buf,i)  DO2(buf,i); DO2(buf,i+2);
  74 #define DO8(buf,i)  DO4(buf,i); DO4(buf,i+4);
  75 #define DO16(buf)   DO8(buf,0); DO8(buf,8);
  76
  77 uint32_t
  78 hibernate_sum(uint8_t *buf, int32_t len)
  79 {
  80     unsigned long s1 = 1; // adler & 0xffff;
  81     unsigned long s2 = 0; // (adler >> 16) & 0xffff;
  82     int k;
  83
  84     while (len > 0) {
  85         k = len < NMAX ? len : NMAX;
  86         len -= k;
  87         while (k >= 16) {
  88             DO16(buf);
  89             buf += 16;
  90             k -= 16;
  91         }
  92         if (k != 0) do {
  93             s1 += *buf++;
  94             s2 += s1;
  95         } while (--k);
  96         s1 %= BASE;
  97         s2 %= BASE;
  98     }
  99     return (s2 << 16) | s1;
 100 }
 101
 102 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 103
 104 static hibernate_bitmap_t *
 105 hibernate_page_bitmap(hibernate_page_list_t * list, uint32_t page)
 106 {
 107     uint32_t             bank;
 108     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 109
 110     for (bank = 0; bank < list->bank_count; bank++)
 111     {
 112         if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
 113             break;
 114         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 115     }
 116     if (bank == list->bank_count)
 117         bitmap = 0;
 118
 119     return (bitmap);
 120 }
 121
 122 hibernate_bitmap_t *
 123 hibernate_page_bitmap_pin(hibernate_page_list_t * list, uint32_t * pPage)
 124 {
 125     uint32_t             bank, page = *pPage;
 126     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 127
 128     for (bank = 0; bank < list->bank_count; bank++)
 129     {
 130         if (page <= bitmap->first_page)
 131         {
 132             *pPage = bitmap->first_page;
 133             break;
 134         }
 135         if (page <= bitmap->last_page)
 136             break;
 137         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 138     }
 139     if (bank == list->bank_count)
 140         bitmap = 0;
 141
 142     return (bitmap);
 143 }
 144
 145 void
 146 hibernate_page_bitset(hibernate_page_list_t * list, boolean_t set, uint32_t page)
 147 {
 148     hibernate_bitmap_t * bitmap;
 149
 150     bitmap = hibernate_page_bitmap(list, page);
 151     if (bitmap)
 152     {
 153         page -= bitmap->first_page;
 154         if (set)
 155             bitmap->bitmap[page >> 5] |= (0x80000000 >> (page & 31));
 156             //setbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
 157         else
 158             bitmap->bitmap[page >> 5] &= ~(0x80000000 >> (page & 31));
 159             //clrbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
 160     }
 161 }
 162
 163 boolean_t
 164 hibernate_page_bittst(hibernate_page_list_t * list, uint32_t page)
 165 {
 166     boolean_t            result = TRUE;
 167     hibernate_bitmap_t * bitmap;
 168
 169     bitmap = hibernate_page_bitmap(list, page);
 170     if (bitmap)
 171     {
 172         page -= bitmap->first_page;
 173         result = (0 != (bitmap->bitmap[page >> 5] & (0x80000000 >> (page & 31))));
 174     }
 175     return (result);
 176 }
 177
 178 // count bits clear or set (set == TRUE) starting at page.
 179 uint32_t
 180 hibernate_page_bitmap_count(hibernate_bitmap_t * bitmap, uint32_t set, uint32_t page)
 181 {
 182     uint32_t index, bit, bits;
 183     uint32_t count;
 184
 185     count = 0;
 186
 187     index = (page - bitmap->first_page) >> 5;
 188     bit = (page - bitmap->first_page) & 31;
 189
 190     bits = bitmap->bitmap[index];
 191     if (set)
 192         bits = ~bits;
 193     bits = (bits << bit);
 194     if (bits)
 195         count += __builtin_clz(bits);
 196     else
 197     {
 198         count += 32 - bit;
 199         while (++index < bitmap->bitmapwords)
 200         {
 201             bits = bitmap->bitmap[index];
 202             if (set)
 203                 bits = ~bits;
 204             if (bits)
 205             {
 206                 count += __builtin_clz(bits);
 207                 break;
 208             }
 209             count += 32;
 210         }
 211     }
 212
 213     return (count);
 214 }
 215
 216 static vm_offset_t
 217 hibernate_page_list_grab(hibernate_page_list_t * list, uint32_t * pNextFree)
 218 {
 219     uint32_t             nextFree = *pNextFree;
 220     uint32_t             nextFreeInBank;
 221     hibernate_bitmap_t * bitmap;
 222
 223     nextFreeInBank = nextFree + 1;
 224     while ((bitmap = hibernate_page_bitmap_pin(list, &nextFreeInBank)))
 225     {
 226         nextFreeInBank += hibernate_page_bitmap_count(bitmap, FALSE, nextFreeInBank);
 227         if (nextFreeInBank <= bitmap->last_page)
 228         {
 229             *pNextFree = nextFreeInBank;
 230             break;
 231         }
 232     }
 233
 234     if (!bitmap)
 235         IOPanic(__FUNCTION__);
 236
 237     return (nextFree);
 238 }
 239
 240 static uint32_t
 241 store_one_page(uint32_t procFlags, uint32_t * src, uint32_t compressedSize,
 242                 uint32_t * buffer, uint32_t ppnum)
 243 {
 244     uint64_t dst;
 245     uint32_t sum;
 246
 247     dst = ptoa_64(ppnum);
 248     if (ppnum < 0x00100000)
 249         buffer = (uint32_t *) (uint32_t) dst;
 250
 251     if (compressedSize != PAGE_SIZE)
 252     {
 253         WKdm_decompress((WK_word*) src, (WK_word*) buffer, PAGE_SIZE >> 2);
 254         src = buffer;
 255     }
 256
 257     sum = hibernate_sum((uint8_t *) src, PAGE_SIZE);
 258
 259     if (((uint64_t) (uint32_t) src) == dst)
 260         src = 0;
 261
 262     hibernate_restore_phys_page((uint64_t) (uint32_t) src, dst, PAGE_SIZE, procFlags);
 263
 264     return (sum);
 265 }
 266
 267 static void
 268 bcopy_internal(const void *src, void *dst, uint32_t len)
 269 {
 270     const char *s = src;
 271     char       *d = dst;
 272     uint32_t   idx = 0;
 273
 274     while (idx < len)
 275     {
 276         d[idx] = s[idx];
 277         idx++;
 278     }
 279 }
 280
 281 #define C_ASSERT(e) typedef char    __C_ASSERT__[(e) ? 1 : -1]
 282
 283 long
 284 hibernate_kernel_entrypoint(IOHibernateImageHeader * header,
 285                             void * p2, void * p3, void * p4)
 286 {
 287     typedef void (*ResetProc)(void);
 288     uint32_t idx;
 289     uint32_t * src;
 290     uint32_t * buffer;
 291     uint32_t * pageIndexSource;
 292     hibernate_page_list_t * map;
 293     uint32_t count;
 294     uint32_t ppnum;
 295     uint32_t page;
 296     uint32_t conflictCount;
 297     uint32_t compressedSize;
 298     uint32_t uncompressedPages;
 299     uint32_t copyPageListHead;
 300     uint32_t * copyPageList;
 301     uint32_t copyPageIndex;
 302     uint32_t sum;
 303     uint32_t nextFree;
 304     uint32_t lastImagePage;
 305     uint32_t lastMapPage;
 306     uint32_t lastPageIndexPage;
 307
 308     C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
 309
 310     bcopy_internal(header,
 311                 gIOHibernateCurrentHeader,
 312                 sizeof(IOHibernateImageHeader));
 313
 314     if (!p2)
 315     {
 316         count = header->graphicsInfoOffset;
 317         if (count)
 318             p2 = (void *)(((uintptr_t) header) - count);
 319     }
 320     if (p2)
 321         bcopy_internal(p2,
 322                 gIOHibernateGraphicsInfo,
 323                 sizeof(hibernate_graphics_t));
 324     else
 325         gIOHibernateGraphicsInfo->physicalAddress = gIOHibernateGraphicsInfo->depth = 0;
 326
 327     if (!p3)
 328     {
 329         count = header->cryptVarsOffset;
 330         if (count)
 331             p3 = (void *)(((uintptr_t) header) - count);
 332     }
 333     if (p3)
 334         bcopy_internal(p3,
 335                 gIOHibernateCryptWakeVars,
 336                 sizeof(hibernate_cryptvars_t));
 337
 338     src = (uint32_t *)
 339                 (((uint32_t) &header->fileExtentMap[0])
 340                             + header->fileExtentMapSize
 341                             + ptoa_32(header->restore1PageCount));
 342
 343     if (header->previewSize)
 344     {
 345         pageIndexSource = src;
 346         map = (hibernate_page_list_t *)(((uint32_t) pageIndexSource) + header->previewSize);
 347         src = (uint32_t *) (((uint32_t) pageIndexSource) + header->previewPageListSize);
 348     }
 349     else
 350     {
 351         pageIndexSource = 0;
 352         map = (hibernate_page_list_t *) src;
 353         src = (uint32_t *) (((uint32_t) map) + header->bitmapSize);
 354     }
 355
 356     lastPageIndexPage = atop_32(src);
 357
 358     lastImagePage = atop_32(((uint32_t) header) + header->image1Size);
 359
 360     lastMapPage = atop_32(((uint32_t) map) + header->bitmapSize);
 361
 362     // knock all the image pages to be used out of free map
 363     for (ppnum = atop_32(header); ppnum <= lastImagePage; ppnum++)
 364     {
 365         hibernate_page_bitset(map, FALSE, ppnum);
 366     }
 367
 368     nextFree = 0;
 369     hibernate_page_list_grab(map, &nextFree);
 370     buffer = (uint32_t *) ptoa_32(hibernate_page_list_grab(map, &nextFree));
 371
 372     if (header->memoryMapSize && (count = header->memoryMapOffset))
 373     {
 374         p4 = (void *)(((uintptr_t) header) - count);
 375         gIOHibernateWakeMap     = hibernate_page_list_grab(map, &nextFree);
 376         gIOHibernateWakeMapSize = header->memoryMapSize;
 377         bcopy_internal(p4, (void  *) ptoa_32(gIOHibernateWakeMap), gIOHibernateWakeMapSize);
 378     }
 379     else
 380         gIOHibernateWakeMapSize = 0;
 381
 382     sum = gIOHibernateCurrentHeader->actualRestore1Sum;
 383     gIOHibernateCurrentHeader->diag[0] = (uint32_t) header;
 384     gIOHibernateCurrentHeader->diag[1] = sum;
 385
 386     uncompressedPages = 0;
 387     conflictCount     = 0;
 388     copyPageListHead  = 0;
 389     copyPageList      = 0;
 390     copyPageIndex     = PAGE_SIZE >> 2;
 391
 392     compressedSize    = PAGE_SIZE;
 393
 394     while (1)
 395     {
 396         if (pageIndexSource)
 397         {
 398             ppnum = pageIndexSource[0];
 399             count = pageIndexSource[1];
 400             pageIndexSource += 2;
 401             if (!count)
 402             {
 403                 pageIndexSource = 0;
 404                 src =  (uint32_t *) (((uint32_t) map) + gIOHibernateCurrentHeader->bitmapSize);
 405                 ppnum = src[0];
 406                 count = src[1];
 407                 src += 2;
 408             }
 409         }
 410         else
 411         {
 412             ppnum = src[0];
 413             count = src[1];
 414             if (!count)
 415                 break;
 416             src += 2;
 417         }
 418
 419         for (page = 0; page < count; page++, ppnum++)
 420         {
 421             uint32_t tag;
 422             int conflicts;
 423
 424             if (!pageIndexSource)
 425             {
 426                 tag = *src++;
 427                 compressedSize = kIOHibernateTagLength & tag;
 428             }
 429
 430 //    SINT(ppnum);
 431
 432             conflicts = (((ppnum >= atop_32(map)) && (ppnum <= lastMapPage))
 433                       || ((ppnum >= atop_32(src)) && (ppnum <= lastImagePage)));
 434
 435             if (pageIndexSource)
 436                 conflicts |= ((ppnum >= atop_32(pageIndexSource)) && (ppnum <= lastPageIndexPage));
 437
 438             if (!conflicts)
 439             {
 440                 if (compressedSize)
 441                     sum += store_one_page(gIOHibernateCurrentHeader->processorFlags,
 442                                             src, compressedSize, buffer, ppnum);
 443                 uncompressedPages++;
 444             }
 445             else
 446             {
 447                 uint32_t   bufferPage;
 448                 uint32_t * dst;
 449
 450                 conflictCount++;
 451
 452                 // alloc new buffer page
 453                 bufferPage = hibernate_page_list_grab(map, &nextFree);
 454
 455                 if (copyPageIndex > ((PAGE_SIZE >> 2) - 3))
 456                 {
 457                     // alloc new copy list page
 458                     uint32_t pageListPage = hibernate_page_list_grab(map, &nextFree);
 459                     // link to current
 460                     if (copyPageList)
 461                         copyPageList[1] = pageListPage;
 462                     else
 463                         copyPageListHead = pageListPage;
 464                     copyPageList = (uint32_t *) ptoa_32(pageListPage);
 465                     copyPageList[1] = 0;
 466                     copyPageIndex = 2;
 467                 }
 468
 469                 copyPageList[copyPageIndex++] = ppnum;
 470                 copyPageList[copyPageIndex++] = bufferPage;
 471                 copyPageList[copyPageIndex++] = compressedSize;
 472                 copyPageList[0] = copyPageIndex;
 473
 474                 dst = (uint32_t *) ptoa_32(bufferPage);
 475                 for (idx = 0; idx < ((compressedSize + 3) >> 2); idx++)
 476                     dst[idx] = src[idx];
 477             }
 478             src += ((compressedSize + 3) >> 2);
 479         }
 480     }
 481
 482     // -- copy back conflicts
 483
 484     copyPageList = (uint32_t *) ptoa_32(copyPageListHead);
 485     while (copyPageList)
 486     {
 487         for (copyPageIndex = 2; copyPageIndex < copyPageList[0]; copyPageIndex += 3)
 488         {
 489             ppnum          =              copyPageList[copyPageIndex + 0];
 490             src            = (uint32_t *) ptoa_32(copyPageList[copyPageIndex + 1]);
 491             compressedSize =              copyPageList[copyPageIndex + 2];
 492
 493             sum += store_one_page(gIOHibernateCurrentHeader->processorFlags,
 494                                     src, compressedSize, buffer, ppnum);
 495             uncompressedPages++;
 496         }
 497         copyPageList = (uint32_t *) ptoa_32(copyPageList[1]);
 498     }
 499
 500     // -- image has been destroyed...
 501
 502     gIOHibernateCurrentHeader->actualImage1Sum         = sum;
 503     gIOHibernateCurrentHeader->actualUncompressedPages = uncompressedPages;
 504     gIOHibernateCurrentHeader->conflictCount           = conflictCount;
 505     gIOHibernateCurrentHeader->nextFree                = nextFree;
 506
 507     gIOHibernateState = kIOHibernateStateWakingFromHibernate;
 508
 509 #if __ppc__
 510     ResetProc proc;
 511     proc = (ResetProc) 0x100;
 512     __asm__ volatile("ori 0, 0, 0" : : );
 513     proc();
 514 #elif __i386__
 515     ResetProc proc;
 516     proc = (ResetProc) acpi_wake_prot_entry;
 517     // flush caches
 518     __asm__("wbinvd");
 519     proc();
 520 #endif
 521
 522     return -1;
 523 }
 524