iokit/Kernel/IOHibernateRestoreKernel.c

   1 /*
   2  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <stdint.h>
  30 #include <mach/mach_types.h>
  31 #include <mach/vm_param.h>
  32 #include <IOKit/IOHibernatePrivate.h>
  33 #include <pexpert/boot.h>
  34 #include <crypto/aes.h>
  35
  36 #include "WKdm.h"
  37 #include "IOHibernateInternal.h"
  38
  39 /*
  40 This code is linked into the kernel but part of the "__HIB" section, which means
  41 its used by code running in the special context of restoring the kernel text and data
  42 from the hibernation image read by the booter. hibernate_kernel_entrypoint() and everything
  43 it calls or references needs to be careful to only touch memory also in the "__HIB" section.
  44 */
  45
  46 uint32_t gIOHibernateState;
  47
  48 static IOHibernateImageHeader _hibernateHeader;
  49 IOHibernateImageHeader * gIOHibernateCurrentHeader = &_hibernateHeader;
  50
  51 static hibernate_graphics_t _hibernateGraphics;
  52 hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
  53
  54 static hibernate_cryptwakevars_t _cryptWakeVars;
  55 hibernate_cryptwakevars_t * gIOHibernateCryptWakeVars = &_cryptWakeVars;
  56
  57 #if __i386__
  58 extern void   acpi_wake_prot_entry(void);
  59 #endif
  60
  61
  62 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  63
  64 #define BASE 65521L /* largest prime smaller than 65536 */
  65 #define NMAX 5000
  66 // NMAX (was 5521) the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1
  67
  68 #define DO1(buf,i)  {s1 += buf[i]; s2 += s1;}
  69 #define DO2(buf,i)  DO1(buf,i); DO1(buf,i+1);
  70 #define DO4(buf,i)  DO2(buf,i); DO2(buf,i+2);
  71 #define DO8(buf,i)  DO4(buf,i); DO4(buf,i+4);
  72 #define DO16(buf)   DO8(buf,0); DO8(buf,8);
  73
  74 uint32_t
  75 hibernate_sum(uint8_t *buf, int32_t len)
  76 {
  77     unsigned long s1 = 1; // adler & 0xffff;
  78     unsigned long s2 = 0; // (adler >> 16) & 0xffff;
  79     int k;
  80
  81     while (len > 0) {
  82         k = len < NMAX ? len : NMAX;
  83         len -= k;
  84         while (k >= 16) {
  85             DO16(buf);
  86             buf += 16;
  87             k -= 16;
  88         }
  89         if (k != 0) do {
  90             s1 += *buf++;
  91             s2 += s1;
  92         } while (--k);
  93         s1 %= BASE;
  94         s2 %= BASE;
  95     }
  96     return (s2 << 16) | s1;
  97 }
  98
  99 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 100
 101 #if __ppc__
 102 static __inline__ unsigned int cntlzw(unsigned int num)
 103 {
 104   unsigned int result;
 105   __asm__ volatile("cntlzw %0, %1" : "=r" (result) : "r" (num));
 106   return result;
 107 }
 108 #elif __i386__
 109 static __inline__ unsigned int cntlzw(unsigned int num)
 110 {
 111     unsigned int result;
 112     __asm__ volatile(   "bsrl   %1, %0\n\t"
 113                         "cmovel %2, %0"
 114                      : "=r" (result)
 115                      : "rm" (num), "r" (63));
 116     return 31 ^ result;
 117 }
 118 #else
 119 #error arch
 120 #endif
 121
 122 void
 123 hibernate_page_bitset(hibernate_page_list_t * list, boolean_t set, uint32_t page)
 124 {
 125     uint32_t             bank;
 126     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 127
 128     for (bank = 0; bank < list->bank_count; bank++)
 129     {
 130         if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
 131         {
 132             page -= bitmap->first_page;
 133             if (set)
 134                 bitmap->bitmap[page >> 5] |= (0x80000000 >> (page & 31));
 135                 //setbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
 136             else
 137                 bitmap->bitmap[page >> 5] &= ~(0x80000000 >> (page & 31));
 138                 //clrbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
 139             break;
 140         }
 141         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 142     }
 143 }
 144
 145 boolean_t
 146 hibernate_page_bittst(hibernate_page_list_t * list, uint32_t page)
 147 {
 148     boolean_t            result = TRUE;
 149     uint32_t             bank;
 150     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 151
 152     for (bank = 0; bank < list->bank_count; bank++)
 153     {
 154         if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
 155         {
 156             page -= bitmap->first_page;
 157             result = (0 != (bitmap->bitmap[page >> 5] & (0x80000000 >> (page & 31))));
 158             break;
 159         }
 160         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 161     }
 162     return (result);
 163 }
 164
 165 // count bits clear or set (set == TRUE) starting at index page.
 166 uint32_t
 167 hibernate_page_list_count(hibernate_page_list_t * list, uint32_t set, uint32_t page)
 168 {
 169     uint32_t                    bank, count;
 170     hibernate_bitmap_t *        bitmap;
 171
 172     bitmap = &list->bank_bitmap[0];
 173     count  = 0;
 174
 175     for (bank = 0; bank < list->bank_count; bank++)
 176     {
 177         // bits between banks are "set"
 178         if (set && (page < bitmap->first_page))
 179         {
 180             count += bitmap->first_page - page;
 181             page  = bitmap->first_page;
 182         }
 183         if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
 184         {
 185             uint32_t index, bit, bits;
 186
 187             index = (page - bitmap->first_page) >> 5;
 188             bit = (page - bitmap->first_page) & 31;
 189
 190             while (TRUE)
 191             {
 192                 bits = bitmap->bitmap[index];
 193                 if (set)
 194                     bits = ~bits;
 195                 bits = (bits << bit);
 196                 count += cntlzw(bits);
 197                 if (bits)
 198                     break;
 199                 count -= bit;
 200
 201                 while (++index < bitmap->bitmapwords)
 202                 {
 203                     bits = bitmap->bitmap[index];
 204                     if (set)
 205                         bits = ~bits;
 206                     count += cntlzw(bits);
 207                     if (bits)
 208                         break;
 209                 }
 210                 if (bits)
 211                     break;
 212                 if (!set)
 213                     break;
 214                 // bits between banks are "set"
 215                 bank++;
 216                 if (bank >= list->bank_count)
 217                     break;
 218                 count -= (bitmap->last_page + 1);
 219                 bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 220                 count += bitmap->first_page;
 221                 index = 0;
 222                 bit = 0;
 223             }
 224             break;
 225         }
 226         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 227     }
 228
 229     return (count);
 230 }
 231
 232
 233 static uint32_t
 234 hibernate_page_list_grab(hibernate_page_list_t * map, uint32_t * _nextFree)
 235 {
 236     uint32_t nextFree = *_nextFree;
 237
 238     if (!nextFree)
 239         nextFree = hibernate_page_list_count(map, 0, 0);
 240
 241     *_nextFree = nextFree + 1 + hibernate_page_list_count(map, 0, nextFree + 1);
 242
 243     return (nextFree);
 244 }
 245
 246 static uint32_t
 247 store_one_page(uint32_t procFlags, uint32_t * src, uint32_t compressedSize,
 248                 uint32_t * buffer, uint32_t ppnum)
 249 {
 250     uint64_t dst;
 251     uint32_t sum;
 252
 253     dst = ptoa_64(ppnum);
 254 #if __ppc__
 255     if (ppnum < 0x00100000)
 256         buffer = (uint32_t *) (uint32_t) dst;
 257 #elif __i386__
 258     if (ppnum < atop_32(0xC0000000)) {
 259         buffer = (uint32_t *) (uint32_t) dst;
 260     }
 261 #endif
 262
 263     if (compressedSize != PAGE_SIZE)
 264     {
 265         WKdm_decompress((WK_word*) src, (WK_word*) buffer, PAGE_SIZE >> 2);
 266         src = buffer;
 267     }
 268
 269     sum = hibernate_sum((uint8_t *) src, PAGE_SIZE);
 270
 271     if (((uint64_t) (uint32_t) src) == dst)
 272         src = 0;
 273
 274     hibernate_restore_phys_page((uint64_t) (uint32_t) src, dst, PAGE_SIZE, procFlags);
 275
 276     return (sum);
 277 }
 278
 279 static void
 280 bcopy_internal(const void *src, void *dst, uint32_t len)
 281 {
 282     const char *s = src;
 283     char       *d = dst;
 284     uint32_t   idx = 0;
 285
 286     while (idx < len)
 287     {
 288         d[idx] = s[idx];
 289         idx++;
 290     }
 291 }
 292
 293 long
 294 hibernate_kernel_entrypoint(IOHibernateImageHeader * header,
 295                             void * p2, void * p3, __unused void * p4)
 296 {
 297     typedef void (*ResetProc)(void);
 298     uint32_t idx;
 299     uint32_t * src;
 300     uint32_t * buffer;
 301     uint32_t * pageIndexSource;
 302     hibernate_page_list_t * map;
 303     uint32_t count;
 304     uint32_t ppnum;
 305     uint32_t page;
 306     uint32_t conflictCount;
 307     uint32_t compressedSize;
 308     uint32_t uncompressedPages;
 309     uint32_t copyPageListHead;
 310     uint32_t * copyPageList;
 311     uint32_t copyPageIndex;
 312     uint32_t sum;
 313     uint32_t nextFree;
 314     uint32_t lastImagePage;
 315     uint32_t lastMapPage;
 316     uint32_t lastPageIndexPage;
 317
 318
 319     bcopy_internal(header,
 320                 gIOHibernateCurrentHeader,
 321                 sizeof(IOHibernateImageHeader));
 322
 323     if (p2)
 324         bcopy_internal(p2,
 325                 gIOHibernateGraphicsInfo,
 326                 sizeof(hibernate_graphics_t));
 327     else
 328         gIOHibernateGraphicsInfo->physicalAddress = gIOHibernateGraphicsInfo->depth = 0;
 329
 330     if (p3)
 331         bcopy_internal(p3,
 332                 gIOHibernateCryptWakeVars,
 333                 sizeof(hibernate_cryptvars_t));
 334
 335     src = (uint32_t *)
 336                 (((uint32_t) &header->fileExtentMap[0])
 337                             + header->fileExtentMapSize
 338                             + ptoa_32(header->restore1PageCount));
 339
 340     if (header->previewSize)
 341     {
 342         pageIndexSource = src;
 343         map = (hibernate_page_list_t *)(((uint32_t) pageIndexSource) + header->previewSize);
 344         src = (uint32_t *) (((uint32_t) pageIndexSource) + header->previewPageListSize);
 345     }
 346     else
 347     {
 348         pageIndexSource = 0;
 349         map = (hibernate_page_list_t *) src;
 350         src = (uint32_t *) (((uint32_t) map) + header->bitmapSize);
 351     }
 352
 353     lastPageIndexPage = atop_32(src);
 354
 355     lastImagePage = atop_32(((uint32_t) header) + header->image1Size);
 356
 357     lastMapPage = atop_32(((uint32_t) map) + header->bitmapSize);
 358
 359     // knock all the image pages to be used out of free map
 360     for (ppnum = atop_32(header); ppnum <= lastImagePage; ppnum++)
 361     {
 362         hibernate_page_bitset(map, FALSE, ppnum);
 363     }
 364
 365     nextFree = 0;
 366     buffer = (uint32_t *) ptoa_32(hibernate_page_list_grab(map, &nextFree));
 367
 368     sum = gIOHibernateCurrentHeader->actualRestore1Sum;
 369     gIOHibernateCurrentHeader->diag[0] = (uint32_t) header;
 370     gIOHibernateCurrentHeader->diag[1] = sum;
 371
 372     uncompressedPages = 0;
 373     conflictCount     = 0;
 374     copyPageListHead  = 0;
 375     copyPageList      = 0;
 376     copyPageIndex     = PAGE_SIZE >> 2;
 377
 378     compressedSize    = PAGE_SIZE;
 379
 380     while (1)
 381     {
 382         if (pageIndexSource)
 383         {
 384             ppnum = pageIndexSource[0];
 385             count = pageIndexSource[1];
 386             pageIndexSource += 2;
 387             if (!count)
 388             {
 389                 pageIndexSource = 0;
 390                 src =  (uint32_t *) (((uint32_t) map) + gIOHibernateCurrentHeader->bitmapSize);
 391                 ppnum = src[0];
 392                 count = src[1];
 393                 src += 2;
 394             }
 395         }
 396         else
 397         {
 398             ppnum = src[0];
 399             count = src[1];
 400             if (!count)
 401                 break;
 402             src += 2;
 403         }
 404
 405         for (page = 0; page < count; page++, ppnum++)
 406         {
 407             uint32_t tag;
 408             int conflicts;
 409
 410             if (!pageIndexSource)
 411             {
 412                 tag = *src++;
 413                 compressedSize = kIOHibernateTagLength & tag;
 414             }
 415
 416             conflicts = (((ppnum >= atop_32(map)) && (ppnum <= lastMapPage))
 417                       || ((ppnum >= atop_32(src)) && (ppnum <= lastImagePage)));
 418
 419             if (pageIndexSource)
 420                 conflicts |= ((ppnum >= atop_32(pageIndexSource)) && (ppnum <= lastPageIndexPage));
 421
 422             if (!conflicts)
 423             {
 424                 if (compressedSize)
 425                     sum += store_one_page(gIOHibernateCurrentHeader->processorFlags,
 426                                             src, compressedSize, buffer, ppnum);
 427                 uncompressedPages++;
 428             }
 429             else
 430             {
 431                 uint32_t   bufferPage;
 432                 uint32_t * dst;
 433
 434                 conflictCount++;
 435
 436                 // alloc new buffer page
 437                 bufferPage = hibernate_page_list_grab(map, &nextFree);
 438
 439                 if (copyPageIndex > ((PAGE_SIZE >> 2) - 3))
 440                 {
 441                     // alloc new copy list page
 442                     uint32_t pageListPage = hibernate_page_list_grab(map, &nextFree);
 443                     // link to current
 444                     if (copyPageList)
 445                         copyPageList[1] = pageListPage;
 446                     else
 447                         copyPageListHead = pageListPage;
 448                     copyPageList = (uint32_t *) ptoa_32(pageListPage);
 449                     copyPageList[1] = 0;
 450                     copyPageIndex = 2;
 451                 }
 452
 453                 copyPageList[copyPageIndex++] = ppnum;
 454                 copyPageList[copyPageIndex++] = bufferPage;
 455                 copyPageList[copyPageIndex++] = compressedSize;
 456                 copyPageList[0] = copyPageIndex;
 457
 458                 dst = (uint32_t *) ptoa_32(bufferPage);
 459                 for (idx = 0; idx < ((compressedSize + 3) >> 2); idx++)
 460                     dst[idx] = src[idx];
 461             }
 462             src += ((compressedSize + 3) >> 2);
 463         }
 464     }
 465
 466     // -- copy back conflicts
 467
 468     copyPageList = (uint32_t *) ptoa_32(copyPageListHead);
 469     while (copyPageList)
 470     {
 471         for (copyPageIndex = 2; copyPageIndex < copyPageList[0]; copyPageIndex += 3)
 472         {
 473             ppnum          =              copyPageList[copyPageIndex + 0];
 474             src            = (uint32_t *) ptoa_32(copyPageList[copyPageIndex + 1]);
 475             compressedSize =              copyPageList[copyPageIndex + 2];
 476
 477             sum += store_one_page(gIOHibernateCurrentHeader->processorFlags,
 478                                     src, compressedSize, buffer, ppnum);
 479             uncompressedPages++;
 480         }
 481         copyPageList = (uint32_t *) ptoa_32(copyPageList[1]);
 482     }
 483
 484     // -- image has been destroyed...
 485
 486     gIOHibernateCurrentHeader->actualImage1Sum         = sum;
 487     gIOHibernateCurrentHeader->actualUncompressedPages = uncompressedPages;
 488     gIOHibernateCurrentHeader->conflictCount           = conflictCount;
 489     gIOHibernateCurrentHeader->nextFree                = nextFree;
 490
 491     gIOHibernateState = kIOHibernateStateWakingFromHibernate;
 492
 493 #if __ppc__
 494     ResetProc proc;
 495     proc = (ResetProc) 0x100;
 496     __asm__ volatile("ori 0, 0, 0" : : );
 497     proc();
 498 #elif __i386__
 499     ResetProc proc;
 500     proc = (ResetProc) acpi_wake_prot_entry;
 501
 502     proc();
 503 #endif
 504
 505     return -1;
 506 }
 507