iokit/Kernel/IOHibernateRestoreKernel.c

   1 /*
   2  * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <stdint.h>
  30 #include <mach/mach_types.h>
  31 #include <mach/vm_param.h>
  32 #include <IOKit/IOHibernatePrivate.h>
  33 #include <IOKit/IOLib.h>
  34 #include <pexpert/boot.h>
  35 #include <crypto/aes.h>
  36 #include <libkern/libkern.h>
  37
  38 #include "WKdm.h"
  39 #include "IOHibernateInternal.h"
  40
  41 /*
  42 This code is linked into the kernel but part of the "__HIB" section, which means
  43 its used by code running in the special context of restoring the kernel text and data
  44 from the hibernation image read by the booter. hibernate_kernel_entrypoint() and everything
  45 it calls or references needs to be careful to only touch memory also in the "__HIB" section.
  46 */
  47
  48 uint32_t gIOHibernateState;
  49
  50 static IOHibernateImageHeader _hibernateHeader;
  51 IOHibernateImageHeader * gIOHibernateCurrentHeader = &_hibernateHeader;
  52
  53 static hibernate_graphics_t _hibernateGraphics;
  54 hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
  55
  56 static hibernate_cryptwakevars_t _cryptWakeVars;
  57 hibernate_cryptwakevars_t * gIOHibernateCryptWakeVars = &_cryptWakeVars;
  58
  59 vm_offset_t gIOHibernateWakeMap;            // ppnum
  60 vm_size_t   gIOHibernateWakeMapSize;
  61
  62 #if __i386__
  63 extern void   acpi_wake_prot_entry(void);
  64 #endif
  65
  66 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  67
  68 #define BASE 65521L /* largest prime smaller than 65536 */
  69 #define NMAX 5000
  70 // NMAX (was 5521) the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1
  71
  72 #define DO1(buf,i)  {s1 += buf[i]; s2 += s1;}
  73 #define DO2(buf,i)  DO1(buf,i); DO1(buf,i+1);
  74 #define DO4(buf,i)  DO2(buf,i); DO2(buf,i+2);
  75 #define DO8(buf,i)  DO4(buf,i); DO4(buf,i+4);
  76 #define DO16(buf)   DO8(buf,0); DO8(buf,8);
  77
  78 uint32_t
  79 hibernate_sum(uint8_t *buf, int32_t len)
  80 {
  81     unsigned long s1 = 1; // adler & 0xffff;
  82     unsigned long s2 = 0; // (adler >> 16) & 0xffff;
  83     int k;
  84
  85     while (len > 0) {
  86         k = len < NMAX ? len : NMAX;
  87         len -= k;
  88         while (k >= 16) {
  89             DO16(buf);
  90             buf += 16;
  91             k -= 16;
  92         }
  93         if (k != 0) do {
  94             s1 += *buf++;
  95             s2 += s1;
  96         } while (--k);
  97         s1 %= BASE;
  98         s2 %= BASE;
  99     }
 100     return (s2 << 16) | s1;
 101 }
 102
 103 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 104
 105 static __inline__ unsigned int cntlzw(unsigned int num)
 106 {
 107         return clz(num);
 108 }
 109
 110 static hibernate_bitmap_t *
 111 hibernate_page_bitmap(hibernate_page_list_t * list, uint32_t page)
 112 {
 113     uint32_t             bank;
 114     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 115
 116     for (bank = 0; bank < list->bank_count; bank++)
 117     {
 118         if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
 119             break;
 120         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 121     }
 122     if (bank == list->bank_count)
 123         bitmap = NULL;
 124
 125     return (bitmap);
 126 }
 127
 128 hibernate_bitmap_t *
 129 hibernate_page_bitmap_pin(hibernate_page_list_t * list, uint32_t * pPage)
 130 {
 131     uint32_t             bank, page = *pPage;
 132     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 133
 134     for (bank = 0; bank < list->bank_count; bank++)
 135     {
 136         if (page <= bitmap->first_page)
 137         {
 138             *pPage = bitmap->first_page;
 139             break;
 140         }
 141         if (page <= bitmap->last_page)
 142             break;
 143         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 144     }
 145     if (bank == list->bank_count)
 146         bitmap = NULL;
 147
 148     return (bitmap);
 149 }
 150
 151 void
 152 hibernate_page_bitset(hibernate_page_list_t * list, boolean_t set, uint32_t page)
 153 {
 154     hibernate_bitmap_t * bitmap;
 155
 156     bitmap = hibernate_page_bitmap(list, page);
 157     if (bitmap)
 158     {
 159         page -= bitmap->first_page;
 160         if (set)
 161             bitmap->bitmap[page >> 5] |= (0x80000000 >> (page & 31));
 162             //setbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
 163         else
 164             bitmap->bitmap[page >> 5] &= ~(0x80000000 >> (page & 31));
 165             //clrbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
 166     }
 167 }
 168
 169 boolean_t
 170 hibernate_page_bittst(hibernate_page_list_t * list, uint32_t page)
 171 {
 172     boolean_t            result = TRUE;
 173     hibernate_bitmap_t * bitmap;
 174
 175     bitmap = hibernate_page_bitmap(list, page);
 176     if (bitmap)
 177     {
 178         page -= bitmap->first_page;
 179         result = (0 != (bitmap->bitmap[page >> 5] & (0x80000000 >> (page & 31))));
 180     }
 181     return (result);
 182 }
 183
 184 // count bits clear or set (set == TRUE) starting at page.
 185 uint32_t
 186 hibernate_page_bitmap_count(hibernate_bitmap_t * bitmap, uint32_t set, uint32_t page)
 187 {
 188     uint32_t index, bit, bits;
 189     uint32_t count;
 190
 191     count = 0;
 192
 193     index = (page - bitmap->first_page) >> 5;
 194     bit = (page - bitmap->first_page) & 31;
 195
 196     bits = bitmap->bitmap[index];
 197     if (set)
 198         bits = ~bits;
 199     bits = (bits << bit);
 200     if (bits)
 201         count += __builtin_clz(bits);
 202     else
 203     {
 204         count += 32 - bit;
 205         while (++index < bitmap->bitmapwords)
 206         {
 207             bits = bitmap->bitmap[index];
 208             if (set)
 209                 bits = ~bits;
 210             if (bits)
 211             {
 212                 count += __builtin_clz(bits);
 213                 break;
 214             }
 215             count += 32;
 216         }
 217     }
 218
 219     return (count);
 220 }
 221
 222 static vm_offset_t
 223 hibernate_page_list_grab(hibernate_page_list_t * list, uint32_t * pNextFree)
 224 {
 225     uint32_t             nextFree = *pNextFree;
 226     uint32_t             nextFreeInBank;
 227     hibernate_bitmap_t * bitmap;
 228
 229     nextFreeInBank = nextFree + 1;
 230     while ((bitmap = hibernate_page_bitmap_pin(list, &nextFreeInBank)))
 231     {
 232         nextFreeInBank += hibernate_page_bitmap_count(bitmap, FALSE, nextFreeInBank);
 233         if (nextFreeInBank <= bitmap->last_page)
 234         {
 235             *pNextFree = nextFreeInBank;
 236             break;
 237         }
 238     }
 239
 240     return (nextFree);
 241 }
 242
 243 static uint32_t
 244 store_one_page(uint32_t procFlags, uint32_t * src, uint32_t compressedSize,
 245                 uint32_t * buffer, uint32_t ppnum)
 246 {
 247     uint64_t dst;
 248     uint32_t sum;
 249
 250     dst = ptoa_64(ppnum);
 251     if (ppnum < 0x00100000)
 252         buffer = (uint32_t *) (uint32_t) dst;
 253
 254     if (compressedSize != PAGE_SIZE)
 255     {
 256         WKdm_decompress((WK_word*) src, (WK_word*) buffer, PAGE_SIZE >> 2);
 257         src = buffer;
 258     }
 259
 260     sum = hibernate_sum((uint8_t *) src, PAGE_SIZE);
 261
 262     if (((uint64_t) (uint32_t) src) == dst)
 263         src = 0;
 264
 265     hibernate_restore_phys_page((uint64_t) (uint32_t) src, dst, PAGE_SIZE, procFlags);
 266
 267     return (sum);
 268 }
 269
 270 static void
 271 bcopy_internal(const void *src, void *dst, uint32_t len)
 272 {
 273     const char *s = src;
 274     char       *d = dst;
 275     uint32_t   idx = 0;
 276
 277     while (idx < len)
 278     {
 279         d[idx] = s[idx];
 280         idx++;
 281     }
 282 }
 283
 284 #define C_ASSERT(e) typedef char    __C_ASSERT__[(e) ? 1 : -1]
 285
 286 long
 287 hibernate_kernel_entrypoint(IOHibernateImageHeader * header,
 288                             void * p2, void * p3, void * p4)
 289 {
 290     typedef void (*ResetProc)(void);
 291     uint32_t idx;
 292     uint32_t * src;
 293     uint32_t * buffer;
 294     uint32_t * pageIndexSource;
 295     hibernate_page_list_t * map;
 296     uint32_t count;
 297     uint32_t ppnum;
 298     uint32_t page;
 299     uint32_t conflictCount;
 300     uint32_t compressedSize;
 301     uint32_t uncompressedPages;
 302     uint32_t copyPageListHead;
 303     uint32_t * copyPageList;
 304     uint32_t copyPageIndex;
 305     uint32_t sum;
 306     uint32_t nextFree;
 307     uint32_t lastImagePage;
 308     uint32_t lastMapPage;
 309     uint32_t lastPageIndexPage;
 310     ResetProc proc;
 311
 312     C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
 313
 314     bcopy_internal(header,
 315                 gIOHibernateCurrentHeader,
 316                 sizeof(IOHibernateImageHeader));
 317
 318     if (!p2)
 319     {
 320         count = header->graphicsInfoOffset;
 321         if (count)
 322             p2 = (void *)(((uintptr_t) header) - count);
 323     }
 324     if (p2)
 325         bcopy_internal(p2,
 326                 gIOHibernateGraphicsInfo,
 327                 sizeof(hibernate_graphics_t));
 328     else
 329         gIOHibernateGraphicsInfo->physicalAddress = gIOHibernateGraphicsInfo->depth = 0;
 330
 331     if (!p3)
 332     {
 333         count = header->cryptVarsOffset;
 334         if (count)
 335             p3 = (void *)(((uintptr_t) header) - count);
 336     }
 337     if (p3)
 338         bcopy_internal(p3,
 339                 gIOHibernateCryptWakeVars,
 340                 sizeof(hibernate_cryptvars_t));
 341
 342     src = (uint32_t *)
 343                 (((uint32_t) &header->fileExtentMap[0])
 344                             + header->fileExtentMapSize
 345                             + ptoa_32(header->restore1PageCount));
 346
 347     if (header->previewSize)
 348     {
 349         pageIndexSource = src;
 350         map = (hibernate_page_list_t *)(((uint32_t) pageIndexSource) + header->previewSize);
 351         src = (uint32_t *) (((uint32_t) pageIndexSource) + header->previewPageListSize);
 352     }
 353     else
 354     {
 355         pageIndexSource = 0;
 356         map = (hibernate_page_list_t *) src;
 357         src = (uint32_t *) (((uint32_t) map) + header->bitmapSize);
 358     }
 359
 360     lastPageIndexPage = atop_32(src);
 361
 362     lastImagePage = atop_32(((uint32_t) header) + header->image1Size);
 363
 364     lastMapPage = atop_32(((uint32_t) map) + header->bitmapSize);
 365
 366     // knock all the image pages to be used out of free map
 367     for (ppnum = atop_32(header); ppnum <= lastImagePage; ppnum++)
 368     {
 369         hibernate_page_bitset(map, FALSE, ppnum);
 370     }
 371
 372     nextFree = 0;
 373     hibernate_page_list_grab(map, &nextFree);
 374     buffer = (uint32_t *) ptoa_32(hibernate_page_list_grab(map, &nextFree));
 375
 376     if (header->memoryMapSize && (count = header->memoryMapOffset))
 377     {
 378         p4 = (void *)(((uintptr_t) header) - count);
 379         gIOHibernateWakeMap     = hibernate_page_list_grab(map, &nextFree);
 380         gIOHibernateWakeMapSize = header->memoryMapSize;
 381         bcopy_internal(p4, (void  *) ptoa_32(gIOHibernateWakeMap), gIOHibernateWakeMapSize);
 382     }
 383     else
 384         gIOHibernateWakeMapSize = 0;
 385
 386     sum = gIOHibernateCurrentHeader->actualRestore1Sum;
 387     gIOHibernateCurrentHeader->diag[0] = (uint32_t) header;
 388     gIOHibernateCurrentHeader->diag[1] = sum;
 389
 390     uncompressedPages = 0;
 391     conflictCount     = 0;
 392     copyPageListHead  = 0;
 393     copyPageList      = 0;
 394     copyPageIndex     = PAGE_SIZE >> 2;
 395
 396     compressedSize    = PAGE_SIZE;
 397
 398     while (1)
 399     {
 400         if (pageIndexSource)
 401         {
 402             ppnum = pageIndexSource[0];
 403             count = pageIndexSource[1];
 404             pageIndexSource += 2;
 405             if (!count)
 406             {
 407                 pageIndexSource = 0;
 408                 src =  (uint32_t *) (((uint32_t) map) + gIOHibernateCurrentHeader->bitmapSize);
 409                 ppnum = src[0];
 410                 count = src[1];
 411                 src += 2;
 412             }
 413         }
 414         else
 415         {
 416             ppnum = src[0];
 417             count = src[1];
 418             if (!count)
 419                 break;
 420             src += 2;
 421         }
 422
 423         for (page = 0; page < count; page++, ppnum++)
 424         {
 425             uint32_t tag;
 426             int conflicts;
 427
 428             if (!pageIndexSource)
 429             {
 430                 tag = *src++;
 431                 compressedSize = kIOHibernateTagLength & tag;
 432             }
 433
 434 //    SINT(ppnum);
 435
 436             conflicts = (((ppnum >= atop_32(map)) && (ppnum <= lastMapPage))
 437                       || ((ppnum >= atop_32(src)) && (ppnum <= lastImagePage)));
 438
 439             if (pageIndexSource)
 440                 conflicts |= ((ppnum >= atop_32(pageIndexSource)) && (ppnum <= lastPageIndexPage));
 441
 442             if (!conflicts)
 443             {
 444                 if (compressedSize)
 445                     sum += store_one_page(gIOHibernateCurrentHeader->processorFlags,
 446                                             src, compressedSize, buffer, ppnum);
 447                 uncompressedPages++;
 448             }
 449             else
 450             {
 451                 uint32_t   bufferPage;
 452                 uint32_t * dst;
 453
 454                 conflictCount++;
 455
 456                 // alloc new buffer page
 457                 bufferPage = hibernate_page_list_grab(map, &nextFree);
 458
 459                 if (copyPageIndex > ((PAGE_SIZE >> 2) - 3))
 460                 {
 461                     // alloc new copy list page
 462                     uint32_t pageListPage = hibernate_page_list_grab(map, &nextFree);
 463                     // link to current
 464                     if (copyPageList)
 465                         copyPageList[1] = pageListPage;
 466                     else
 467                         copyPageListHead = pageListPage;
 468                     copyPageList = (uint32_t *) ptoa_32(pageListPage);
 469                     copyPageList[1] = 0;
 470                     copyPageIndex = 2;
 471                 }
 472
 473                 copyPageList[copyPageIndex++] = ppnum;
 474                 copyPageList[copyPageIndex++] = bufferPage;
 475                 copyPageList[copyPageIndex++] = compressedSize;
 476                 copyPageList[0] = copyPageIndex;
 477
 478                 dst = (uint32_t *) ptoa_32(bufferPage);
 479                 for (idx = 0; idx < ((compressedSize + 3) >> 2); idx++)
 480                     dst[idx] = src[idx];
 481             }
 482             src += ((compressedSize + 3) >> 2);
 483         }
 484     }
 485
 486     // -- copy back conflicts
 487
 488     copyPageList = (uint32_t *) ptoa_32(copyPageListHead);
 489     while (copyPageList)
 490     {
 491         for (copyPageIndex = 2; copyPageIndex < copyPageList[0]; copyPageIndex += 3)
 492         {
 493             ppnum          =              copyPageList[copyPageIndex + 0];
 494             src            = (uint32_t *) ptoa_32(copyPageList[copyPageIndex + 1]);
 495             compressedSize =              copyPageList[copyPageIndex + 2];
 496
 497             sum += store_one_page(gIOHibernateCurrentHeader->processorFlags,
 498                                     src, compressedSize, buffer, ppnum);
 499             uncompressedPages++;
 500         }
 501         copyPageList = (uint32_t *) ptoa_32(copyPageList[1]);
 502     }
 503
 504     // -- image has been destroyed...
 505
 506     gIOHibernateCurrentHeader->actualImage1Sum         = sum;
 507     gIOHibernateCurrentHeader->actualUncompressedPages = uncompressedPages;
 508     gIOHibernateCurrentHeader->conflictCount           = conflictCount;
 509     gIOHibernateCurrentHeader->nextFree                = nextFree;
 510
 511     gIOHibernateState = kIOHibernateStateWakingFromHibernate;
 512
 513 #if __ppc__
 514     proc = (ResetProc) 0x100;
 515     __asm__ volatile("ori 0, 0, 0" : : );
 516     proc();
 517 #elif __i386__
 518     proc = (ResetProc) acpi_wake_prot_entry;
 519     // flush caches
 520     __asm__("wbinvd");
 521     proc();
 522 #elif __arm__
 523     proc = (ResetProc)0x00000000;
 524     proc();
 525 #endif
 526
 527     return -1;
 528 }