iokit/Kernel/IOHibernateRestoreKernel.c

   1 /*
   2  * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <stdint.h>
  30 #include <mach/mach_types.h>
  31 #include <mach/vm_param.h>
  32 #include <IOKit/IOHibernatePrivate.h>
  33 #include <IOKit/IOLib.h>
  34 #include <pexpert/boot.h>
  35 #include <crypto/aes.h>
  36 #include <libkern/libkern.h>
  37
  38 #include <libkern/WKdm.h>
  39 #include "IOHibernateInternal.h"
  40
  41 #if defined(__i386__) || defined(__x86_64__)
  42 #include <i386/pal_hibernate.h>
  43 #endif
  44
  45 /*
  46 This code is linked into the kernel but part of the "__HIB" section, which means
  47 its used by code running in the special context of restoring the kernel text and data
  48 from the hibernation image read by the booter. hibernate_kernel_entrypoint() and everything
  49 it calls or references needs to be careful to only touch memory also in the "__HIB" section.
  50 */
  51
  52 uint32_t gIOHibernateState;
  53
  54 uint32_t gIOHibernateDebugFlags;
  55
  56 static IOHibernateImageHeader _hibernateHeader;
  57 IOHibernateImageHeader * gIOHibernateCurrentHeader = &_hibernateHeader;
  58
  59 ppnum_t gIOHibernateHandoffPages[64];
  60 uint32_t gIOHibernateHandoffPageCount = sizeof(gIOHibernateHandoffPages)
  61                                         / sizeof(gIOHibernateHandoffPages[0]);
  62
  63 #if CONFIG_DEBUG
  64 void hibprintf(const char *fmt, ...);
  65 #else
  66 #define hibprintf(x...)
  67 #endif
  68
  69
  70 #if CONFIG_SLEEP
  71 #if defined(__i386__) || defined(__x86_64__)
  72 extern void acpi_wake_prot_entry(void);
  73 #endif
  74 #endif
  75
  76 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  77
  78 #if defined(__i386__) || defined(__x86_64__)
  79
  80 #define DBGLOG  1
  81
  82 #include <architecture/i386/pio.h>
  83
  84 /* standard port addresses */
  85 enum {
  86     COM1_PORT_ADDR = 0x3f8,
  87     COM2_PORT_ADDR = 0x2f8
  88 };
  89
  90 /* UART register offsets */
  91 enum {
  92     UART_RBR = 0,  /* receive buffer Register   (R) */
  93     UART_THR = 0,  /* transmit holding register (W) */
  94     UART_DLL = 0,  /* DLAB = 1, divisor latch (LSB) */
  95     UART_IER = 1,  /* interrupt enable register     */
  96     UART_DLM = 1,  /* DLAB = 1, divisor latch (MSB) */
  97     UART_IIR = 2,  /* interrupt ident register (R)  */
  98     UART_FCR = 2,  /* fifo control register (W)     */
  99     UART_LCR = 3,  /* line control register         */
 100     UART_MCR = 4,  /* modem control register        */
 101     UART_LSR = 5,  /* line status register          */
 102     UART_MSR = 6,  /* modem status register         */
 103     UART_SCR = 7   /* scratch register              */
 104 };
 105
 106 enum {
 107     UART_LCR_8BITS = 0x03,
 108     UART_LCR_DLAB  = 0x80
 109 };
 110
 111 enum {
 112     UART_MCR_DTR   = 0x01,
 113     UART_MCR_RTS   = 0x02,
 114     UART_MCR_OUT1  = 0x04,
 115     UART_MCR_OUT2  = 0x08,
 116     UART_MCR_LOOP  = 0x10
 117 };
 118
 119 enum {
 120     UART_LSR_DR    = 0x01,
 121     UART_LSR_OE    = 0x02,
 122     UART_LSR_PE    = 0x04,
 123     UART_LSR_FE    = 0x08,
 124     UART_LSR_THRE  = 0x20
 125 };
 126
 127 static void uart_putc(char c)
 128 {
 129     while (!(inb(COM1_PORT_ADDR + UART_LSR) & UART_LSR_THRE))
 130         {}
 131     outb(COM1_PORT_ADDR + UART_THR, c);
 132 }
 133
 134 static int debug_probe( void )
 135 {
 136     /* Verify that the Scratch Register is accessible */
 137     outb(COM1_PORT_ADDR + UART_SCR, 0x5a);
 138     if (inb(COM1_PORT_ADDR + UART_SCR) != 0x5a) return false;
 139     outb(COM1_PORT_ADDR + UART_SCR, 0xa5);
 140     if (inb(COM1_PORT_ADDR + UART_SCR) != 0xa5) return false;
 141     uart_putc('\n');
 142     return true;
 143 }
 144
 145 static void uart_puthex(uint64_t num)
 146 {
 147     int bit;
 148     char c;
 149     bool leading = true;
 150
 151     for (bit = 60; bit >= 0; bit -= 4)
 152     {
 153         c = 0xf & (num >> bit);
 154         if (c)
 155             leading = false;
 156         else if (leading && bit)
 157             continue;
 158         if (c <= 9)
 159             c += '0';
 160         else
 161             c+= 'a' - 10;
 162         uart_putc(c);
 163     }
 164 }
 165
 166 static void debug_code(uint32_t code, uint64_t value)
 167 {
 168     int bit;
 169     char c;
 170
 171     if (!(kIOHibernateDebugRestoreLogs & gIOHibernateDebugFlags))
 172         return;
 173
 174     for (bit = 24; bit >= 0; bit -= 8)
 175     {
 176         c = 0xFF & (code >> bit);
 177         if (c)
 178             uart_putc(c);
 179     }
 180     uart_putc('=');
 181     uart_puthex(value);
 182     uart_putc('\n');
 183     uart_putc('\r');
 184 }
 185
 186 #endif /* defined(__i386__) || defined(__x86_64__) */
 187
 188 #if !defined(DBGLOG)
 189 #define debug_probe()       (false)
 190 #define debug_code(c, v)    {}
 191 #endif
 192
 193 enum
 194 {
 195     kIOHibernateRestoreCodeImageStart       = 'imgS',
 196     kIOHibernateRestoreCodeImageEnd         = 'imgE',
 197     kIOHibernateRestoreCodePageIndexStart   = 'pgiS',
 198     kIOHibernateRestoreCodePageIndexEnd     = 'pgiE',
 199     kIOHibernateRestoreCodeMapStart         = 'mapS',
 200     kIOHibernateRestoreCodeMapEnd           = 'mapE',
 201     kIOHibernateRestoreCodeWakeMapSize      = 'wkms',
 202     kIOHibernateRestoreCodeConflictPage     = 'cfpg',
 203     kIOHibernateRestoreCodeConflictSource   = 'cfsr',
 204     kIOHibernateRestoreCodeNoMemory         = 'nomm'
 205 };
 206
 207 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 208
 209
 210 static void fatal(void)
 211 {
 212 #if defined(__i386__) || defined(__x86_64__)
 213     outb(0xcf9, 6);
 214 #else
 215     while (true) {}
 216 #endif
 217 }
 218
 219 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 220
 221 uint32_t
 222 hibernate_sum_page(uint8_t *buf, uint32_t ppnum)
 223 {
 224     return (((uint32_t *)buf)[((PAGE_SIZE >> 2) - 1) & ppnum]);
 225 }
 226
 227 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 228
 229 static hibernate_bitmap_t *
 230 hibernate_page_bitmap(hibernate_page_list_t * list, uint32_t page)
 231 {
 232     uint32_t             bank;
 233     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 234
 235     for (bank = 0; bank < list->bank_count; bank++)
 236     {
 237         if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
 238             break;
 239         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 240     }
 241     if (bank == list->bank_count)
 242         bitmap = NULL;
 243
 244     return (bitmap);
 245 }
 246
 247 hibernate_bitmap_t *
 248 hibernate_page_bitmap_pin(hibernate_page_list_t * list, uint32_t * pPage)
 249 {
 250     uint32_t             bank, page = *pPage;
 251     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 252
 253     for (bank = 0; bank < list->bank_count; bank++)
 254     {
 255         if (page <= bitmap->first_page)
 256         {
 257             *pPage = bitmap->first_page;
 258             break;
 259         }
 260         if (page <= bitmap->last_page)
 261             break;
 262         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 263     }
 264     if (bank == list->bank_count)
 265         bitmap = NULL;
 266
 267     return (bitmap);
 268 }
 269
 270 void
 271 hibernate_page_bitset(hibernate_page_list_t * list, boolean_t set, uint32_t page)
 272 {
 273     hibernate_bitmap_t * bitmap;
 274
 275     bitmap = hibernate_page_bitmap(list, page);
 276     if (bitmap)
 277     {
 278         page -= bitmap->first_page;
 279         if (set)
 280             bitmap->bitmap[page >> 5] |= (0x80000000 >> (page & 31));
 281             //setbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
 282         else
 283             bitmap->bitmap[page >> 5] &= ~(0x80000000 >> (page & 31));
 284             //clrbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
 285     }
 286 }
 287
 288 boolean_t
 289 hibernate_page_bittst(hibernate_page_list_t * list, uint32_t page)
 290 {
 291     boolean_t            result = TRUE;
 292     hibernate_bitmap_t * bitmap;
 293
 294     bitmap = hibernate_page_bitmap(list, page);
 295     if (bitmap)
 296     {
 297         page -= bitmap->first_page;
 298         result = (0 != (bitmap->bitmap[page >> 5] & (0x80000000 >> (page & 31))));
 299     }
 300     return (result);
 301 }
 302
 303 // count bits clear or set (set == TRUE) starting at page.
 304 uint32_t
 305 hibernate_page_bitmap_count(hibernate_bitmap_t * bitmap, uint32_t set, uint32_t page)
 306 {
 307     uint32_t index, bit, bits;
 308     uint32_t count;
 309
 310     count = 0;
 311
 312     index = (page - bitmap->first_page) >> 5;
 313     bit = (page - bitmap->first_page) & 31;
 314
 315     bits = bitmap->bitmap[index];
 316     if (set)
 317         bits = ~bits;
 318     bits = (bits << bit);
 319     if (bits)
 320         count += __builtin_clz(bits);
 321     else
 322     {
 323         count += 32 - bit;
 324         while (++index < bitmap->bitmapwords)
 325         {
 326             bits = bitmap->bitmap[index];
 327             if (set)
 328                 bits = ~bits;
 329             if (bits)
 330             {
 331                 count += __builtin_clz(bits);
 332                 break;
 333             }
 334             count += 32;
 335         }
 336     }
 337
 338     return (count);
 339 }
 340
 341 static ppnum_t
 342 hibernate_page_list_grab(hibernate_page_list_t * list, uint32_t * pNextFree)
 343 {
 344     uint32_t             nextFree = *pNextFree;
 345     uint32_t             nextFreeInBank;
 346     hibernate_bitmap_t * bitmap;
 347
 348     nextFreeInBank = nextFree + 1;
 349     while ((bitmap = hibernate_page_bitmap_pin(list, &nextFreeInBank)))
 350     {
 351         nextFreeInBank += hibernate_page_bitmap_count(bitmap, FALSE, nextFreeInBank);
 352         if (nextFreeInBank <= bitmap->last_page)
 353         {
 354             *pNextFree = nextFreeInBank;
 355             break;
 356         }
 357     }
 358
 359     if (!bitmap)
 360     {
 361         debug_code(kIOHibernateRestoreCodeNoMemory, nextFree);
 362         fatal();
 363         nextFree = 0;
 364     }
 365
 366     return (nextFree);
 367 }
 368
 369 static uint32_t
 370 store_one_page(uint32_t procFlags, uint32_t * src, uint32_t compressedSize,
 371                 uint32_t * buffer, uint32_t ppnum)
 372 {
 373         uint64_t dst = ptoa_64(ppnum);
 374
 375         if (compressedSize != PAGE_SIZE)
 376         {
 377                 dst = pal_hib_map(DEST_COPY_AREA, dst);
 378                 WKdm_decompress((WK_word*) src, (WK_word*)(uintptr_t)dst, PAGE_SIZE >> 2);
 379         }
 380         else
 381         {
 382                 dst = hibernate_restore_phys_page((uint64_t) (uintptr_t) src, dst, PAGE_SIZE, procFlags);
 383         }
 384
 385         return hibernate_sum_page((uint8_t *)(uintptr_t)dst, ppnum);
 386 }
 387
 388 // used only for small struct copies
 389 static void
 390 bcopy_internal(const void *src, void *dst, uint32_t len)
 391 {
 392     const char *s = src;
 393     char       *d = dst;
 394     uint32_t   idx = 0;
 395
 396     while (idx < len)
 397     {
 398         d[idx] = s[idx];
 399         idx++;
 400     }
 401 }
 402
 403 #define C_ASSERT(e) typedef char    __C_ASSERT__[(e) ? 1 : -1]
 404
 405 long
 406 hibernate_kernel_entrypoint(IOHibernateImageHeader * header,
 407                             void * p2, void * p3, void * p4)
 408 {
 409     uint32_t idx;
 410     uint32_t * src;
 411     uint32_t * imageReadPos;
 412     uint32_t * pageIndexSource;
 413     hibernate_page_list_t * map;
 414     uint32_t stage;
 415     uint32_t count;
 416     uint32_t ppnum;
 417     uint32_t page;
 418     uint32_t conflictCount;
 419     uint32_t compressedSize;
 420     uint32_t uncompressedPages;
 421     uint32_t copyPageListHead;
 422     uint32_t * copyPageList;
 423     uint32_t copyPageIndex;
 424     uint32_t sum;
 425     uint32_t pageSum;
 426     uint32_t nextFree;
 427     uint32_t lastImagePage;
 428     uint32_t lastMapPage;
 429     uint32_t lastPageIndexPage;
 430     uint32_t handoffPages;
 431     uint32_t handoffPageCount;
 432
 433     C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
 434
 435     if ((kIOHibernateDebugRestoreLogs & gIOHibernateDebugFlags) && !debug_probe())
 436         gIOHibernateDebugFlags &= ~kIOHibernateDebugRestoreLogs;
 437
 438     debug_code(kIOHibernateRestoreCodeImageStart, (uintptr_t) header);
 439
 440     bcopy_internal(header,
 441                 gIOHibernateCurrentHeader,
 442                 sizeof(IOHibernateImageHeader));
 443
 444     map = (hibernate_page_list_t *)
 445                 (((uintptr_t) &header->fileExtentMap[0])
 446                             + header->fileExtentMapSize
 447                             + ptoa_32(header->restore1PageCount)
 448                             + header->previewSize);
 449
 450     lastImagePage = atop_32(((uintptr_t) header) + header->image1Size);
 451
 452     lastMapPage = atop_32(((uintptr_t) map) + header->bitmapSize);
 453
 454     handoffPages     = header->handoffPages;
 455     handoffPageCount = header->handoffPageCount;
 456
 457     debug_code(kIOHibernateRestoreCodeImageEnd,       ptoa_64(lastImagePage));
 458     debug_code(kIOHibernateRestoreCodeMapStart,       (uintptr_t) map);
 459     debug_code(kIOHibernateRestoreCodeMapEnd,         ptoa_64(lastMapPage));
 460
 461     debug_code('hand', ptoa_64(handoffPages));
 462     debug_code('hnde', ptoa_64(handoffPageCount));
 463
 464     // knock all the image pages to be used out of free map
 465     for (ppnum = atop_32((uintptr_t) header); ppnum <= lastImagePage; ppnum++)
 466     {
 467         hibernate_page_bitset(map, FALSE, ppnum);
 468     }
 469     // knock all the handoff pages to be used out of free map
 470     for (ppnum = handoffPages; ppnum < (handoffPages + handoffPageCount); ppnum++)
 471     {
 472         hibernate_page_bitset(map, FALSE, ppnum);
 473     }
 474
 475     nextFree = 0;
 476     hibernate_page_list_grab(map, &nextFree);
 477
 478     pal_hib_window_setup(hibernate_page_list_grab(map, &nextFree));
 479
 480     sum = header->actualRestore1Sum;
 481     gIOHibernateCurrentHeader->diag[0] = (uint32_t)(uintptr_t) header;
 482     gIOHibernateCurrentHeader->diag[1] = sum;
 483
 484     uncompressedPages = 0;
 485     conflictCount     = 0;
 486     copyPageListHead  = 0;
 487     copyPageList      = 0;
 488     copyPageIndex     = PAGE_SIZE >> 2;
 489
 490     compressedSize    = PAGE_SIZE;
 491     stage             = 2;
 492     count             = 0;
 493     src               = NULL;
 494
 495     if (gIOHibernateCurrentHeader->previewSize)
 496     {
 497         pageIndexSource = (uint32_t *)
 498                      (((uintptr_t) &header->fileExtentMap[0])
 499                                  + gIOHibernateCurrentHeader->fileExtentMapSize
 500                                  + ptoa_32(gIOHibernateCurrentHeader->restore1PageCount));
 501         imageReadPos = (uint32_t *) (((uintptr_t) pageIndexSource) + gIOHibernateCurrentHeader->previewPageListSize);
 502         lastPageIndexPage = atop_32((uintptr_t) imageReadPos);
 503     }
 504     else
 505     {
 506         pageIndexSource   = NULL;
 507         lastPageIndexPage = 0;
 508         imageReadPos =  (uint32_t *) (((uintptr_t) map) + gIOHibernateCurrentHeader->bitmapSize);
 509     }
 510
 511     debug_code(kIOHibernateRestoreCodePageIndexStart, (uintptr_t) pageIndexSource);
 512     debug_code(kIOHibernateRestoreCodePageIndexEnd,   ptoa_64(lastPageIndexPage));
 513
 514     while (1)
 515     {
 516         switch (stage)
 517         {
 518             case 2:
 519                 // copy handoff data
 520                 count = src ? 0 : handoffPageCount;
 521                 if (!count)
 522                     break;
 523                 if (count > gIOHibernateHandoffPageCount)
 524                     count = gIOHibernateHandoffPageCount;
 525                 src = (uint32_t *) (uintptr_t) ptoa_64(handoffPages);
 526                 break;
 527
 528             case 1:
 529                 // copy pageIndexSource pages == preview image data
 530                 if (!src)
 531                 {
 532                     if (!pageIndexSource)
 533                         break;
 534                     src = imageReadPos;
 535                 }
 536                 ppnum = pageIndexSource[0];
 537                 count = pageIndexSource[1];
 538                 pageIndexSource += 2;
 539                 imageReadPos = src;
 540                 break;
 541
 542             case 0:
 543                 // copy pages
 544                 if (!src)
 545                 {
 546                     src =  (uint32_t *) (((uintptr_t) map) + gIOHibernateCurrentHeader->bitmapSize);
 547                 }
 548                 ppnum = src[0];
 549                 count = src[1];
 550                 src += 2;
 551                 imageReadPos = src;
 552                 break;
 553         }
 554
 555
 556         if (!count)
 557         {
 558             if (!stage)
 559                 break;
 560             stage--;
 561             src = NULL;
 562             continue;
 563         }
 564
 565         for (page = 0; page < count; page++, ppnum++)
 566         {
 567             uint32_t tag;
 568             int conflicts;
 569
 570             if (2 == stage)
 571                 ppnum = gIOHibernateHandoffPages[page];
 572             else if (!stage)
 573             {
 574                 tag = *src++;
 575                 compressedSize = kIOHibernateTagLength & tag;
 576             }
 577
 578             conflicts = (ppnum >= atop_32((uintptr_t) map)) && (ppnum <= lastMapPage);
 579
 580             conflicts |= ((ppnum >= atop_32((uintptr_t) imageReadPos)) && (ppnum <= lastImagePage));
 581
 582             if (stage >= 2)
 583                 conflicts |= ((ppnum >= atop_32((uintptr_t) src)) && (ppnum <= (handoffPages + handoffPageCount - 1)));
 584
 585             if (stage >= 1)
 586                 conflicts |= ((ppnum >= atop_32((uintptr_t) pageIndexSource)) && (ppnum <= lastPageIndexPage));
 587
 588             if (!conflicts)
 589             {
 590 //              if (compressedSize)
 591                 pageSum = store_one_page(gIOHibernateCurrentHeader->processorFlags,
 592                                          src, compressedSize, 0, ppnum);
 593                 if (stage != 2)
 594                     sum += pageSum;
 595                 uncompressedPages++;
 596             }
 597             else
 598             {
 599                 uint32_t   bufferPage;
 600                 uint32_t * dst;
 601
 602 //              debug_code(kIOHibernateRestoreCodeConflictPage,   ppnum);
 603 //              debug_code(kIOHibernateRestoreCodeConflictSource, (uintptr_t) src);
 604
 605                 conflictCount++;
 606
 607                 // alloc new buffer page
 608                 bufferPage = hibernate_page_list_grab(map, &nextFree);
 609
 610                 if (copyPageIndex > ((PAGE_SIZE >> 2) - 3))
 611                 {
 612                     // alloc new copy list page
 613                     uint32_t pageListPage = hibernate_page_list_grab(map, &nextFree);
 614                     // link to current
 615                     if (copyPageList) {
 616                             copyPageList[1] = pageListPage;
 617                     } else {
 618                             copyPageListHead = pageListPage;
 619                     }
 620                     copyPageList = (uint32_t *)pal_hib_map(SRC_COPY_AREA,
 621                                     ptoa_32(pageListPage));
 622                     copyPageList[1] = 0;
 623                     copyPageIndex = 2;
 624                 }
 625
 626                 copyPageList[copyPageIndex++] = ppnum;
 627                 copyPageList[copyPageIndex++] = bufferPage;
 628                 copyPageList[copyPageIndex++] = (compressedSize | (stage << 24));
 629                 copyPageList[0] = copyPageIndex;
 630
 631                 dst = (uint32_t *)pal_hib_map(DEST_COPY_AREA, ptoa_32(bufferPage));
 632                 for (idx = 0; idx < ((compressedSize + 3) >> 2); idx++)
 633                         dst[idx] = src[idx];
 634             }
 635             src += ((compressedSize + 3) >> 2);
 636         }
 637     }
 638
 639     /* src points to the last page restored, so we need to skip over that */
 640     hibernateRestorePALState(src);
 641
 642     // -- copy back conflicts
 643
 644     copyPageList = (uint32_t *)(uintptr_t) ptoa_32(copyPageListHead);
 645
 646     while (copyPageList)
 647     {
 648         copyPageList = (uint32_t *)pal_hib_map(COPY_PAGE_AREA, (uintptr_t)copyPageList);
 649         for (copyPageIndex = 2; copyPageIndex < copyPageList[0]; copyPageIndex += 3)
 650         {
 651             ppnum          = copyPageList[copyPageIndex + 0];
 652             src            = (uint32_t *) (uintptr_t) ptoa_32(copyPageList[copyPageIndex + 1]);
 653             src            = (uint32_t *)pal_hib_map(SRC_COPY_AREA, (uintptr_t)src);
 654             compressedSize = copyPageList[copyPageIndex + 2];
 655             stage          = compressedSize >> 24;
 656             compressedSize &= 0x1FFF;
 657             pageSum        = store_one_page(gIOHibernateCurrentHeader->processorFlags,
 658                                                 src, compressedSize, 0, ppnum);
 659             if (stage != 2)
 660                 sum += pageSum;
 661             uncompressedPages++;
 662         }
 663         copyPageList = (uint32_t *) (uintptr_t) ptoa_32(copyPageList[1]);
 664     }
 665
 666     pal_hib_patchup();
 667
 668     // -- image has been destroyed...
 669
 670     gIOHibernateCurrentHeader->actualImage1Sum         = sum;
 671     gIOHibernateCurrentHeader->actualUncompressedPages = uncompressedPages;
 672     gIOHibernateCurrentHeader->conflictCount           = conflictCount;
 673     gIOHibernateCurrentHeader->nextFree                = nextFree;
 674
 675     gIOHibernateState = kIOHibernateStateWakingFromHibernate;
 676
 677 #if CONFIG_SLEEP
 678 #if defined(__i386__) || defined(__x86_64__)
 679     typedef void (*ResetProc)(void);
 680     ResetProc proc;
 681     proc = HIB_ENTRYPOINT;
 682     // flush caches
 683     __asm__("wbinvd");
 684     proc();
 685 #else
 686 // implement me
 687 #endif
 688 #endif
 689
 690     return -1;
 691 }
 692
 693 #if CONFIG_DEBUG
 694 /* standalone printf implementation */
 695 /*-
 696  * Copyright (c) 1986, 1988, 1991, 1993
 697  *      The Regents of the University of California.  All rights reserved.
 698  * (c) UNIX System Laboratories, Inc.
 699  * All or some portions of this file are derived from material licensed
 700  * to the University of California by American Telephone and Telegraph
 701  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
 702  * the permission of UNIX System Laboratories, Inc.
 703  *
 704  * Redistribution and use in source and binary forms, with or without
 705  * modification, are permitted provided that the following conditions
 706  * are met:
 707  * 1. Redistributions of source code must retain the above copyright
 708  *    notice, this list of conditions and the following disclaimer.
 709  * 2. Redistributions in binary form must reproduce the above copyright
 710  *    notice, this list of conditions and the following disclaimer in the
 711  *    documentation and/or other materials provided with the distribution.
 712  * 4. Neither the name of the University nor the names of its contributors
 713  *    may be used to endorse or promote products derived from this software
 714  *    without specific prior written permission.
 715  *
 716  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 717  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 718  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 719  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 720  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 721  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 722  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 723  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 724  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 725  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 726  * SUCH DAMAGE.
 727  *
 728  *      @(#)subr_prf.c  8.3 (Berkeley) 1/21/94
 729  */
 730
 731 typedef long ptrdiff_t;
 732 char const hibhex2ascii_data[] = "0123456789abcdefghijklmnopqrstuvwxyz";
 733 #define hibhex2ascii(hex)  (hibhex2ascii_data[hex])
 734 #define toupper(c)      ((c) - 0x20 * (((c) >= 'a') && ((c) <= 'z')))
 735 static size_t
 736 hibstrlen(const char *s)
 737 {
 738         size_t l = 0;
 739         while (*s++)
 740                 l++;
 741         return l;
 742 }
 743
 744 /* Max number conversion buffer length: a u_quad_t in base 2, plus NUL byte. */
 745 #define MAXNBUF (sizeof(intmax_t) * NBBY + 1)
 746
 747 /*
 748  * Put a NUL-terminated ASCII number (base <= 36) in a buffer in reverse
 749  * order; return an optional length and a pointer to the last character
 750  * written in the buffer (i.e., the first character of the string).
 751  * The buffer pointed to by `nbuf' must have length >= MAXNBUF.
 752  */
 753 static char *
 754 ksprintn(char *nbuf, uintmax_t num, int base, int *lenp, int upper)
 755 {
 756         char *p, c;
 757
 758         /* Truncate so we don't call umoddi3, which isn't in __HIB */
 759 #if !defined(__LP64__)
 760         uint32_t num2 = (uint32_t) num;
 761 #else
 762         uintmax_t num2 = num;
 763 #endif
 764
 765         p = nbuf;
 766         *p = '\0';
 767         do {
 768                 c = hibhex2ascii(num2 % base);
 769                 *++p = upper ? toupper(c) : c;
 770         } while (num2 /= base);
 771         if (lenp)
 772                 *lenp = (int)(p - nbuf);
 773         return (p);
 774 }
 775
 776 /*
 777  * Scaled down version of printf(3).
 778  *
 779  * Two additional formats:
 780  *
 781  * The format %b is supported to decode error registers.
 782  * Its usage is:
 783  *
 784  *      printf("reg=%b\n", regval, "*");
 785  *
 786  * where  is the output base expressed as a control character, e.g.
 787  * \10 gives octal; \20 gives hex.  Each arg is a sequence of characters,
 788  * the first of which gives the bit number to be inspected (origin 1), and
 789  * the next characters (up to a control character, i.e. a character <= 32),
 790  * give the name of the register.  Thus:
 791  *
 792  *      kvprintf("reg=%b\n", 3, "\10\2BITTWO\1BITONE\n");
 793  *
 794  * would produce output:
 795  *
 796  *      reg=3
 797  *
 798  * XXX:  %D  -- Hexdump, takes pointer and separator string:
 799  *              ("%6D", ptr, ":")   -> XX:XX:XX:XX:XX:XX
 800  *              ("%*D", len, ptr, " " -> XX XX XX XX ...
 801  */
 802 static int
 803 hibkvprintf(char const *fmt, void (*func)(int, void*), void *arg, int radix, va_list ap)
 804 {
 805 #define PCHAR(c) {int cc=(c); if (func) (*func)(cc,arg); else *d++ = cc; retval++; }
 806         char nbuf[MAXNBUF];
 807         char *d;
 808         const char *p, *percent, *q;
 809         u_char *up;
 810         int ch, n;
 811         uintmax_t num;
 812         int base, lflag, qflag, tmp, width, ladjust, sharpflag, neg, sign, dot;
 813         int cflag, hflag, jflag, tflag, zflag;
 814         int dwidth, upper;
 815         char padc;
 816         int stop = 0, retval = 0;
 817
 818         num = 0;
 819         if (!func)
 820                 d = (char *) arg;
 821         else
 822                 d = NULL;
 823
 824         if (fmt == NULL)
 825                 fmt = "(fmt null)\n";
 826
 827         if (radix < 2 || radix > 36)
 828                 radix = 10;
 829
 830         for (;;) {
 831                 padc = ' ';
 832                 width = 0;
 833                 while ((ch = (u_char)*fmt++) != '%' || stop) {
 834                         if (ch == '\0')
 835                                 return (retval);
 836                         PCHAR(ch);
 837                 }
 838                 percent = fmt - 1;
 839                 qflag = 0; lflag = 0; ladjust = 0; sharpflag = 0; neg = 0;
 840                 sign = 0; dot = 0; dwidth = 0; upper = 0;
 841                 cflag = 0; hflag = 0; jflag = 0; tflag = 0; zflag = 0;
 842 reswitch:       switch (ch = (u_char)*fmt++) {
 843                 case '.':
 844                         dot = 1;
 845                         goto reswitch;
 846                 case '#':
 847                         sharpflag = 1;
 848                         goto reswitch;
 849                 case '+':
 850                         sign = 1;
 851                         goto reswitch;
 852                 case '-':
 853                         ladjust = 1;
 854                         goto reswitch;
 855                 case '%':
 856                         PCHAR(ch);
 857                         break;
 858                 case '*':
 859                         if (!dot) {
 860                                 width = va_arg(ap, int);
 861                                 if (width < 0) {
 862                                         ladjust = !ladjust;
 863                                         width = -width;
 864                                 }
 865                         } else {
 866                                 dwidth = va_arg(ap, int);
 867                         }
 868                         goto reswitch;
 869                 case '0':
 870                         if (!dot) {
 871                                 padc = '0';
 872                                 goto reswitch;
 873                         }
 874                 case '1': case '2': case '3': case '4':
 875                 case '5': case '6': case '7': case '8': case '9':
 876                                 for (n = 0;; ++fmt) {
 877                                         n = n * 10 + ch - '0';
 878                                         ch = *fmt;
 879                                         if (ch < '0' || ch > '9')
 880                                                 break;
 881                                 }
 882                         if (dot)
 883                                 dwidth = n;
 884                         else
 885                                 width = n;
 886                         goto reswitch;
 887                 case 'b':
 888                         num = (u_int)va_arg(ap, int);
 889                         p = va_arg(ap, char *);
 890                         for (q = ksprintn(nbuf, num, *p++, NULL, 0); *q;)
 891                                 PCHAR(*q--);
 892
 893                         if (num == 0)
 894                                 break;
 895
 896                         for (tmp = 0; *p;) {
 897                                 n = *p++;
 898                                 if (num & (1 << (n - 1))) {
 899                                         PCHAR(tmp ? ',' : '<');
 900                                         for (; (n = *p) > ' '; ++p)
 901                                                 PCHAR(n);
 902                                         tmp = 1;
 903                                 } else
 904                                         for (; *p > ' '; ++p)
 905                                                 continue;
 906                         }
 907                         if (tmp)
 908                                 PCHAR('>');
 909                         break;
 910                 case 'c':
 911                         PCHAR(va_arg(ap, int));
 912                         break;
 913                 case 'D':
 914                         up = va_arg(ap, u_char *);
 915                         p = va_arg(ap, char *);
 916                         if (!width)
 917                                 width = 16;
 918                         while(width--) {
 919                                 PCHAR(hibhex2ascii(*up >> 4));
 920                                 PCHAR(hibhex2ascii(*up & 0x0f));
 921                                 up++;
 922                                 if (width)
 923                                         for (q=p;*q;q++)
 924                                                 PCHAR(*q);
 925                         }
 926                         break;
 927                 case 'd':
 928                 case 'i':
 929                         base = 10;
 930                         sign = 1;
 931                         goto handle_sign;
 932                 case 'h':
 933                         if (hflag) {
 934                                 hflag = 0;
 935                                 cflag = 1;
 936                         } else
 937                                 hflag = 1;
 938                         goto reswitch;
 939                 case 'j':
 940                         jflag = 1;
 941                         goto reswitch;
 942                 case 'l':
 943                         if (lflag) {
 944                                 lflag = 0;
 945                                 qflag = 1;
 946                         } else
 947                                 lflag = 1;
 948                         goto reswitch;
 949                 case 'n':
 950                         if (jflag)
 951                                 *(va_arg(ap, intmax_t *)) = retval;
 952                         else if (qflag)
 953                                 *(va_arg(ap, quad_t *)) = retval;
 954                         else if (lflag)
 955                                 *(va_arg(ap, long *)) = retval;
 956                         else if (zflag)
 957                                 *(va_arg(ap, size_t *)) = retval;
 958                         else if (hflag)
 959                                 *(va_arg(ap, short *)) = retval;
 960                         else if (cflag)
 961                                 *(va_arg(ap, char *)) = retval;
 962                         else
 963                                 *(va_arg(ap, int *)) = retval;
 964                         break;
 965                 case 'o':
 966                         base = 8;
 967                         goto handle_nosign;
 968                 case 'p':
 969                         base = 16;
 970                         sharpflag = (width == 0);
 971                         sign = 0;
 972                         num = (uintptr_t)va_arg(ap, void *);
 973                         goto number;
 974                 case 'q':
 975                         qflag = 1;
 976                         goto reswitch;
 977                 case 'r':
 978                         base = radix;
 979                         if (sign)
 980                                 goto handle_sign;
 981                         goto handle_nosign;
 982                 case 's':
 983                         p = va_arg(ap, char *);
 984                         if (p == NULL)
 985                                 p = "(null)";
 986                         if (!dot)
 987                                 n = (typeof(n))hibstrlen (p);
 988                         else
 989                                 for (n = 0; n < dwidth && p[n]; n++)
 990                                         continue;
 991
 992                         width -= n;
 993
 994                         if (!ladjust && width > 0)
 995                                 while (width--)
 996                                         PCHAR(padc);
 997                         while (n--)
 998                                 PCHAR(*p++);
 999                         if (ladjust && width > 0)
1000                                 while (width--)
1001                                         PCHAR(padc);
1002                         break;
1003                 case 't':
1004                         tflag = 1;
1005                         goto reswitch;
1006                 case 'u':
1007                         base = 10;
1008                         goto handle_nosign;
1009                 case 'X':
1010                         upper = 1;
1011                 case 'x':
1012                         base = 16;
1013                         goto handle_nosign;
1014                 case 'y':
1015                         base = 16;
1016                         sign = 1;
1017                         goto handle_sign;
1018                 case 'z':
1019                         zflag = 1;
1020                         goto reswitch;
1021 handle_nosign:
1022                         sign = 0;
1023                         if (jflag)
1024                                 num = va_arg(ap, uintmax_t);
1025                         else if (qflag)
1026                                 num = va_arg(ap, u_quad_t);
1027                         else if (tflag)
1028                                 num = va_arg(ap, ptrdiff_t);
1029                         else if (lflag)
1030                                 num = va_arg(ap, u_long);
1031                         else if (zflag)
1032                                 num = va_arg(ap, size_t);
1033                         else if (hflag)
1034                                 num = (u_short)va_arg(ap, int);
1035                         else if (cflag)
1036                                 num = (u_char)va_arg(ap, int);
1037                         else
1038                                 num = va_arg(ap, u_int);
1039                         goto number;
1040 handle_sign:
1041                         if (jflag)
1042                                 num = va_arg(ap, intmax_t);
1043                         else if (qflag)
1044                                 num = va_arg(ap, quad_t);
1045                         else if (tflag)
1046                                 num = va_arg(ap, ptrdiff_t);
1047                         else if (lflag)
1048                                 num = va_arg(ap, long);
1049                         else if (zflag)
1050                                 num = va_arg(ap, ssize_t);
1051                         else if (hflag)
1052                                 num = (short)va_arg(ap, int);
1053                         else if (cflag)
1054                                 num = (char)va_arg(ap, int);
1055                         else
1056                                 num = va_arg(ap, int);
1057 number:
1058                         if (sign && (intmax_t)num < 0) {
1059                                 neg = 1;
1060                                 num = -(intmax_t)num;
1061                         }
1062                         p = ksprintn(nbuf, num, base, &tmp, upper);
1063                         if (sharpflag && num != 0) {
1064                                 if (base == 8)
1065                                         tmp++;
1066                                 else if (base == 16)
1067                                         tmp += 2;
1068                         }
1069                         if (neg)
1070                                 tmp++;
1071
1072                         if (!ladjust && padc != '0' && width
1073                             && (width -= tmp) > 0)
1074                                 while (width--)
1075                                         PCHAR(padc);
1076                         if (neg)
1077                                 PCHAR('-');
1078                         if (sharpflag && num != 0) {
1079                                 if (base == 8) {
1080                                         PCHAR('0');
1081                                 } else if (base == 16) {
1082                                         PCHAR('0');
1083                                         PCHAR('x');
1084                                 }
1085                         }
1086                         if (!ladjust && width && (width -= tmp) > 0)
1087                                 while (width--)
1088                                         PCHAR(padc);
1089
1090                         while (*p)
1091                                 PCHAR(*p--);
1092
1093                         if (ladjust && width && (width -= tmp) > 0)
1094                                 while (width--)
1095                                         PCHAR(padc);
1096
1097                         break;
1098                 default:
1099                         while (percent < fmt)
1100                                 PCHAR(*percent++);
1101                         /*
1102                          * Since we ignore an formatting argument it is no
1103                          * longer safe to obey the remaining formatting
1104                          * arguments as the arguments will no longer match
1105                          * the format specs.
1106                          */
1107                         stop = 1;
1108                         break;
1109                 }
1110         }
1111 #undef PCHAR
1112 }
1113
1114
1115 static void
1116 putchar(int c, void *arg)
1117 {
1118         (void)arg;
1119         uart_putc(c);
1120 }
1121
1122 void
1123 hibprintf(const char *fmt, ...)
1124 {
1125         /* http://www.pagetable.com/?p=298 */
1126         va_list ap;
1127
1128         va_start(ap, fmt);
1129         hibkvprintf(fmt, putchar, NULL, 10, ap);
1130         va_end(ap);
1131 }
1132 #endif /* CONFIG_DEBUG */
1133