iokit/Kernel/IOHibernateRestoreKernel.c

   1 /*
   2  * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <stdint.h>
  30 #include <mach/mach_types.h>
  31 #include <mach/vm_param.h>
  32 #include <IOKit/IOHibernatePrivate.h>
  33 #include <IOKit/IOLib.h>
  34 #include <pexpert/boot.h>
  35 #include <libkern/libkern.h>
  36
  37 #include <libkern/WKdm.h>
  38 #include "IOHibernateInternal.h"
  39
  40 #if defined(__i386__) || defined(__x86_64__)
  41 #include <i386/pal_hibernate.h>
  42 #endif
  43
  44 /*
  45 This code is linked into the kernel but part of the "__HIB" section, which means
  46 its used by code running in the special context of restoring the kernel text and data
  47 from the hibernation image read by the booter. hibernate_kernel_entrypoint() and everything
  48 it calls or references needs to be careful to only touch memory also in the "__HIB" section.
  49 */
  50
  51 uint32_t gIOHibernateState;
  52
  53 uint32_t gIOHibernateDebugFlags;
  54
  55 static IOHibernateImageHeader _hibernateHeader;
  56 IOHibernateImageHeader * gIOHibernateCurrentHeader = &_hibernateHeader;
  57
  58 ppnum_t gIOHibernateHandoffPages[64];
  59 uint32_t gIOHibernateHandoffPageCount = sizeof(gIOHibernateHandoffPages)
  60                                         / sizeof(gIOHibernateHandoffPages[0]);
  61
  62 #if CONFIG_DEBUG
  63 void hibprintf(const char *fmt, ...);
  64 #else
  65 #define hibprintf(x...)
  66 #endif
  67
  68
  69 #if CONFIG_SLEEP
  70 #if defined(__i386__) || defined(__x86_64__)
  71 extern void acpi_wake_prot_entry(void);
  72 #endif
  73 #endif
  74
  75 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  76
  77 #if defined(__i386__) || defined(__x86_64__)
  78
  79 #define DBGLOG  1
  80
  81 #include <architecture/i386/pio.h>
  82
  83 /* standard port addresses */
  84 enum {
  85     COM1_PORT_ADDR = 0x3f8,
  86     COM2_PORT_ADDR = 0x2f8
  87 };
  88
  89 /* UART register offsets */
  90 enum {
  91     UART_RBR = 0,  /* receive buffer Register   (R) */
  92     UART_THR = 0,  /* transmit holding register (W) */
  93     UART_DLL = 0,  /* DLAB = 1, divisor latch (LSB) */
  94     UART_IER = 1,  /* interrupt enable register     */
  95     UART_DLM = 1,  /* DLAB = 1, divisor latch (MSB) */
  96     UART_IIR = 2,  /* interrupt ident register (R)  */
  97     UART_FCR = 2,  /* fifo control register (W)     */
  98     UART_LCR = 3,  /* line control register         */
  99     UART_MCR = 4,  /* modem control register        */
 100     UART_LSR = 5,  /* line status register          */
 101     UART_MSR = 6,  /* modem status register         */
 102     UART_SCR = 7   /* scratch register              */
 103 };
 104
 105 enum {
 106     UART_LCR_8BITS = 0x03,
 107     UART_LCR_DLAB  = 0x80
 108 };
 109
 110 enum {
 111     UART_MCR_DTR   = 0x01,
 112     UART_MCR_RTS   = 0x02,
 113     UART_MCR_OUT1  = 0x04,
 114     UART_MCR_OUT2  = 0x08,
 115     UART_MCR_LOOP  = 0x10
 116 };
 117
 118 enum {
 119     UART_LSR_DR    = 0x01,
 120     UART_LSR_OE    = 0x02,
 121     UART_LSR_PE    = 0x04,
 122     UART_LSR_FE    = 0x08,
 123     UART_LSR_THRE  = 0x20
 124 };
 125
 126 static void uart_putc(char c)
 127 {
 128     while (!(inb(COM1_PORT_ADDR + UART_LSR) & UART_LSR_THRE))
 129         {}
 130     outb(COM1_PORT_ADDR + UART_THR, c);
 131 }
 132
 133 static int debug_probe( void )
 134 {
 135     /* Verify that the Scratch Register is accessible */
 136     outb(COM1_PORT_ADDR + UART_SCR, 0x5a);
 137     if (inb(COM1_PORT_ADDR + UART_SCR) != 0x5a) return false;
 138     outb(COM1_PORT_ADDR + UART_SCR, 0xa5);
 139     if (inb(COM1_PORT_ADDR + UART_SCR) != 0xa5) return false;
 140     uart_putc('\n');
 141     return true;
 142 }
 143
 144 static void uart_puthex(uint64_t num)
 145 {
 146     int bit;
 147     char c;
 148     bool leading = true;
 149
 150     for (bit = 60; bit >= 0; bit -= 4)
 151     {
 152         c = 0xf & (num >> bit);
 153         if (c)
 154             leading = false;
 155         else if (leading && bit)
 156             continue;
 157         if (c <= 9)
 158             c += '0';
 159         else
 160             c+= 'a' - 10;
 161         uart_putc(c);
 162     }
 163 }
 164
 165 static void debug_code(uint32_t code, uint64_t value)
 166 {
 167     int bit;
 168     char c;
 169
 170     if (!(kIOHibernateDebugRestoreLogs & gIOHibernateDebugFlags))
 171         return;
 172
 173     for (bit = 24; bit >= 0; bit -= 8)
 174     {
 175         c = 0xFF & (code >> bit);
 176         if (c)
 177             uart_putc(c);
 178     }
 179     uart_putc('=');
 180     uart_puthex(value);
 181     uart_putc('\n');
 182     uart_putc('\r');
 183 }
 184
 185 #endif /* defined(__i386__) || defined(__x86_64__) */
 186
 187 #if !defined(DBGLOG)
 188 #define debug_probe()       (false)
 189 #define debug_code(c, v)    {}
 190 #endif
 191
 192 enum
 193 {
 194     kIOHibernateRestoreCodeImageStart       = 'imgS',
 195     kIOHibernateRestoreCodeImageEnd         = 'imgE',
 196     kIOHibernateRestoreCodePageIndexStart   = 'pgiS',
 197     kIOHibernateRestoreCodePageIndexEnd     = 'pgiE',
 198     kIOHibernateRestoreCodeMapStart         = 'mapS',
 199     kIOHibernateRestoreCodeMapEnd           = 'mapE',
 200     kIOHibernateRestoreCodeWakeMapSize      = 'wkms',
 201     kIOHibernateRestoreCodeConflictPage     = 'cfpg',
 202     kIOHibernateRestoreCodeConflictSource   = 'cfsr',
 203     kIOHibernateRestoreCodeNoMemory         = 'nomm',
 204     kIOHibernateRestoreCodeTag              = 'tag ',
 205     kIOHibernateRestoreCodeSignature        = 'sign',
 206     kIOHibernateRestoreCodeMapVirt          = 'mapV',
 207     kIOHibernateRestoreCodeHandoffPages     = 'hand',
 208     kIOHibernateRestoreCodeHandoffCount     = 'hndc',
 209 };
 210
 211 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 212
 213
 214 static void fatal(void)
 215 {
 216 #if defined(__i386__) || defined(__x86_64__)
 217     outb(0xcf9, 6);
 218 #else
 219     while (true) {}
 220 #endif
 221 }
 222
 223 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 224
 225 uint32_t
 226 hibernate_sum_page(uint8_t *buf, uint32_t ppnum)
 227 {
 228     return (((uint32_t *)buf)[((PAGE_SIZE >> 2) - 1) & ppnum]);
 229 }
 230
 231 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 232
 233 static hibernate_bitmap_t *
 234 hibernate_page_bitmap(hibernate_page_list_t * list, uint32_t page)
 235 {
 236     uint32_t             bank;
 237     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 238
 239     for (bank = 0; bank < list->bank_count; bank++)
 240     {
 241         if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
 242             break;
 243         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 244     }
 245     if (bank == list->bank_count)
 246         bitmap = NULL;
 247
 248     return (bitmap);
 249 }
 250
 251 hibernate_bitmap_t *
 252 hibernate_page_bitmap_pin(hibernate_page_list_t * list, uint32_t * pPage)
 253 {
 254     uint32_t             bank, page = *pPage;
 255     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 256
 257     for (bank = 0; bank < list->bank_count; bank++)
 258     {
 259         if (page <= bitmap->first_page)
 260         {
 261             *pPage = bitmap->first_page;
 262             break;
 263         }
 264         if (page <= bitmap->last_page)
 265             break;
 266         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 267     }
 268     if (bank == list->bank_count)
 269         bitmap = NULL;
 270
 271     return (bitmap);
 272 }
 273
 274 void
 275 hibernate_page_bitset(hibernate_page_list_t * list, boolean_t set, uint32_t page)
 276 {
 277     hibernate_bitmap_t * bitmap;
 278
 279     bitmap = hibernate_page_bitmap(list, page);
 280     if (bitmap)
 281     {
 282         page -= bitmap->first_page;
 283         if (set)
 284             bitmap->bitmap[page >> 5] |= (0x80000000 >> (page & 31));
 285             //setbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
 286         else
 287             bitmap->bitmap[page >> 5] &= ~(0x80000000 >> (page & 31));
 288             //clrbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
 289     }
 290 }
 291
 292 boolean_t
 293 hibernate_page_bittst(hibernate_page_list_t * list, uint32_t page)
 294 {
 295     boolean_t            result = TRUE;
 296     hibernate_bitmap_t * bitmap;
 297
 298     bitmap = hibernate_page_bitmap(list, page);
 299     if (bitmap)
 300     {
 301         page -= bitmap->first_page;
 302         result = (0 != (bitmap->bitmap[page >> 5] & (0x80000000 >> (page & 31))));
 303     }
 304     return (result);
 305 }
 306
 307 // count bits clear or set (set == TRUE) starting at page.
 308 uint32_t
 309 hibernate_page_bitmap_count(hibernate_bitmap_t * bitmap, uint32_t set, uint32_t page)
 310 {
 311     uint32_t index, bit, bits;
 312     uint32_t count;
 313
 314     count = 0;
 315
 316     index = (page - bitmap->first_page) >> 5;
 317     bit = (page - bitmap->first_page) & 31;
 318
 319     bits = bitmap->bitmap[index];
 320     if (set)
 321         bits = ~bits;
 322     bits = (bits << bit);
 323     if (bits)
 324         count += __builtin_clz(bits);
 325     else
 326     {
 327         count += 32 - bit;
 328         while (++index < bitmap->bitmapwords)
 329         {
 330             bits = bitmap->bitmap[index];
 331             if (set)
 332                 bits = ~bits;
 333             if (bits)
 334             {
 335                 count += __builtin_clz(bits);
 336                 break;
 337             }
 338             count += 32;
 339         }
 340     }
 341
 342     if ((page + count) > (bitmap->last_page + 1)) count = (bitmap->last_page + 1) - page;
 343
 344     return (count);
 345 }
 346
 347 static ppnum_t
 348 hibernate_page_list_grab(hibernate_page_list_t * list, uint32_t * pNextFree)
 349 {
 350     uint32_t             nextFree = *pNextFree;
 351     uint32_t             nextFreeInBank;
 352     hibernate_bitmap_t * bitmap;
 353
 354     nextFreeInBank = nextFree + 1;
 355     while ((bitmap = hibernate_page_bitmap_pin(list, &nextFreeInBank)))
 356     {
 357         nextFreeInBank += hibernate_page_bitmap_count(bitmap, FALSE, nextFreeInBank);
 358         if (nextFreeInBank <= bitmap->last_page)
 359         {
 360             *pNextFree = nextFreeInBank;
 361             break;
 362         }
 363     }
 364
 365     if (!bitmap)
 366     {
 367         debug_code(kIOHibernateRestoreCodeNoMemory, nextFree);
 368         fatal();
 369         nextFree = 0;
 370     }
 371
 372     return (nextFree);
 373 }
 374
 375 static uint32_t
 376 store_one_page(uint32_t procFlags, uint32_t * src, uint32_t compressedSize,
 377                 uint32_t * buffer, uint32_t ppnum)
 378 {
 379         uint64_t dst = ptoa_64(ppnum);
 380
 381         if (compressedSize != PAGE_SIZE)
 382         {
 383                 dst = pal_hib_map(DEST_COPY_AREA, dst);
 384                 WKdm_decompress((WK_word*) src, (WK_word*)(uintptr_t)dst, PAGE_SIZE >> 2);
 385         }
 386         else
 387         {
 388                 dst = hibernate_restore_phys_page((uint64_t) (uintptr_t) src, dst, PAGE_SIZE, procFlags);
 389         }
 390
 391         return hibernate_sum_page((uint8_t *)(uintptr_t)dst, ppnum);
 392 }
 393
 394 // used only for small struct copies
 395 static void
 396 bcopy_internal(const void *src, void *dst, uint32_t len)
 397 {
 398     const char *s = src;
 399     char       *d = dst;
 400     uint32_t   idx = 0;
 401
 402     while (idx < len)
 403     {
 404         d[idx] = s[idx];
 405         idx++;
 406     }
 407 }
 408
 409 #define C_ASSERT(e) typedef char    __C_ASSERT__[(e) ? 1 : -1]
 410
 411 long
 412 hibernate_kernel_entrypoint(uint32_t p1,
 413                             uint32_t p2, uint32_t p3, uint32_t p4)
 414 {
 415     uint64_t headerPhys;
 416     uint64_t mapPhys;
 417     uint64_t srcPhys;
 418     uint64_t imageReadPhys;
 419     uint64_t pageIndexPhys;
 420     uint32_t idx;
 421     uint32_t * pageIndexSource;
 422     hibernate_page_list_t * map;
 423     uint32_t stage;
 424     uint32_t count;
 425     uint32_t ppnum;
 426     uint32_t page;
 427     uint32_t conflictCount;
 428     uint32_t compressedSize;
 429     uint32_t uncompressedPages;
 430     uint32_t copyPageListHeadPage;
 431     uint32_t pageListPage;
 432     uint32_t * copyPageList;
 433     uint32_t * src;
 434     uint32_t copyPageIndex;
 435     uint32_t sum;
 436     uint32_t pageSum;
 437     uint32_t nextFree;
 438     uint32_t lastImagePage;
 439     uint32_t lastMapPage;
 440     uint32_t lastPageIndexPage;
 441     uint32_t handoffPages;
 442     uint32_t handoffPageCount;
 443
 444     C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
 445
 446     headerPhys = ptoa_64(p1);
 447
 448     if ((kIOHibernateDebugRestoreLogs & gIOHibernateDebugFlags) && !debug_probe())
 449         gIOHibernateDebugFlags &= ~kIOHibernateDebugRestoreLogs;
 450
 451     debug_code(kIOHibernateRestoreCodeImageStart, headerPhys);
 452
 453     bcopy_internal((void *) pal_hib_map(IMAGE_AREA, headerPhys),
 454                    gIOHibernateCurrentHeader,
 455                    sizeof(IOHibernateImageHeader));
 456
 457     debug_code(kIOHibernateRestoreCodeSignature, gIOHibernateCurrentHeader->signature);
 458
 459     mapPhys = headerPhys
 460              + (offsetof(IOHibernateImageHeader, fileExtentMap)
 461              + gIOHibernateCurrentHeader->fileExtentMapSize
 462              + ptoa_32(gIOHibernateCurrentHeader->restore1PageCount)
 463              + gIOHibernateCurrentHeader->previewSize);
 464
 465     map = (hibernate_page_list_t *) pal_hib_map(BITMAP_AREA, mapPhys);
 466
 467     lastImagePage = atop_64(headerPhys + gIOHibernateCurrentHeader->image1Size);
 468     lastMapPage = atop_64(mapPhys + gIOHibernateCurrentHeader->bitmapSize);
 469
 470     handoffPages     = gIOHibernateCurrentHeader->handoffPages;
 471     handoffPageCount = gIOHibernateCurrentHeader->handoffPageCount;
 472
 473     debug_code(kIOHibernateRestoreCodeImageEnd,       ptoa_64(lastImagePage));
 474     debug_code(kIOHibernateRestoreCodeMapStart,       mapPhys);
 475     debug_code(kIOHibernateRestoreCodeMapEnd,         ptoa_64(lastMapPage));
 476
 477     debug_code(kIOHibernateRestoreCodeMapVirt, (uintptr_t) map);
 478     debug_code(kIOHibernateRestoreCodeHandoffPages, ptoa_64(handoffPages));
 479     debug_code(kIOHibernateRestoreCodeHandoffCount, handoffPageCount);
 480
 481     // knock all the image pages to be used out of free map
 482     for (ppnum = atop_64(headerPhys); ppnum <= lastImagePage; ppnum++)
 483     {
 484         hibernate_page_bitset(map, FALSE, ppnum);
 485     }
 486     // knock all the handoff pages to be used out of free map
 487     for (ppnum = handoffPages; ppnum < (handoffPages + handoffPageCount); ppnum++)
 488     {
 489         hibernate_page_bitset(map, FALSE, ppnum);
 490     }
 491
 492     nextFree = 0;
 493     hibernate_page_list_grab(map, &nextFree);
 494
 495     sum = gIOHibernateCurrentHeader->actualRestore1Sum;
 496     gIOHibernateCurrentHeader->diag[0] = atop_64(headerPhys);
 497     gIOHibernateCurrentHeader->diag[1] = sum;
 498
 499     uncompressedPages    = 0;
 500     conflictCount        = 0;
 501     copyPageListHeadPage = 0;
 502     copyPageList         = 0;
 503     copyPageIndex        = PAGE_SIZE >> 2;
 504
 505     compressedSize       = PAGE_SIZE;
 506     stage                = 2;
 507     count                = 0;
 508     srcPhys              = 0;
 509
 510     if (gIOHibernateCurrentHeader->previewSize)
 511     {
 512         pageIndexPhys     = headerPhys
 513                            + (offsetof(IOHibernateImageHeader, fileExtentMap)
 514                            + gIOHibernateCurrentHeader->fileExtentMapSize
 515                            + ptoa_32(gIOHibernateCurrentHeader->restore1PageCount));
 516         imageReadPhys     = (pageIndexPhys + gIOHibernateCurrentHeader->previewPageListSize);
 517         lastPageIndexPage = atop_64(imageReadPhys);
 518         pageIndexSource   = (uint32_t *) pal_hib_map(IMAGE2_AREA, pageIndexPhys);
 519     }
 520     else
 521     {
 522         pageIndexPhys     = 0;
 523         lastPageIndexPage = 0;
 524         imageReadPhys     = (mapPhys + gIOHibernateCurrentHeader->bitmapSize);
 525     }
 526
 527     debug_code(kIOHibernateRestoreCodePageIndexStart, pageIndexPhys);
 528     debug_code(kIOHibernateRestoreCodePageIndexEnd,   ptoa_64(lastPageIndexPage));
 529
 530     while (1)
 531     {
 532         switch (stage)
 533         {
 534             case 2:
 535                 // copy handoff data
 536                 count = srcPhys ? 0 : handoffPageCount;
 537                 if (!count)
 538                     break;
 539                 if (count > gIOHibernateHandoffPageCount) count = gIOHibernateHandoffPageCount;
 540                 srcPhys = ptoa_64(handoffPages);
 541                 break;
 542
 543             case 1:
 544                 // copy pageIndexSource pages == preview image data
 545                 if (!srcPhys)
 546                 {
 547                     if (!pageIndexPhys) break;
 548                     srcPhys = imageReadPhys;
 549                 }
 550                 ppnum = pageIndexSource[0];
 551                 count = pageIndexSource[1];
 552                 pageIndexSource += 2;
 553                 pageIndexPhys   += 2 * sizeof(pageIndexSource[0]);
 554                 imageReadPhys = srcPhys;
 555                 break;
 556
 557             case 0:
 558                 // copy pages
 559                 if (!srcPhys) srcPhys = (mapPhys + gIOHibernateCurrentHeader->bitmapSize);
 560                 src = (uint32_t *) pal_hib_map(IMAGE_AREA, srcPhys);
 561                 ppnum = src[0];
 562                 count = src[1];
 563                 srcPhys += 2 * sizeof(*src);
 564                 imageReadPhys = srcPhys;
 565                 break;
 566         }
 567
 568
 569         if (!count)
 570         {
 571             if (!stage)
 572                 break;
 573             stage--;
 574             srcPhys = 0;
 575             continue;
 576         }
 577
 578         for (page = 0; page < count; page++, ppnum++)
 579         {
 580             uint32_t tag;
 581             int conflicts;
 582
 583             src = (uint32_t *) pal_hib_map(IMAGE_AREA, srcPhys);
 584
 585             if (2 == stage) ppnum = gIOHibernateHandoffPages[page];
 586             else if (!stage)
 587             {
 588                 tag = *src++;
 589 //              debug_code(kIOHibernateRestoreCodeTag, (uintptr_t) tag);
 590                 srcPhys += sizeof(*src);
 591                 compressedSize = kIOHibernateTagLength & tag;
 592             }
 593
 594             conflicts = (ppnum >= atop_64(mapPhys)) && (ppnum <= lastMapPage);
 595
 596             conflicts |= ((ppnum >= atop_64(imageReadPhys)) && (ppnum <= lastImagePage));
 597
 598             if (stage >= 2)
 599                 conflicts |= ((ppnum >= atop_64(srcPhys)) && (ppnum <= (handoffPages + handoffPageCount - 1)));
 600
 601             if (stage >= 1)
 602                 conflicts |= ((ppnum >= atop_64(pageIndexPhys)) && (ppnum <= lastPageIndexPage));
 603
 604             if (!conflicts)
 605             {
 606 //              if (compressedSize)
 607                 pageSum = store_one_page(gIOHibernateCurrentHeader->processorFlags,
 608                                          src, compressedSize, 0, ppnum);
 609                 if (stage != 2)
 610                     sum += pageSum;
 611                 uncompressedPages++;
 612             }
 613             else
 614             {
 615                 uint32_t   bufferPage;
 616                 uint32_t * dst;
 617
 618 //              debug_code(kIOHibernateRestoreCodeConflictPage,   ppnum);
 619 //              debug_code(kIOHibernateRestoreCodeConflictSource, (uintptr_t) src);
 620
 621                 conflictCount++;
 622
 623                 // alloc new buffer page
 624                 bufferPage = hibernate_page_list_grab(map, &nextFree);
 625
 626                 if (copyPageIndex > ((PAGE_SIZE >> 2) - 3))
 627                 {
 628                     // alloc new copy list page
 629                     pageListPage = hibernate_page_list_grab(map, &nextFree);
 630                     // link to current
 631                     if (copyPageList) {
 632                             copyPageList[1] = pageListPage;
 633                     } else {
 634                             copyPageListHeadPage = pageListPage;
 635                     }
 636                     copyPageList = (uint32_t *)pal_hib_map(SRC_COPY_AREA,
 637                                     ptoa_64(pageListPage));
 638                     copyPageList[1] = 0;
 639                     copyPageIndex = 2;
 640                 }
 641
 642                 copyPageList[copyPageIndex++] = ppnum;
 643                 copyPageList[copyPageIndex++] = bufferPage;
 644                 copyPageList[copyPageIndex++] = (compressedSize | (stage << 24));
 645                 copyPageList[0] = copyPageIndex;
 646
 647                 dst = (uint32_t *)pal_hib_map(DEST_COPY_AREA, ptoa_64(bufferPage));
 648                 for (idx = 0; idx < ((compressedSize + 3) >> 2); idx++)
 649                         dst[idx] = src[idx];
 650             }
 651             srcPhys += ((compressedSize + 3) & ~3);
 652             src     += ((compressedSize + 3) >> 2);
 653         }
 654     }
 655
 656     /* src points to the last page restored, so we need to skip over that */
 657     hibernateRestorePALState(src);
 658
 659     // -- copy back conflicts
 660
 661     pageListPage = copyPageListHeadPage;
 662     while (pageListPage)
 663     {
 664         copyPageList = (uint32_t *)pal_hib_map(COPY_PAGE_AREA, ptoa_64(pageListPage));
 665         for (copyPageIndex = 2; copyPageIndex < copyPageList[0]; copyPageIndex += 3)
 666         {
 667             ppnum          = copyPageList[copyPageIndex + 0];
 668             srcPhys        = ptoa_64(copyPageList[copyPageIndex + 1]);
 669             src            = (uint32_t *) pal_hib_map(SRC_COPY_AREA, srcPhys);
 670             compressedSize = copyPageList[copyPageIndex + 2];
 671             stage          = compressedSize >> 24;
 672             compressedSize &= 0x1FFF;
 673             pageSum        = store_one_page(gIOHibernateCurrentHeader->processorFlags,
 674                                                 src, compressedSize, 0, ppnum);
 675             if (stage != 2)
 676                 sum += pageSum;
 677             uncompressedPages++;
 678         }
 679         pageListPage = copyPageList[1];
 680     }
 681
 682     pal_hib_patchup();
 683
 684     // -- image has been destroyed...
 685
 686     gIOHibernateCurrentHeader->actualImage1Sum         = sum;
 687     gIOHibernateCurrentHeader->actualUncompressedPages = uncompressedPages;
 688     gIOHibernateCurrentHeader->conflictCount           = conflictCount;
 689     gIOHibernateCurrentHeader->nextFree                = nextFree;
 690
 691     gIOHibernateState = kIOHibernateStateWakingFromHibernate;
 692
 693 #if CONFIG_SLEEP
 694 #if defined(__i386__) || defined(__x86_64__)
 695     typedef void (*ResetProc)(void);
 696     ResetProc proc;
 697     proc = HIB_ENTRYPOINT;
 698     // flush caches
 699     __asm__("wbinvd");
 700     proc();
 701 #else
 702 // implement me
 703 #endif
 704 #endif
 705
 706     return -1;
 707 }
 708
 709 #if CONFIG_DEBUG
 710 /* standalone printf implementation */
 711 /*-
 712  * Copyright (c) 1986, 1988, 1991, 1993
 713  *      The Regents of the University of California.  All rights reserved.
 714  * (c) UNIX System Laboratories, Inc.
 715  * All or some portions of this file are derived from material licensed
 716  * to the University of California by American Telephone and Telegraph
 717  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
 718  * the permission of UNIX System Laboratories, Inc.
 719  *
 720  * Redistribution and use in source and binary forms, with or without
 721  * modification, are permitted provided that the following conditions
 722  * are met:
 723  * 1. Redistributions of source code must retain the above copyright
 724  *    notice, this list of conditions and the following disclaimer.
 725  * 2. Redistributions in binary form must reproduce the above copyright
 726  *    notice, this list of conditions and the following disclaimer in the
 727  *    documentation and/or other materials provided with the distribution.
 728  * 4. Neither the name of the University nor the names of its contributors
 729  *    may be used to endorse or promote products derived from this software
 730  *    without specific prior written permission.
 731  *
 732  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 733  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 734  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 735  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 736  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 737  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 738  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 739  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 740  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 741  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 742  * SUCH DAMAGE.
 743  *
 744  *      @(#)subr_prf.c  8.3 (Berkeley) 1/21/94
 745  */
 746
 747 typedef long ptrdiff_t;
 748 char const hibhex2ascii_data[] = "0123456789abcdefghijklmnopqrstuvwxyz";
 749 #define hibhex2ascii(hex)  (hibhex2ascii_data[hex])
 750 #define toupper(c)      ((c) - 0x20 * (((c) >= 'a') && ((c) <= 'z')))
 751 static size_t
 752 hibstrlen(const char *s)
 753 {
 754         size_t l = 0;
 755         while (*s++)
 756                 l++;
 757         return l;
 758 }
 759
 760 /* Max number conversion buffer length: a u_quad_t in base 2, plus NUL byte. */
 761 #define MAXNBUF (sizeof(intmax_t) * NBBY + 1)
 762
 763 /*
 764  * Put a NUL-terminated ASCII number (base <= 36) in a buffer in reverse
 765  * order; return an optional length and a pointer to the last character
 766  * written in the buffer (i.e., the first character of the string).
 767  * The buffer pointed to by `nbuf' must have length >= MAXNBUF.
 768  */
 769 static char *
 770 ksprintn(char *nbuf, uintmax_t num, int base, int *lenp, int upper)
 771 {
 772         char *p, c;
 773
 774         /* Truncate so we don't call umoddi3, which isn't in __HIB */
 775 #if !defined(__LP64__)
 776         uint32_t num2 = (uint32_t) num;
 777 #else
 778         uintmax_t num2 = num;
 779 #endif
 780
 781         p = nbuf;
 782         *p = '\0';
 783         do {
 784                 c = hibhex2ascii(num2 % base);
 785                 *++p = upper ? toupper(c) : c;
 786         } while (num2 /= base);
 787         if (lenp)
 788                 *lenp = (int)(p - nbuf);
 789         return (p);
 790 }
 791
 792 /*
 793  * Scaled down version of printf(3).
 794  *
 795  * Two additional formats:
 796  *
 797  * The format %b is supported to decode error registers.
 798  * Its usage is:
 799  *
 800  *      printf("reg=%b\n", regval, "*");
 801  *
 802  * where  is the output base expressed as a control character, e.g.
 803  * \10 gives octal; \20 gives hex.  Each arg is a sequence of characters,
 804  * the first of which gives the bit number to be inspected (origin 1), and
 805  * the next characters (up to a control character, i.e. a character <= 32),
 806  * give the name of the register.  Thus:
 807  *
 808  *      kvprintf("reg=%b\n", 3, "\10\2BITTWO\1BITONE\n");
 809  *
 810  * would produce output:
 811  *
 812  *      reg=3
 813  *
 814  * XXX:  %D  -- Hexdump, takes pointer and separator string:
 815  *              ("%6D", ptr, ":")   -> XX:XX:XX:XX:XX:XX
 816  *              ("%*D", len, ptr, " " -> XX XX XX XX ...
 817  */
 818 static int
 819 hibkvprintf(char const *fmt, void (*func)(int, void*), void *arg, int radix, va_list ap)
 820 {
 821 #define PCHAR(c) {int cc=(c); if (func) (*func)(cc,arg); else *d++ = cc; retval++; }
 822         char nbuf[MAXNBUF];
 823         char *d;
 824         const char *p, *percent, *q;
 825         u_char *up;
 826         int ch, n;
 827         uintmax_t num;
 828         int base, lflag, qflag, tmp, width, ladjust, sharpflag, neg, sign, dot;
 829         int cflag, hflag, jflag, tflag, zflag;
 830         int dwidth, upper;
 831         char padc;
 832         int stop = 0, retval = 0;
 833
 834         num = 0;
 835         if (!func)
 836                 d = (char *) arg;
 837         else
 838                 d = NULL;
 839
 840         if (fmt == NULL)
 841                 fmt = "(fmt null)\n";
 842
 843         if (radix < 2 || radix > 36)
 844                 radix = 10;
 845
 846         for (;;) {
 847                 padc = ' ';
 848                 width = 0;
 849                 while ((ch = (u_char)*fmt++) != '%' || stop) {
 850                         if (ch == '\0')
 851                                 return (retval);
 852                         PCHAR(ch);
 853                 }
 854                 percent = fmt - 1;
 855                 qflag = 0; lflag = 0; ladjust = 0; sharpflag = 0; neg = 0;
 856                 sign = 0; dot = 0; dwidth = 0; upper = 0;
 857                 cflag = 0; hflag = 0; jflag = 0; tflag = 0; zflag = 0;
 858 reswitch:       switch (ch = (u_char)*fmt++) {
 859                 case '.':
 860                         dot = 1;
 861                         goto reswitch;
 862                 case '#':
 863                         sharpflag = 1;
 864                         goto reswitch;
 865                 case '+':
 866                         sign = 1;
 867                         goto reswitch;
 868                 case '-':
 869                         ladjust = 1;
 870                         goto reswitch;
 871                 case '%':
 872                         PCHAR(ch);
 873                         break;
 874                 case '*':
 875                         if (!dot) {
 876                                 width = va_arg(ap, int);
 877                                 if (width < 0) {
 878                                         ladjust = !ladjust;
 879                                         width = -width;
 880                                 }
 881                         } else {
 882                                 dwidth = va_arg(ap, int);
 883                         }
 884                         goto reswitch;
 885                 case '0':
 886                         if (!dot) {
 887                                 padc = '0';
 888                                 goto reswitch;
 889                         }
 890                 case '1': case '2': case '3': case '4':
 891                 case '5': case '6': case '7': case '8': case '9':
 892                                 for (n = 0;; ++fmt) {
 893                                         n = n * 10 + ch - '0';
 894                                         ch = *fmt;
 895                                         if (ch < '0' || ch > '9')
 896                                                 break;
 897                                 }
 898                         if (dot)
 899                                 dwidth = n;
 900                         else
 901                                 width = n;
 902                         goto reswitch;
 903                 case 'b':
 904                         num = (u_int)va_arg(ap, int);
 905                         p = va_arg(ap, char *);
 906                         for (q = ksprintn(nbuf, num, *p++, NULL, 0); *q;)
 907                                 PCHAR(*q--);
 908
 909                         if (num == 0)
 910                                 break;
 911
 912                         for (tmp = 0; *p;) {
 913                                 n = *p++;
 914                                 if (num & (1 << (n - 1))) {
 915                                         PCHAR(tmp ? ',' : '<');
 916                                         for (; (n = *p) > ' '; ++p)
 917                                                 PCHAR(n);
 918                                         tmp = 1;
 919                                 } else
 920                                         for (; *p > ' '; ++p)
 921                                                 continue;
 922                         }
 923                         if (tmp)
 924                                 PCHAR('>');
 925                         break;
 926                 case 'c':
 927                         PCHAR(va_arg(ap, int));
 928                         break;
 929                 case 'D':
 930                         up = va_arg(ap, u_char *);
 931                         p = va_arg(ap, char *);
 932                         if (!width)
 933                                 width = 16;
 934                         while(width--) {
 935                                 PCHAR(hibhex2ascii(*up >> 4));
 936                                 PCHAR(hibhex2ascii(*up & 0x0f));
 937                                 up++;
 938                                 if (width)
 939                                         for (q=p;*q;q++)
 940                                                 PCHAR(*q);
 941                         }
 942                         break;
 943                 case 'd':
 944                 case 'i':
 945                         base = 10;
 946                         sign = 1;
 947                         goto handle_sign;
 948                 case 'h':
 949                         if (hflag) {
 950                                 hflag = 0;
 951                                 cflag = 1;
 952                         } else
 953                                 hflag = 1;
 954                         goto reswitch;
 955                 case 'j':
 956                         jflag = 1;
 957                         goto reswitch;
 958                 case 'l':
 959                         if (lflag) {
 960                                 lflag = 0;
 961                                 qflag = 1;
 962                         } else
 963                                 lflag = 1;
 964                         goto reswitch;
 965                 case 'n':
 966                         if (jflag)
 967                                 *(va_arg(ap, intmax_t *)) = retval;
 968                         else if (qflag)
 969                                 *(va_arg(ap, quad_t *)) = retval;
 970                         else if (lflag)
 971                                 *(va_arg(ap, long *)) = retval;
 972                         else if (zflag)
 973                                 *(va_arg(ap, size_t *)) = retval;
 974                         else if (hflag)
 975                                 *(va_arg(ap, short *)) = retval;
 976                         else if (cflag)
 977                                 *(va_arg(ap, char *)) = retval;
 978                         else
 979                                 *(va_arg(ap, int *)) = retval;
 980                         break;
 981                 case 'o':
 982                         base = 8;
 983                         goto handle_nosign;
 984                 case 'p':
 985                         base = 16;
 986                         sharpflag = (width == 0);
 987                         sign = 0;
 988                         num = (uintptr_t)va_arg(ap, void *);
 989                         goto number;
 990                 case 'q':
 991                         qflag = 1;
 992                         goto reswitch;
 993                 case 'r':
 994                         base = radix;
 995                         if (sign)
 996                                 goto handle_sign;
 997                         goto handle_nosign;
 998                 case 's':
 999                         p = va_arg(ap, char *);
1000                         if (p == NULL)
1001                                 p = "(null)";
1002                         if (!dot)
1003                                 n = (typeof(n))hibstrlen (p);
1004                         else
1005                                 for (n = 0; n < dwidth && p[n]; n++)
1006                                         continue;
1007
1008                         width -= n;
1009
1010                         if (!ladjust && width > 0)
1011                                 while (width--)
1012                                         PCHAR(padc);
1013                         while (n--)
1014                                 PCHAR(*p++);
1015                         if (ladjust && width > 0)
1016                                 while (width--)
1017                                         PCHAR(padc);
1018                         break;
1019                 case 't':
1020                         tflag = 1;
1021                         goto reswitch;
1022                 case 'u':
1023                         base = 10;
1024                         goto handle_nosign;
1025                 case 'X':
1026                         upper = 1;
1027                 case 'x':
1028                         base = 16;
1029                         goto handle_nosign;
1030                 case 'y':
1031                         base = 16;
1032                         sign = 1;
1033                         goto handle_sign;
1034                 case 'z':
1035                         zflag = 1;
1036                         goto reswitch;
1037 handle_nosign:
1038                         sign = 0;
1039                         if (jflag)
1040                                 num = va_arg(ap, uintmax_t);
1041                         else if (qflag)
1042                                 num = va_arg(ap, u_quad_t);
1043                         else if (tflag)
1044                                 num = va_arg(ap, ptrdiff_t);
1045                         else if (lflag)
1046                                 num = va_arg(ap, u_long);
1047                         else if (zflag)
1048                                 num = va_arg(ap, size_t);
1049                         else if (hflag)
1050                                 num = (u_short)va_arg(ap, int);
1051                         else if (cflag)
1052                                 num = (u_char)va_arg(ap, int);
1053                         else
1054                                 num = va_arg(ap, u_int);
1055                         goto number;
1056 handle_sign:
1057                         if (jflag)
1058                                 num = va_arg(ap, intmax_t);
1059                         else if (qflag)
1060                                 num = va_arg(ap, quad_t);
1061                         else if (tflag)
1062                                 num = va_arg(ap, ptrdiff_t);
1063                         else if (lflag)
1064                                 num = va_arg(ap, long);
1065                         else if (zflag)
1066                                 num = va_arg(ap, ssize_t);
1067                         else if (hflag)
1068                                 num = (short)va_arg(ap, int);
1069                         else if (cflag)
1070                                 num = (char)va_arg(ap, int);
1071                         else
1072                                 num = va_arg(ap, int);
1073 number:
1074                         if (sign && (intmax_t)num < 0) {
1075                                 neg = 1;
1076                                 num = -(intmax_t)num;
1077                         }
1078                         p = ksprintn(nbuf, num, base, &tmp, upper);
1079                         if (sharpflag && num != 0) {
1080                                 if (base == 8)
1081                                         tmp++;
1082                                 else if (base == 16)
1083                                         tmp += 2;
1084                         }
1085                         if (neg)
1086                                 tmp++;
1087
1088                         if (!ladjust && padc != '0' && width
1089                             && (width -= tmp) > 0)
1090                                 while (width--)
1091                                         PCHAR(padc);
1092                         if (neg)
1093                                 PCHAR('-');
1094                         if (sharpflag && num != 0) {
1095                                 if (base == 8) {
1096                                         PCHAR('0');
1097                                 } else if (base == 16) {
1098                                         PCHAR('0');
1099                                         PCHAR('x');
1100                                 }
1101                         }
1102                         if (!ladjust && width && (width -= tmp) > 0)
1103                                 while (width--)
1104                                         PCHAR(padc);
1105
1106                         while (*p)
1107                                 PCHAR(*p--);
1108
1109                         if (ladjust && width && (width -= tmp) > 0)
1110                                 while (width--)
1111                                         PCHAR(padc);
1112
1113                         break;
1114                 default:
1115                         while (percent < fmt)
1116                                 PCHAR(*percent++);
1117                         /*
1118                          * Since we ignore an formatting argument it is no
1119                          * longer safe to obey the remaining formatting
1120                          * arguments as the arguments will no longer match
1121                          * the format specs.
1122                          */
1123                         stop = 1;
1124                         break;
1125                 }
1126         }
1127 #undef PCHAR
1128 }
1129
1130
1131 static void
1132 putchar(int c, void *arg)
1133 {
1134         (void)arg;
1135         uart_putc(c);
1136 }
1137
1138 void
1139 hibprintf(const char *fmt, ...)
1140 {
1141         /* http://www.pagetable.com/?p=298 */
1142         va_list ap;
1143
1144         va_start(ap, fmt);
1145         hibkvprintf(fmt, putchar, NULL, 10, ap);
1146         va_end(ap);
1147 }
1148 #endif /* CONFIG_DEBUG */
1149