iokit/Kernel/IOHibernateRestoreKernel.c

   1 /*
   2  * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <stdint.h>
  30 #include <mach/mach_types.h>
  31 #include <mach/vm_param.h>
  32 #include <IOKit/IOHibernatePrivate.h>
  33 #include <IOKit/IOLib.h>
  34 #include <pexpert/boot.h>
  35 #include <crypto/aes.h>
  36 #include <libkern/libkern.h>
  37
  38 #include <libkern/WKdm.h>
  39 #include "IOHibernateInternal.h"
  40
  41 #if defined(__i386__) || defined(__x86_64__)
  42 #include <i386/pal_hibernate.h>
  43 #endif
  44
  45 /*
  46 This code is linked into the kernel but part of the "__HIB" section, which means
  47 its used by code running in the special context of restoring the kernel text and data
  48 from the hibernation image read by the booter. hibernate_kernel_entrypoint() and everything
  49 it calls or references needs to be careful to only touch memory also in the "__HIB" section.
  50 */
  51
  52 uint32_t gIOHibernateState;
  53
  54 uint32_t gIOHibernateDebugFlags;
  55
  56 static IOHibernateImageHeader _hibernateHeader;
  57 IOHibernateImageHeader * gIOHibernateCurrentHeader = &_hibernateHeader;
  58
  59 ppnum_t gIOHibernateHandoffPages[64];
  60 uint32_t gIOHibernateHandoffPageCount = sizeof(gIOHibernateHandoffPages)
  61                                         / sizeof(gIOHibernateHandoffPages[0]);
  62
  63 #if CONFIG_DEBUG
  64 void hibprintf(const char *fmt, ...);
  65 #else
  66 #define hibprintf(x...)
  67 #endif
  68
  69
  70 #if CONFIG_SLEEP
  71 #if defined(__i386__) || defined(__x86_64__)
  72 extern void acpi_wake_prot_entry(void);
  73 #endif
  74 #endif
  75
  76 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  77
  78 #if defined(__i386__) || defined(__x86_64__)
  79
  80 #define DBGLOG  1
  81
  82 #include <architecture/i386/pio.h>
  83
  84 /* standard port addresses */
  85 enum {
  86     COM1_PORT_ADDR = 0x3f8,
  87     COM2_PORT_ADDR = 0x2f8
  88 };
  89
  90 /* UART register offsets */
  91 enum {
  92     UART_RBR = 0,  /* receive buffer Register   (R) */
  93     UART_THR = 0,  /* transmit holding register (W) */
  94     UART_DLL = 0,  /* DLAB = 1, divisor latch (LSB) */
  95     UART_IER = 1,  /* interrupt enable register     */
  96     UART_DLM = 1,  /* DLAB = 1, divisor latch (MSB) */
  97     UART_IIR = 2,  /* interrupt ident register (R)  */
  98     UART_FCR = 2,  /* fifo control register (W)     */
  99     UART_LCR = 3,  /* line control register         */
 100     UART_MCR = 4,  /* modem control register        */
 101     UART_LSR = 5,  /* line status register          */
 102     UART_MSR = 6,  /* modem status register         */
 103     UART_SCR = 7   /* scratch register              */
 104 };
 105
 106 enum {
 107     UART_LCR_8BITS = 0x03,
 108     UART_LCR_DLAB  = 0x80
 109 };
 110
 111 enum {
 112     UART_MCR_DTR   = 0x01,
 113     UART_MCR_RTS   = 0x02,
 114     UART_MCR_OUT1  = 0x04,
 115     UART_MCR_OUT2  = 0x08,
 116     UART_MCR_LOOP  = 0x10
 117 };
 118
 119 enum {
 120     UART_LSR_DR    = 0x01,
 121     UART_LSR_OE    = 0x02,
 122     UART_LSR_PE    = 0x04,
 123     UART_LSR_FE    = 0x08,
 124     UART_LSR_THRE  = 0x20
 125 };
 126
 127 static void uart_putc(char c)
 128 {
 129     while (!(inb(COM1_PORT_ADDR + UART_LSR) & UART_LSR_THRE))
 130         {}
 131     outb(COM1_PORT_ADDR + UART_THR, c);
 132 }
 133
 134 static int debug_probe( void )
 135 {
 136     /* Verify that the Scratch Register is accessible */
 137     outb(COM1_PORT_ADDR + UART_SCR, 0x5a);
 138     if (inb(COM1_PORT_ADDR + UART_SCR) != 0x5a) return false;
 139     outb(COM1_PORT_ADDR + UART_SCR, 0xa5);
 140     if (inb(COM1_PORT_ADDR + UART_SCR) != 0xa5) return false;
 141     uart_putc('\n');
 142     return true;
 143 }
 144
 145 static void uart_puthex(uint64_t num)
 146 {
 147     int bit;
 148     char c;
 149     bool leading = true;
 150
 151     for (bit = 60; bit >= 0; bit -= 4)
 152     {
 153         c = 0xf & (num >> bit);
 154         if (c)
 155             leading = false;
 156         else if (leading && bit)
 157             continue;
 158         if (c <= 9)
 159             c += '0';
 160         else
 161             c+= 'a' - 10;
 162         uart_putc(c);
 163     }
 164 }
 165
 166 static void debug_code(uint32_t code, uint64_t value)
 167 {
 168     int bit;
 169     char c;
 170
 171     if (!(kIOHibernateDebugRestoreLogs & gIOHibernateDebugFlags))
 172         return;
 173
 174     for (bit = 24; bit >= 0; bit -= 8)
 175     {
 176         c = 0xFF & (code >> bit);
 177         if (c)
 178             uart_putc(c);
 179     }
 180     uart_putc('=');
 181     uart_puthex(value);
 182     uart_putc('\n');
 183     uart_putc('\r');
 184 }
 185
 186 #endif /* defined(__i386__) || defined(__x86_64__) */
 187
 188 #if !defined(DBGLOG)
 189 #define debug_probe()       (false)
 190 #define debug_code(c, v)    {}
 191 #endif
 192
 193 enum
 194 {
 195     kIOHibernateRestoreCodeImageStart       = 'imgS',
 196     kIOHibernateRestoreCodeImageEnd         = 'imgE',
 197     kIOHibernateRestoreCodePageIndexStart   = 'pgiS',
 198     kIOHibernateRestoreCodePageIndexEnd     = 'pgiE',
 199     kIOHibernateRestoreCodeMapStart         = 'mapS',
 200     kIOHibernateRestoreCodeMapEnd           = 'mapE',
 201     kIOHibernateRestoreCodeWakeMapSize      = 'wkms',
 202     kIOHibernateRestoreCodeConflictPage     = 'cfpg',
 203     kIOHibernateRestoreCodeConflictSource   = 'cfsr',
 204     kIOHibernateRestoreCodeNoMemory         = 'nomm',
 205     kIOHibernateRestoreCodeTag              = 'tag ',
 206     kIOHibernateRestoreCodeSignature        = 'sign',
 207     kIOHibernateRestoreCodeMapVirt          = 'mapV',
 208     kIOHibernateRestoreCodeHandoffPages     = 'hand',
 209     kIOHibernateRestoreCodeHandoffCount     = 'hndc',
 210 };
 211
 212 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 213
 214
 215 static void fatal(void)
 216 {
 217 #if defined(__i386__) || defined(__x86_64__)
 218     outb(0xcf9, 6);
 219 #else
 220     while (true) {}
 221 #endif
 222 }
 223
 224 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 225
 226 uint32_t
 227 hibernate_sum_page(uint8_t *buf, uint32_t ppnum)
 228 {
 229     return (((uint32_t *)buf)[((PAGE_SIZE >> 2) - 1) & ppnum]);
 230 }
 231
 232 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 233
 234 static hibernate_bitmap_t *
 235 hibernate_page_bitmap(hibernate_page_list_t * list, uint32_t page)
 236 {
 237     uint32_t             bank;
 238     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 239
 240     for (bank = 0; bank < list->bank_count; bank++)
 241     {
 242         if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
 243             break;
 244         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 245     }
 246     if (bank == list->bank_count)
 247         bitmap = NULL;
 248
 249     return (bitmap);
 250 }
 251
 252 hibernate_bitmap_t *
 253 hibernate_page_bitmap_pin(hibernate_page_list_t * list, uint32_t * pPage)
 254 {
 255     uint32_t             bank, page = *pPage;
 256     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 257
 258     for (bank = 0; bank < list->bank_count; bank++)
 259     {
 260         if (page <= bitmap->first_page)
 261         {
 262             *pPage = bitmap->first_page;
 263             break;
 264         }
 265         if (page <= bitmap->last_page)
 266             break;
 267         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 268     }
 269     if (bank == list->bank_count)
 270         bitmap = NULL;
 271
 272     return (bitmap);
 273 }
 274
 275 void
 276 hibernate_page_bitset(hibernate_page_list_t * list, boolean_t set, uint32_t page)
 277 {
 278     hibernate_bitmap_t * bitmap;
 279
 280     bitmap = hibernate_page_bitmap(list, page);
 281     if (bitmap)
 282     {
 283         page -= bitmap->first_page;
 284         if (set)
 285             bitmap->bitmap[page >> 5] |= (0x80000000 >> (page & 31));
 286             //setbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
 287         else
 288             bitmap->bitmap[page >> 5] &= ~(0x80000000 >> (page & 31));
 289             //clrbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
 290     }
 291 }
 292
 293 boolean_t
 294 hibernate_page_bittst(hibernate_page_list_t * list, uint32_t page)
 295 {
 296     boolean_t            result = TRUE;
 297     hibernate_bitmap_t * bitmap;
 298
 299     bitmap = hibernate_page_bitmap(list, page);
 300     if (bitmap)
 301     {
 302         page -= bitmap->first_page;
 303         result = (0 != (bitmap->bitmap[page >> 5] & (0x80000000 >> (page & 31))));
 304     }
 305     return (result);
 306 }
 307
 308 // count bits clear or set (set == TRUE) starting at page.
 309 uint32_t
 310 hibernate_page_bitmap_count(hibernate_bitmap_t * bitmap, uint32_t set, uint32_t page)
 311 {
 312     uint32_t index, bit, bits;
 313     uint32_t count;
 314
 315     count = 0;
 316
 317     index = (page - bitmap->first_page) >> 5;
 318     bit = (page - bitmap->first_page) & 31;
 319
 320     bits = bitmap->bitmap[index];
 321     if (set)
 322         bits = ~bits;
 323     bits = (bits << bit);
 324     if (bits)
 325         count += __builtin_clz(bits);
 326     else
 327     {
 328         count += 32 - bit;
 329         while (++index < bitmap->bitmapwords)
 330         {
 331             bits = bitmap->bitmap[index];
 332             if (set)
 333                 bits = ~bits;
 334             if (bits)
 335             {
 336                 count += __builtin_clz(bits);
 337                 break;
 338             }
 339             count += 32;
 340         }
 341     }
 342
 343     if ((page + count) > (bitmap->last_page + 1)) count = (bitmap->last_page + 1) - page;
 344
 345     return (count);
 346 }
 347
 348 static ppnum_t
 349 hibernate_page_list_grab(hibernate_page_list_t * list, uint32_t * pNextFree)
 350 {
 351     uint32_t             nextFree = *pNextFree;
 352     uint32_t             nextFreeInBank;
 353     hibernate_bitmap_t * bitmap;
 354
 355     nextFreeInBank = nextFree + 1;
 356     while ((bitmap = hibernate_page_bitmap_pin(list, &nextFreeInBank)))
 357     {
 358         nextFreeInBank += hibernate_page_bitmap_count(bitmap, FALSE, nextFreeInBank);
 359         if (nextFreeInBank <= bitmap->last_page)
 360         {
 361             *pNextFree = nextFreeInBank;
 362             break;
 363         }
 364     }
 365
 366     if (!bitmap)
 367     {
 368         debug_code(kIOHibernateRestoreCodeNoMemory, nextFree);
 369         fatal();
 370         nextFree = 0;
 371     }
 372
 373     return (nextFree);
 374 }
 375
 376 static uint32_t
 377 store_one_page(uint32_t procFlags, uint32_t * src, uint32_t compressedSize,
 378                 uint32_t * buffer, uint32_t ppnum)
 379 {
 380         uint64_t dst = ptoa_64(ppnum);
 381
 382         if (compressedSize != PAGE_SIZE)
 383         {
 384                 dst = pal_hib_map(DEST_COPY_AREA, dst);
 385                 WKdm_decompress((WK_word*) src, (WK_word*)(uintptr_t)dst, PAGE_SIZE >> 2);
 386         }
 387         else
 388         {
 389                 dst = hibernate_restore_phys_page((uint64_t) (uintptr_t) src, dst, PAGE_SIZE, procFlags);
 390         }
 391
 392         return hibernate_sum_page((uint8_t *)(uintptr_t)dst, ppnum);
 393 }
 394
 395 // used only for small struct copies
 396 static void
 397 bcopy_internal(const void *src, void *dst, uint32_t len)
 398 {
 399     const char *s = src;
 400     char       *d = dst;
 401     uint32_t   idx = 0;
 402
 403     while (idx < len)
 404     {
 405         d[idx] = s[idx];
 406         idx++;
 407     }
 408 }
 409
 410 #define C_ASSERT(e) typedef char    __C_ASSERT__[(e) ? 1 : -1]
 411
 412 long
 413 hibernate_kernel_entrypoint(uint32_t p1,
 414                             uint32_t p2, uint32_t p3, uint32_t p4)
 415 {
 416     uint64_t headerPhys;
 417     uint64_t mapPhys;
 418     uint64_t srcPhys;
 419     uint64_t imageReadPhys;
 420     uint64_t pageIndexPhys;
 421     uint32_t idx;
 422     uint32_t * pageIndexSource;
 423     hibernate_page_list_t * map;
 424     uint32_t stage;
 425     uint32_t count;
 426     uint32_t ppnum;
 427     uint32_t page;
 428     uint32_t conflictCount;
 429     uint32_t compressedSize;
 430     uint32_t uncompressedPages;
 431     uint32_t copyPageListHeadPage;
 432     uint32_t pageListPage;
 433     uint32_t * copyPageList;
 434     uint32_t * src;
 435     uint32_t copyPageIndex;
 436     uint32_t sum;
 437     uint32_t pageSum;
 438     uint32_t nextFree;
 439     uint32_t lastImagePage;
 440     uint32_t lastMapPage;
 441     uint32_t lastPageIndexPage;
 442     uint32_t handoffPages;
 443     uint32_t handoffPageCount;
 444
 445     C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
 446
 447     headerPhys = ptoa_64(p1);
 448
 449     if ((kIOHibernateDebugRestoreLogs & gIOHibernateDebugFlags) && !debug_probe())
 450         gIOHibernateDebugFlags &= ~kIOHibernateDebugRestoreLogs;
 451
 452     debug_code(kIOHibernateRestoreCodeImageStart, headerPhys);
 453
 454     bcopy_internal((void *) pal_hib_map(IMAGE_AREA, headerPhys),
 455                    gIOHibernateCurrentHeader,
 456                    sizeof(IOHibernateImageHeader));
 457
 458     debug_code(kIOHibernateRestoreCodeSignature, gIOHibernateCurrentHeader->signature);
 459
 460     mapPhys = headerPhys
 461              + (offsetof(IOHibernateImageHeader, fileExtentMap)
 462              + gIOHibernateCurrentHeader->fileExtentMapSize
 463              + ptoa_32(gIOHibernateCurrentHeader->restore1PageCount)
 464              + gIOHibernateCurrentHeader->previewSize);
 465
 466     map = (hibernate_page_list_t *) pal_hib_map(BITMAP_AREA, mapPhys);
 467
 468     lastImagePage = atop_64(headerPhys + gIOHibernateCurrentHeader->image1Size);
 469     lastMapPage = atop_64(mapPhys + gIOHibernateCurrentHeader->bitmapSize);
 470
 471     handoffPages     = gIOHibernateCurrentHeader->handoffPages;
 472     handoffPageCount = gIOHibernateCurrentHeader->handoffPageCount;
 473
 474     debug_code(kIOHibernateRestoreCodeImageEnd,       ptoa_64(lastImagePage));
 475     debug_code(kIOHibernateRestoreCodeMapStart,       mapPhys);
 476     debug_code(kIOHibernateRestoreCodeMapEnd,         ptoa_64(lastMapPage));
 477
 478     debug_code(kIOHibernateRestoreCodeMapVirt, (uintptr_t) map);
 479     debug_code(kIOHibernateRestoreCodeHandoffPages, ptoa_64(handoffPages));
 480     debug_code(kIOHibernateRestoreCodeHandoffCount, handoffPageCount);
 481
 482     // knock all the image pages to be used out of free map
 483     for (ppnum = atop_64(headerPhys); ppnum <= lastImagePage; ppnum++)
 484     {
 485         hibernate_page_bitset(map, FALSE, ppnum);
 486     }
 487     // knock all the handoff pages to be used out of free map
 488     for (ppnum = handoffPages; ppnum < (handoffPages + handoffPageCount); ppnum++)
 489     {
 490         hibernate_page_bitset(map, FALSE, ppnum);
 491     }
 492
 493     nextFree = 0;
 494     hibernate_page_list_grab(map, &nextFree);
 495
 496     sum = gIOHibernateCurrentHeader->actualRestore1Sum;
 497     gIOHibernateCurrentHeader->diag[0] = atop_64(headerPhys);
 498     gIOHibernateCurrentHeader->diag[1] = sum;
 499
 500     uncompressedPages    = 0;
 501     conflictCount        = 0;
 502     copyPageListHeadPage = 0;
 503     copyPageList         = 0;
 504     copyPageIndex        = PAGE_SIZE >> 2;
 505
 506     compressedSize       = PAGE_SIZE;
 507     stage                = 2;
 508     count                = 0;
 509     srcPhys              = 0;
 510
 511     if (gIOHibernateCurrentHeader->previewSize)
 512     {
 513         pageIndexPhys     = headerPhys
 514                            + (offsetof(IOHibernateImageHeader, fileExtentMap)
 515                            + gIOHibernateCurrentHeader->fileExtentMapSize
 516                            + ptoa_32(gIOHibernateCurrentHeader->restore1PageCount));
 517         imageReadPhys     = (pageIndexPhys + gIOHibernateCurrentHeader->previewPageListSize);
 518         lastPageIndexPage = atop_64(imageReadPhys);
 519         pageIndexSource   = (uint32_t *) pal_hib_map(IMAGE2_AREA, pageIndexPhys);
 520     }
 521     else
 522     {
 523         pageIndexPhys     = 0;
 524         lastPageIndexPage = 0;
 525         imageReadPhys     = (mapPhys + gIOHibernateCurrentHeader->bitmapSize);
 526     }
 527
 528     debug_code(kIOHibernateRestoreCodePageIndexStart, pageIndexPhys);
 529     debug_code(kIOHibernateRestoreCodePageIndexEnd,   ptoa_64(lastPageIndexPage));
 530
 531     while (1)
 532     {
 533         switch (stage)
 534         {
 535             case 2:
 536                 // copy handoff data
 537                 count = srcPhys ? 0 : handoffPageCount;
 538                 if (!count)
 539                     break;
 540                 if (count > gIOHibernateHandoffPageCount) count = gIOHibernateHandoffPageCount;
 541                 srcPhys = ptoa_64(handoffPages);
 542                 break;
 543
 544             case 1:
 545                 // copy pageIndexSource pages == preview image data
 546                 if (!srcPhys)
 547                 {
 548                     if (!pageIndexPhys) break;
 549                     srcPhys = imageReadPhys;
 550                 }
 551                 ppnum = pageIndexSource[0];
 552                 count = pageIndexSource[1];
 553                 pageIndexSource += 2;
 554                 pageIndexPhys   += 2 * sizeof(pageIndexSource[0]);
 555                 imageReadPhys = srcPhys;
 556                 break;
 557
 558             case 0:
 559                 // copy pages
 560                 if (!srcPhys) srcPhys = (mapPhys + gIOHibernateCurrentHeader->bitmapSize);
 561                 src = (uint32_t *) pal_hib_map(IMAGE_AREA, srcPhys);
 562                 ppnum = src[0];
 563                 count = src[1];
 564                 srcPhys += 2 * sizeof(*src);
 565                 imageReadPhys = srcPhys;
 566                 break;
 567         }
 568
 569
 570         if (!count)
 571         {
 572             if (!stage)
 573                 break;
 574             stage--;
 575             srcPhys = 0;
 576             continue;
 577         }
 578
 579         for (page = 0; page < count; page++, ppnum++)
 580         {
 581             uint32_t tag;
 582             int conflicts;
 583
 584             src = (uint32_t *) pal_hib_map(IMAGE_AREA, srcPhys);
 585
 586             if (2 == stage) ppnum = gIOHibernateHandoffPages[page];
 587             else if (!stage)
 588             {
 589                 tag = *src++;
 590 //              debug_code(kIOHibernateRestoreCodeTag, (uintptr_t) tag);
 591                 srcPhys += sizeof(*src);
 592                 compressedSize = kIOHibernateTagLength & tag;
 593             }
 594
 595             conflicts = (ppnum >= atop_64(mapPhys)) && (ppnum <= lastMapPage);
 596
 597             conflicts |= ((ppnum >= atop_64(imageReadPhys)) && (ppnum <= lastImagePage));
 598
 599             if (stage >= 2)
 600                 conflicts |= ((ppnum >= atop_64(srcPhys)) && (ppnum <= (handoffPages + handoffPageCount - 1)));
 601
 602             if (stage >= 1)
 603                 conflicts |= ((ppnum >= atop_64(pageIndexPhys)) && (ppnum <= lastPageIndexPage));
 604
 605             if (!conflicts)
 606             {
 607 //              if (compressedSize)
 608                 pageSum = store_one_page(gIOHibernateCurrentHeader->processorFlags,
 609                                          src, compressedSize, 0, ppnum);
 610                 if (stage != 2)
 611                     sum += pageSum;
 612                 uncompressedPages++;
 613             }
 614             else
 615             {
 616                 uint32_t   bufferPage;
 617                 uint32_t * dst;
 618
 619 //              debug_code(kIOHibernateRestoreCodeConflictPage,   ppnum);
 620 //              debug_code(kIOHibernateRestoreCodeConflictSource, (uintptr_t) src);
 621
 622                 conflictCount++;
 623
 624                 // alloc new buffer page
 625                 bufferPage = hibernate_page_list_grab(map, &nextFree);
 626
 627                 if (copyPageIndex > ((PAGE_SIZE >> 2) - 3))
 628                 {
 629                     // alloc new copy list page
 630                     pageListPage = hibernate_page_list_grab(map, &nextFree);
 631                     // link to current
 632                     if (copyPageList) {
 633                             copyPageList[1] = pageListPage;
 634                     } else {
 635                             copyPageListHeadPage = pageListPage;
 636                     }
 637                     copyPageList = (uint32_t *)pal_hib_map(SRC_COPY_AREA,
 638                                     ptoa_64(pageListPage));
 639                     copyPageList[1] = 0;
 640                     copyPageIndex = 2;
 641                 }
 642
 643                 copyPageList[copyPageIndex++] = ppnum;
 644                 copyPageList[copyPageIndex++] = bufferPage;
 645                 copyPageList[copyPageIndex++] = (compressedSize | (stage << 24));
 646                 copyPageList[0] = copyPageIndex;
 647
 648                 dst = (uint32_t *)pal_hib_map(DEST_COPY_AREA, ptoa_64(bufferPage));
 649                 for (idx = 0; idx < ((compressedSize + 3) >> 2); idx++)
 650                         dst[idx] = src[idx];
 651             }
 652             srcPhys += ((compressedSize + 3) & ~3);
 653             src     += ((compressedSize + 3) >> 2);
 654         }
 655     }
 656
 657     /* src points to the last page restored, so we need to skip over that */
 658     hibernateRestorePALState(src);
 659
 660     // -- copy back conflicts
 661
 662     pageListPage = copyPageListHeadPage;
 663     while (pageListPage)
 664     {
 665         copyPageList = (uint32_t *)pal_hib_map(COPY_PAGE_AREA, ptoa_64(pageListPage));
 666         for (copyPageIndex = 2; copyPageIndex < copyPageList[0]; copyPageIndex += 3)
 667         {
 668             ppnum          = copyPageList[copyPageIndex + 0];
 669             srcPhys        = ptoa_64(copyPageList[copyPageIndex + 1]);
 670             src            = (uint32_t *) pal_hib_map(SRC_COPY_AREA, srcPhys);
 671             compressedSize = copyPageList[copyPageIndex + 2];
 672             stage          = compressedSize >> 24;
 673             compressedSize &= 0x1FFF;
 674             pageSum        = store_one_page(gIOHibernateCurrentHeader->processorFlags,
 675                                                 src, compressedSize, 0, ppnum);
 676             if (stage != 2)
 677                 sum += pageSum;
 678             uncompressedPages++;
 679         }
 680         pageListPage = copyPageList[1];
 681     }
 682
 683     pal_hib_patchup();
 684
 685     // -- image has been destroyed...
 686
 687     gIOHibernateCurrentHeader->actualImage1Sum         = sum;
 688     gIOHibernateCurrentHeader->actualUncompressedPages = uncompressedPages;
 689     gIOHibernateCurrentHeader->conflictCount           = conflictCount;
 690     gIOHibernateCurrentHeader->nextFree                = nextFree;
 691
 692     gIOHibernateState = kIOHibernateStateWakingFromHibernate;
 693
 694 #if CONFIG_SLEEP
 695 #if defined(__i386__) || defined(__x86_64__)
 696     typedef void (*ResetProc)(void);
 697     ResetProc proc;
 698     proc = HIB_ENTRYPOINT;
 699     // flush caches
 700     __asm__("wbinvd");
 701     proc();
 702 #else
 703 // implement me
 704 #endif
 705 #endif
 706
 707     return -1;
 708 }
 709
 710 #if CONFIG_DEBUG
 711 /* standalone printf implementation */
 712 /*-
 713  * Copyright (c) 1986, 1988, 1991, 1993
 714  *      The Regents of the University of California.  All rights reserved.
 715  * (c) UNIX System Laboratories, Inc.
 716  * All or some portions of this file are derived from material licensed
 717  * to the University of California by American Telephone and Telegraph
 718  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
 719  * the permission of UNIX System Laboratories, Inc.
 720  *
 721  * Redistribution and use in source and binary forms, with or without
 722  * modification, are permitted provided that the following conditions
 723  * are met:
 724  * 1. Redistributions of source code must retain the above copyright
 725  *    notice, this list of conditions and the following disclaimer.
 726  * 2. Redistributions in binary form must reproduce the above copyright
 727  *    notice, this list of conditions and the following disclaimer in the
 728  *    documentation and/or other materials provided with the distribution.
 729  * 4. Neither the name of the University nor the names of its contributors
 730  *    may be used to endorse or promote products derived from this software
 731  *    without specific prior written permission.
 732  *
 733  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 734  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 735  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 736  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 737  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 738  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 739  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 740  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 741  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 742  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 743  * SUCH DAMAGE.
 744  *
 745  *      @(#)subr_prf.c  8.3 (Berkeley) 1/21/94
 746  */
 747
 748 typedef long ptrdiff_t;
 749 char const hibhex2ascii_data[] = "0123456789abcdefghijklmnopqrstuvwxyz";
 750 #define hibhex2ascii(hex)  (hibhex2ascii_data[hex])
 751 #define toupper(c)      ((c) - 0x20 * (((c) >= 'a') && ((c) <= 'z')))
 752 static size_t
 753 hibstrlen(const char *s)
 754 {
 755         size_t l = 0;
 756         while (*s++)
 757                 l++;
 758         return l;
 759 }
 760
 761 /* Max number conversion buffer length: a u_quad_t in base 2, plus NUL byte. */
 762 #define MAXNBUF (sizeof(intmax_t) * NBBY + 1)
 763
 764 /*
 765  * Put a NUL-terminated ASCII number (base <= 36) in a buffer in reverse
 766  * order; return an optional length and a pointer to the last character
 767  * written in the buffer (i.e., the first character of the string).
 768  * The buffer pointed to by `nbuf' must have length >= MAXNBUF.
 769  */
 770 static char *
 771 ksprintn(char *nbuf, uintmax_t num, int base, int *lenp, int upper)
 772 {
 773         char *p, c;
 774
 775         /* Truncate so we don't call umoddi3, which isn't in __HIB */
 776 #if !defined(__LP64__)
 777         uint32_t num2 = (uint32_t) num;
 778 #else
 779         uintmax_t num2 = num;
 780 #endif
 781
 782         p = nbuf;
 783         *p = '\0';
 784         do {
 785                 c = hibhex2ascii(num2 % base);
 786                 *++p = upper ? toupper(c) : c;
 787         } while (num2 /= base);
 788         if (lenp)
 789                 *lenp = (int)(p - nbuf);
 790         return (p);
 791 }
 792
 793 /*
 794  * Scaled down version of printf(3).
 795  *
 796  * Two additional formats:
 797  *
 798  * The format %b is supported to decode error registers.
 799  * Its usage is:
 800  *
 801  *      printf("reg=%b\n", regval, "*");
 802  *
 803  * where  is the output base expressed as a control character, e.g.
 804  * \10 gives octal; \20 gives hex.  Each arg is a sequence of characters,
 805  * the first of which gives the bit number to be inspected (origin 1), and
 806  * the next characters (up to a control character, i.e. a character <= 32),
 807  * give the name of the register.  Thus:
 808  *
 809  *      kvprintf("reg=%b\n", 3, "\10\2BITTWO\1BITONE\n");
 810  *
 811  * would produce output:
 812  *
 813  *      reg=3
 814  *
 815  * XXX:  %D  -- Hexdump, takes pointer and separator string:
 816  *              ("%6D", ptr, ":")   -> XX:XX:XX:XX:XX:XX
 817  *              ("%*D", len, ptr, " " -> XX XX XX XX ...
 818  */
 819 static int
 820 hibkvprintf(char const *fmt, void (*func)(int, void*), void *arg, int radix, va_list ap)
 821 {
 822 #define PCHAR(c) {int cc=(c); if (func) (*func)(cc,arg); else *d++ = cc; retval++; }
 823         char nbuf[MAXNBUF];
 824         char *d;
 825         const char *p, *percent, *q;
 826         u_char *up;
 827         int ch, n;
 828         uintmax_t num;
 829         int base, lflag, qflag, tmp, width, ladjust, sharpflag, neg, sign, dot;
 830         int cflag, hflag, jflag, tflag, zflag;
 831         int dwidth, upper;
 832         char padc;
 833         int stop = 0, retval = 0;
 834
 835         num = 0;
 836         if (!func)
 837                 d = (char *) arg;
 838         else
 839                 d = NULL;
 840
 841         if (fmt == NULL)
 842                 fmt = "(fmt null)\n";
 843
 844         if (radix < 2 || radix > 36)
 845                 radix = 10;
 846
 847         for (;;) {
 848                 padc = ' ';
 849                 width = 0;
 850                 while ((ch = (u_char)*fmt++) != '%' || stop) {
 851                         if (ch == '\0')
 852                                 return (retval);
 853                         PCHAR(ch);
 854                 }
 855                 percent = fmt - 1;
 856                 qflag = 0; lflag = 0; ladjust = 0; sharpflag = 0; neg = 0;
 857                 sign = 0; dot = 0; dwidth = 0; upper = 0;
 858                 cflag = 0; hflag = 0; jflag = 0; tflag = 0; zflag = 0;
 859 reswitch:       switch (ch = (u_char)*fmt++) {
 860                 case '.':
 861                         dot = 1;
 862                         goto reswitch;
 863                 case '#':
 864                         sharpflag = 1;
 865                         goto reswitch;
 866                 case '+':
 867                         sign = 1;
 868                         goto reswitch;
 869                 case '-':
 870                         ladjust = 1;
 871                         goto reswitch;
 872                 case '%':
 873                         PCHAR(ch);
 874                         break;
 875                 case '*':
 876                         if (!dot) {
 877                                 width = va_arg(ap, int);
 878                                 if (width < 0) {
 879                                         ladjust = !ladjust;
 880                                         width = -width;
 881                                 }
 882                         } else {
 883                                 dwidth = va_arg(ap, int);
 884                         }
 885                         goto reswitch;
 886                 case '0':
 887                         if (!dot) {
 888                                 padc = '0';
 889                                 goto reswitch;
 890                         }
 891                 case '1': case '2': case '3': case '4':
 892                 case '5': case '6': case '7': case '8': case '9':
 893                                 for (n = 0;; ++fmt) {
 894                                         n = n * 10 + ch - '0';
 895                                         ch = *fmt;
 896                                         if (ch < '0' || ch > '9')
 897                                                 break;
 898                                 }
 899                         if (dot)
 900                                 dwidth = n;
 901                         else
 902                                 width = n;
 903                         goto reswitch;
 904                 case 'b':
 905                         num = (u_int)va_arg(ap, int);
 906                         p = va_arg(ap, char *);
 907                         for (q = ksprintn(nbuf, num, *p++, NULL, 0); *q;)
 908                                 PCHAR(*q--);
 909
 910                         if (num == 0)
 911                                 break;
 912
 913                         for (tmp = 0; *p;) {
 914                                 n = *p++;
 915                                 if (num & (1 << (n - 1))) {
 916                                         PCHAR(tmp ? ',' : '<');
 917                                         for (; (n = *p) > ' '; ++p)
 918                                                 PCHAR(n);
 919                                         tmp = 1;
 920                                 } else
 921                                         for (; *p > ' '; ++p)
 922                                                 continue;
 923                         }
 924                         if (tmp)
 925                                 PCHAR('>');
 926                         break;
 927                 case 'c':
 928                         PCHAR(va_arg(ap, int));
 929                         break;
 930                 case 'D':
 931                         up = va_arg(ap, u_char *);
 932                         p = va_arg(ap, char *);
 933                         if (!width)
 934                                 width = 16;
 935                         while(width--) {
 936                                 PCHAR(hibhex2ascii(*up >> 4));
 937                                 PCHAR(hibhex2ascii(*up & 0x0f));
 938                                 up++;
 939                                 if (width)
 940                                         for (q=p;*q;q++)
 941                                                 PCHAR(*q);
 942                         }
 943                         break;
 944                 case 'd':
 945                 case 'i':
 946                         base = 10;
 947                         sign = 1;
 948                         goto handle_sign;
 949                 case 'h':
 950                         if (hflag) {
 951                                 hflag = 0;
 952                                 cflag = 1;
 953                         } else
 954                                 hflag = 1;
 955                         goto reswitch;
 956                 case 'j':
 957                         jflag = 1;
 958                         goto reswitch;
 959                 case 'l':
 960                         if (lflag) {
 961                                 lflag = 0;
 962                                 qflag = 1;
 963                         } else
 964                                 lflag = 1;
 965                         goto reswitch;
 966                 case 'n':
 967                         if (jflag)
 968                                 *(va_arg(ap, intmax_t *)) = retval;
 969                         else if (qflag)
 970                                 *(va_arg(ap, quad_t *)) = retval;
 971                         else if (lflag)
 972                                 *(va_arg(ap, long *)) = retval;
 973                         else if (zflag)
 974                                 *(va_arg(ap, size_t *)) = retval;
 975                         else if (hflag)
 976                                 *(va_arg(ap, short *)) = retval;
 977                         else if (cflag)
 978                                 *(va_arg(ap, char *)) = retval;
 979                         else
 980                                 *(va_arg(ap, int *)) = retval;
 981                         break;
 982                 case 'o':
 983                         base = 8;
 984                         goto handle_nosign;
 985                 case 'p':
 986                         base = 16;
 987                         sharpflag = (width == 0);
 988                         sign = 0;
 989                         num = (uintptr_t)va_arg(ap, void *);
 990                         goto number;
 991                 case 'q':
 992                         qflag = 1;
 993                         goto reswitch;
 994                 case 'r':
 995                         base = radix;
 996                         if (sign)
 997                                 goto handle_sign;
 998                         goto handle_nosign;
 999                 case 's':
1000                         p = va_arg(ap, char *);
1001                         if (p == NULL)
1002                                 p = "(null)";
1003                         if (!dot)
1004                                 n = (typeof(n))hibstrlen (p);
1005                         else
1006                                 for (n = 0; n < dwidth && p[n]; n++)
1007                                         continue;
1008
1009                         width -= n;
1010
1011                         if (!ladjust && width > 0)
1012                                 while (width--)
1013                                         PCHAR(padc);
1014                         while (n--)
1015                                 PCHAR(*p++);
1016                         if (ladjust && width > 0)
1017                                 while (width--)
1018                                         PCHAR(padc);
1019                         break;
1020                 case 't':
1021                         tflag = 1;
1022                         goto reswitch;
1023                 case 'u':
1024                         base = 10;
1025                         goto handle_nosign;
1026                 case 'X':
1027                         upper = 1;
1028                 case 'x':
1029                         base = 16;
1030                         goto handle_nosign;
1031                 case 'y':
1032                         base = 16;
1033                         sign = 1;
1034                         goto handle_sign;
1035                 case 'z':
1036                         zflag = 1;
1037                         goto reswitch;
1038 handle_nosign:
1039                         sign = 0;
1040                         if (jflag)
1041                                 num = va_arg(ap, uintmax_t);
1042                         else if (qflag)
1043                                 num = va_arg(ap, u_quad_t);
1044                         else if (tflag)
1045                                 num = va_arg(ap, ptrdiff_t);
1046                         else if (lflag)
1047                                 num = va_arg(ap, u_long);
1048                         else if (zflag)
1049                                 num = va_arg(ap, size_t);
1050                         else if (hflag)
1051                                 num = (u_short)va_arg(ap, int);
1052                         else if (cflag)
1053                                 num = (u_char)va_arg(ap, int);
1054                         else
1055                                 num = va_arg(ap, u_int);
1056                         goto number;
1057 handle_sign:
1058                         if (jflag)
1059                                 num = va_arg(ap, intmax_t);
1060                         else if (qflag)
1061                                 num = va_arg(ap, quad_t);
1062                         else if (tflag)
1063                                 num = va_arg(ap, ptrdiff_t);
1064                         else if (lflag)
1065                                 num = va_arg(ap, long);
1066                         else if (zflag)
1067                                 num = va_arg(ap, ssize_t);
1068                         else if (hflag)
1069                                 num = (short)va_arg(ap, int);
1070                         else if (cflag)
1071                                 num = (char)va_arg(ap, int);
1072                         else
1073                                 num = va_arg(ap, int);
1074 number:
1075                         if (sign && (intmax_t)num < 0) {
1076                                 neg = 1;
1077                                 num = -(intmax_t)num;
1078                         }
1079                         p = ksprintn(nbuf, num, base, &tmp, upper);
1080                         if (sharpflag && num != 0) {
1081                                 if (base == 8)
1082                                         tmp++;
1083                                 else if (base == 16)
1084                                         tmp += 2;
1085                         }
1086                         if (neg)
1087                                 tmp++;
1088
1089                         if (!ladjust && padc != '0' && width
1090                             && (width -= tmp) > 0)
1091                                 while (width--)
1092                                         PCHAR(padc);
1093                         if (neg)
1094                                 PCHAR('-');
1095                         if (sharpflag && num != 0) {
1096                                 if (base == 8) {
1097                                         PCHAR('0');
1098                                 } else if (base == 16) {
1099                                         PCHAR('0');
1100                                         PCHAR('x');
1101                                 }
1102                         }
1103                         if (!ladjust && width && (width -= tmp) > 0)
1104                                 while (width--)
1105                                         PCHAR(padc);
1106
1107                         while (*p)
1108                                 PCHAR(*p--);
1109
1110                         if (ladjust && width && (width -= tmp) > 0)
1111                                 while (width--)
1112                                         PCHAR(padc);
1113
1114                         break;
1115                 default:
1116                         while (percent < fmt)
1117                                 PCHAR(*percent++);
1118                         /*
1119                          * Since we ignore an formatting argument it is no
1120                          * longer safe to obey the remaining formatting
1121                          * arguments as the arguments will no longer match
1122                          * the format specs.
1123                          */
1124                         stop = 1;
1125                         break;
1126                 }
1127         }
1128 #undef PCHAR
1129 }
1130
1131
1132 static void
1133 putchar(int c, void *arg)
1134 {
1135         (void)arg;
1136         uart_putc(c);
1137 }
1138
1139 void
1140 hibprintf(const char *fmt, ...)
1141 {
1142         /* http://www.pagetable.com/?p=298 */
1143         va_list ap;
1144
1145         va_start(ap, fmt);
1146         hibkvprintf(fmt, putchar, NULL, 10, ap);
1147         va_end(ap);
1148 }
1149 #endif /* CONFIG_DEBUG */
1150