iokit/Kernel/IOHibernateRestoreKernel.c

   1 /*
   2  * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <stdint.h>
  30 #include <mach/mach_types.h>
  31 #include <mach/vm_param.h>
  32 #include <IOKit/IOHibernatePrivate.h>
  33 #include <IOKit/IOLib.h>
  34 #include <pexpert/boot.h>
  35 #include <libkern/libkern.h>
  36
  37 #include <vm/WKdm_new.h>
  38 #include "IOHibernateInternal.h"
  39
  40 #include <machine/pal_hibernate.h>
  41
  42 /*
  43 This code is linked into the kernel but part of the "__HIB" section, which means
  44 its used by code running in the special context of restoring the kernel text and data
  45 from the hibernation image read by the booter. hibernate_kernel_entrypoint() and everything
  46 it calls or references needs to be careful to only touch memory also in the "__HIB" section.
  47 */
  48
  49 uint32_t gIOHibernateState;
  50
  51 uint32_t gIOHibernateDebugFlags;
  52
  53 static IOHibernateImageHeader _hibernateHeader;
  54 IOHibernateImageHeader * gIOHibernateCurrentHeader = &_hibernateHeader;
  55
  56 ppnum_t gIOHibernateHandoffPages[64];
  57 uint32_t gIOHibernateHandoffPageCount = sizeof(gIOHibernateHandoffPages)
  58                                         / sizeof(gIOHibernateHandoffPages[0]);
  59
  60 #if CONFIG_DEBUG
  61 void hibprintf(const char *fmt, ...);
  62 #else
  63 #define hibprintf(x...)
  64 #endif
  65
  66
  67 #if CONFIG_SLEEP
  68 #if defined(__i386__) || defined(__x86_64__)
  69 extern void acpi_wake_prot_entry(void);
  70 #endif
  71 #endif
  72
  73 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  74
  75 #if defined(__i386__) || defined(__x86_64__)
  76 #include <i386/proc_reg.h>
  77 #else
  78
  79 static inline uint64_t rdtsc64(void)
  80 {
  81     return (0);
  82 }
  83
  84 #endif /* defined(__i386__) || defined(__x86_64__) */
  85
  86 #if defined(__i386__) || defined(__x86_64__)
  87
  88 #define DBGLOG  1
  89
  90 #include <architecture/i386/pio.h>
  91
  92 /* standard port addresses */
  93 enum {
  94     COM1_PORT_ADDR = 0x3f8,
  95     COM2_PORT_ADDR = 0x2f8
  96 };
  97
  98 /* UART register offsets */
  99 enum {
 100     UART_RBR = 0,  /* receive buffer Register   (R) */
 101     UART_THR = 0,  /* transmit holding register (W) */
 102     UART_DLL = 0,  /* DLAB = 1, divisor latch (LSB) */
 103     UART_IER = 1,  /* interrupt enable register     */
 104     UART_DLM = 1,  /* DLAB = 1, divisor latch (MSB) */
 105     UART_IIR = 2,  /* interrupt ident register (R)  */
 106     UART_FCR = 2,  /* fifo control register (W)     */
 107     UART_LCR = 3,  /* line control register         */
 108     UART_MCR = 4,  /* modem control register        */
 109     UART_LSR = 5,  /* line status register          */
 110     UART_MSR = 6,  /* modem status register         */
 111     UART_SCR = 7   /* scratch register              */
 112 };
 113
 114 enum {
 115     UART_LCR_8BITS = 0x03,
 116     UART_LCR_DLAB  = 0x80
 117 };
 118
 119 enum {
 120     UART_MCR_DTR   = 0x01,
 121     UART_MCR_RTS   = 0x02,
 122     UART_MCR_OUT1  = 0x04,
 123     UART_MCR_OUT2  = 0x08,
 124     UART_MCR_LOOP  = 0x10
 125 };
 126
 127 enum {
 128     UART_LSR_DR    = 0x01,
 129     UART_LSR_OE    = 0x02,
 130     UART_LSR_PE    = 0x04,
 131     UART_LSR_FE    = 0x08,
 132     UART_LSR_THRE  = 0x20
 133 };
 134
 135 static void uart_putc(char c)
 136 {
 137     while (!(inb(COM1_PORT_ADDR + UART_LSR) & UART_LSR_THRE))
 138         {}
 139     outb(COM1_PORT_ADDR + UART_THR, c);
 140 }
 141
 142 static int debug_probe( void )
 143 {
 144     /* Verify that the Scratch Register is accessible */
 145     outb(COM1_PORT_ADDR + UART_SCR, 0x5a);
 146     if (inb(COM1_PORT_ADDR + UART_SCR) != 0x5a) return false;
 147     outb(COM1_PORT_ADDR + UART_SCR, 0xa5);
 148     if (inb(COM1_PORT_ADDR + UART_SCR) != 0xa5) return false;
 149     uart_putc('\n');
 150     return true;
 151 }
 152
 153 static void uart_puthex(uint64_t num)
 154 {
 155     int bit;
 156     char c;
 157     bool leading = true;
 158
 159     for (bit = 60; bit >= 0; bit -= 4)
 160     {
 161         c = 0xf & (num >> bit);
 162         if (c)
 163             leading = false;
 164         else if (leading && bit)
 165             continue;
 166         if (c <= 9)
 167             c += '0';
 168         else
 169             c+= 'a' - 10;
 170         uart_putc(c);
 171     }
 172 }
 173
 174 static void debug_code(uint32_t code, uint64_t value)
 175 {
 176     int bit;
 177     char c;
 178
 179     if (!(kIOHibernateDebugRestoreLogs & gIOHibernateDebugFlags))
 180         return;
 181
 182     for (bit = 24; bit >= 0; bit -= 8)
 183     {
 184         c = 0xFF & (code >> bit);
 185         if (c)
 186             uart_putc(c);
 187     }
 188     uart_putc('=');
 189     uart_puthex(value);
 190     uart_putc('\n');
 191     uart_putc('\r');
 192 }
 193
 194 #endif /* defined(__i386__) || defined(__x86_64__) */
 195
 196 #if !defined(DBGLOG)
 197 #define debug_probe()       (false)
 198 #define debug_code(c, v)    {}
 199 #endif
 200
 201 enum
 202 {
 203     kIOHibernateRestoreCodeImageStart       = 'imgS',
 204     kIOHibernateRestoreCodeImageEnd         = 'imgE',
 205     kIOHibernateRestoreCodePageIndexStart   = 'pgiS',
 206     kIOHibernateRestoreCodePageIndexEnd     = 'pgiE',
 207     kIOHibernateRestoreCodeMapStart         = 'mapS',
 208     kIOHibernateRestoreCodeMapEnd           = 'mapE',
 209     kIOHibernateRestoreCodeWakeMapSize      = 'wkms',
 210     kIOHibernateRestoreCodeConflictPage     = 'cfpg',
 211     kIOHibernateRestoreCodeConflictSource   = 'cfsr',
 212     kIOHibernateRestoreCodeNoMemory         = 'nomm',
 213     kIOHibernateRestoreCodeTag              = 'tag ',
 214     kIOHibernateRestoreCodeSignature        = 'sign',
 215     kIOHibernateRestoreCodeMapVirt          = 'mapV',
 216     kIOHibernateRestoreCodeHandoffPages     = 'hand',
 217     kIOHibernateRestoreCodeHandoffCount     = 'hndc',
 218 };
 219
 220 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 221
 222
 223 static void fatal(void)
 224 {
 225 #if defined(__i386__) || defined(__x86_64__)
 226     outb(0xcf9, 6);
 227 #else
 228     while (true) {}
 229 #endif
 230 }
 231
 232 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 233
 234 uint32_t
 235 hibernate_sum_page(uint8_t *buf, uint32_t ppnum)
 236 {
 237     return (((uint32_t *)buf)[((PAGE_SIZE >> 2) - 1) & ppnum]);
 238 }
 239
 240 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 241
 242 static hibernate_bitmap_t *
 243 hibernate_page_bitmap(hibernate_page_list_t * list, uint32_t page)
 244 {
 245     uint32_t             bank;
 246     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 247
 248     for (bank = 0; bank < list->bank_count; bank++)
 249     {
 250         if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
 251             break;
 252         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 253     }
 254     if (bank == list->bank_count)
 255         bitmap = NULL;
 256
 257     return (bitmap);
 258 }
 259
 260 hibernate_bitmap_t *
 261 hibernate_page_bitmap_pin(hibernate_page_list_t * list, uint32_t * pPage)
 262 {
 263     uint32_t             bank, page = *pPage;
 264     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 265
 266     for (bank = 0; bank < list->bank_count; bank++)
 267     {
 268         if (page <= bitmap->first_page)
 269         {
 270             *pPage = bitmap->first_page;
 271             break;
 272         }
 273         if (page <= bitmap->last_page)
 274             break;
 275         bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 276     }
 277     if (bank == list->bank_count)
 278         bitmap = NULL;
 279
 280     return (bitmap);
 281 }
 282
 283 void
 284 hibernate_page_bitset(hibernate_page_list_t * list, boolean_t set, uint32_t page)
 285 {
 286     hibernate_bitmap_t * bitmap;
 287
 288     bitmap = hibernate_page_bitmap(list, page);
 289     if (bitmap)
 290     {
 291         page -= bitmap->first_page;
 292         if (set)
 293             bitmap->bitmap[page >> 5] |= (0x80000000 >> (page & 31));
 294             //setbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
 295         else
 296             bitmap->bitmap[page >> 5] &= ~(0x80000000 >> (page & 31));
 297             //clrbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
 298     }
 299 }
 300
 301 boolean_t
 302 hibernate_page_bittst(hibernate_page_list_t * list, uint32_t page)
 303 {
 304     boolean_t            result = TRUE;
 305     hibernate_bitmap_t * bitmap;
 306
 307     bitmap = hibernate_page_bitmap(list, page);
 308     if (bitmap)
 309     {
 310         page -= bitmap->first_page;
 311         result = (0 != (bitmap->bitmap[page >> 5] & (0x80000000 >> (page & 31))));
 312     }
 313     return (result);
 314 }
 315
 316 // count bits clear or set (set == TRUE) starting at page.
 317 uint32_t
 318 hibernate_page_bitmap_count(hibernate_bitmap_t * bitmap, uint32_t set, uint32_t page)
 319 {
 320     uint32_t index, bit, bits;
 321     uint32_t count;
 322
 323     count = 0;
 324
 325     index = (page - bitmap->first_page) >> 5;
 326     bit = (page - bitmap->first_page) & 31;
 327
 328     bits = bitmap->bitmap[index];
 329     if (set)
 330         bits = ~bits;
 331     bits = (bits << bit);
 332     if (bits)
 333         count += __builtin_clz(bits);
 334     else
 335     {
 336         count += 32 - bit;
 337         while (++index < bitmap->bitmapwords)
 338         {
 339             bits = bitmap->bitmap[index];
 340             if (set)
 341                 bits = ~bits;
 342             if (bits)
 343             {
 344                 count += __builtin_clz(bits);
 345                 break;
 346             }
 347             count += 32;
 348         }
 349     }
 350
 351     if ((page + count) > (bitmap->last_page + 1)) count = (bitmap->last_page + 1) - page;
 352
 353     return (count);
 354 }
 355
 356 static ppnum_t
 357 hibernate_page_list_grab(hibernate_page_list_t * list, uint32_t * pNextFree)
 358 {
 359     uint32_t             nextFree = *pNextFree;
 360     uint32_t             nextFreeInBank;
 361     hibernate_bitmap_t * bitmap;
 362
 363     nextFreeInBank = nextFree + 1;
 364     while ((bitmap = hibernate_page_bitmap_pin(list, &nextFreeInBank)))
 365     {
 366         nextFreeInBank += hibernate_page_bitmap_count(bitmap, FALSE, nextFreeInBank);
 367         if (nextFreeInBank <= bitmap->last_page)
 368         {
 369             *pNextFree = nextFreeInBank;
 370             break;
 371         }
 372     }
 373
 374     if (!bitmap)
 375     {
 376         debug_code(kIOHibernateRestoreCodeNoMemory, nextFree);
 377         fatal();
 378         nextFree = 0;
 379     }
 380
 381     return (nextFree);
 382 }
 383
 384 static uint32_t
 385 store_one_page(uint32_t procFlags, uint32_t * src, uint32_t compressedSize,
 386                 uint32_t * buffer, uint32_t ppnum)
 387 {
 388         uint64_t dst = ptoa_64(ppnum);
 389         uint8_t scratch[WKdm_SCRATCH_BUF_SIZE_INTERNAL] __attribute__ ((aligned (16)));
 390
 391         if (compressedSize != PAGE_SIZE)
 392         {
 393                 dst = pal_hib_map(DEST_COPY_AREA, dst);
 394                 if (compressedSize != 4) WKdm_decompress_new((WK_word*) src, (WK_word*)(uintptr_t)dst, (WK_word*) &scratch[0], compressedSize);
 395                 else
 396                 {
 397                         size_t i;
 398                         uint32_t s, *d;
 399
 400                         s = *src;
 401                         d = (uint32_t *)(uintptr_t)dst;
 402             if (!s) __nosan_bzero((void *) dst, PAGE_SIZE);
 403             else    for (i = 0; i < (PAGE_SIZE / sizeof(int32_t)); i++) *d++ = s;
 404                 }
 405         }
 406         else
 407         {
 408                 dst = hibernate_restore_phys_page((uint64_t) (uintptr_t) src, dst, PAGE_SIZE, procFlags);
 409         }
 410
 411         return hibernate_sum_page((uint8_t *)(uintptr_t)dst, ppnum);
 412 }
 413
 414 long
 415 hibernate_kernel_entrypoint(uint32_t p1,
 416                             uint32_t p2, uint32_t p3, uint32_t p4)
 417 {
 418     uint64_t headerPhys;
 419     uint64_t mapPhys;
 420     uint64_t srcPhys;
 421     uint64_t imageReadPhys;
 422     uint64_t pageIndexPhys;
 423     uint32_t * pageIndexSource;
 424     hibernate_page_list_t * map;
 425     uint32_t stage;
 426     uint32_t count;
 427     uint32_t ppnum;
 428     uint32_t page;
 429     uint32_t conflictCount;
 430     uint32_t compressedSize;
 431     uint32_t uncompressedPages;
 432     uint32_t copyPageListHeadPage;
 433     uint32_t pageListPage;
 434     uint32_t * copyPageList;
 435     uint32_t * src;
 436     uint32_t copyPageIndex;
 437     uint32_t sum;
 438     uint32_t pageSum;
 439     uint32_t nextFree;
 440     uint32_t lastImagePage;
 441     uint32_t lastMapPage;
 442     uint32_t lastPageIndexPage;
 443     uint32_t handoffPages;
 444     uint32_t handoffPageCount;
 445
 446     uint64_t timeStart;
 447     timeStart = rdtsc64();
 448
 449     static_assert(sizeof(IOHibernateImageHeader) == 512);
 450
 451     headerPhys = ptoa_64(p1);
 452
 453     if ((kIOHibernateDebugRestoreLogs & gIOHibernateDebugFlags) && !debug_probe())
 454         gIOHibernateDebugFlags &= ~kIOHibernateDebugRestoreLogs;
 455
 456     debug_code(kIOHibernateRestoreCodeImageStart, headerPhys);
 457
 458     __nosan_memcpy(gIOHibernateCurrentHeader,
 459            (void *) pal_hib_map(IMAGE_AREA, headerPhys),
 460            sizeof(IOHibernateImageHeader));
 461
 462     debug_code(kIOHibernateRestoreCodeSignature, gIOHibernateCurrentHeader->signature);
 463
 464     mapPhys = headerPhys
 465              + (offsetof(IOHibernateImageHeader, fileExtentMap)
 466              + gIOHibernateCurrentHeader->fileExtentMapSize
 467              + ptoa_32(gIOHibernateCurrentHeader->restore1PageCount)
 468              + gIOHibernateCurrentHeader->previewSize);
 469
 470     map = (hibernate_page_list_t *) pal_hib_map(BITMAP_AREA, mapPhys);
 471
 472     lastImagePage = atop_64(headerPhys + gIOHibernateCurrentHeader->image1Size);
 473     lastMapPage = atop_64(mapPhys + gIOHibernateCurrentHeader->bitmapSize);
 474
 475     handoffPages     = gIOHibernateCurrentHeader->handoffPages;
 476     handoffPageCount = gIOHibernateCurrentHeader->handoffPageCount;
 477
 478     debug_code(kIOHibernateRestoreCodeImageEnd,       ptoa_64(lastImagePage));
 479     debug_code(kIOHibernateRestoreCodeMapStart,       mapPhys);
 480     debug_code(kIOHibernateRestoreCodeMapEnd,         ptoa_64(lastMapPage));
 481
 482     debug_code(kIOHibernateRestoreCodeMapVirt, (uintptr_t) map);
 483     debug_code(kIOHibernateRestoreCodeHandoffPages, ptoa_64(handoffPages));
 484     debug_code(kIOHibernateRestoreCodeHandoffCount, handoffPageCount);
 485
 486     // knock all the image pages to be used out of free map
 487     for (ppnum = atop_64(headerPhys); ppnum <= lastImagePage; ppnum++)
 488     {
 489         hibernate_page_bitset(map, FALSE, ppnum);
 490     }
 491     // knock all the handoff pages to be used out of free map
 492     for (ppnum = handoffPages; ppnum < (handoffPages + handoffPageCount); ppnum++)
 493     {
 494         hibernate_page_bitset(map, FALSE, ppnum);
 495     }
 496
 497     nextFree = 0;
 498     hibernate_page_list_grab(map, &nextFree);
 499
 500     sum = gIOHibernateCurrentHeader->actualRestore1Sum;
 501     gIOHibernateCurrentHeader->diag[0] = atop_64(headerPhys);
 502     gIOHibernateCurrentHeader->diag[1] = sum;
 503     gIOHibernateCurrentHeader->trampolineTime = 0;
 504
 505     uncompressedPages    = 0;
 506     conflictCount        = 0;
 507     copyPageListHeadPage = 0;
 508     copyPageList         = 0;
 509     copyPageIndex        = PAGE_SIZE >> 2;
 510
 511     compressedSize       = PAGE_SIZE;
 512     stage                = 2;
 513     count                = 0;
 514     srcPhys              = 0;
 515
 516     if (gIOHibernateCurrentHeader->previewSize)
 517     {
 518         pageIndexPhys     = headerPhys
 519                            + (offsetof(IOHibernateImageHeader, fileExtentMap)
 520                            + gIOHibernateCurrentHeader->fileExtentMapSize
 521                            + ptoa_32(gIOHibernateCurrentHeader->restore1PageCount));
 522         imageReadPhys     = (pageIndexPhys + gIOHibernateCurrentHeader->previewPageListSize);
 523         lastPageIndexPage = atop_64(imageReadPhys);
 524         pageIndexSource   = (uint32_t *) pal_hib_map(IMAGE2_AREA, pageIndexPhys);
 525     }
 526     else
 527     {
 528         pageIndexPhys     = 0;
 529         lastPageIndexPage = 0;
 530         imageReadPhys     = (mapPhys + gIOHibernateCurrentHeader->bitmapSize);
 531     }
 532
 533     debug_code(kIOHibernateRestoreCodePageIndexStart, pageIndexPhys);
 534     debug_code(kIOHibernateRestoreCodePageIndexEnd,   ptoa_64(lastPageIndexPage));
 535
 536     while (1)
 537     {
 538         switch (stage)
 539         {
 540             case 2:
 541                 // copy handoff data
 542                 count = srcPhys ? 0 : handoffPageCount;
 543                 if (!count)
 544                     break;
 545                 if (count > gIOHibernateHandoffPageCount) count = gIOHibernateHandoffPageCount;
 546                 srcPhys = ptoa_64(handoffPages);
 547                 break;
 548
 549             case 1:
 550                 // copy pageIndexSource pages == preview image data
 551                 if (!srcPhys)
 552                 {
 553                     if (!pageIndexPhys) break;
 554                     srcPhys = imageReadPhys;
 555                 }
 556                 ppnum = pageIndexSource[0];
 557                 count = pageIndexSource[1];
 558                 pageIndexSource += 2;
 559                 pageIndexPhys   += 2 * sizeof(pageIndexSource[0]);
 560                 imageReadPhys = srcPhys;
 561                 break;
 562
 563             case 0:
 564                 // copy pages
 565                 if (!srcPhys) srcPhys = (mapPhys + gIOHibernateCurrentHeader->bitmapSize);
 566                 src = (uint32_t *) pal_hib_map(IMAGE_AREA, srcPhys);
 567                 ppnum = src[0];
 568                 count = src[1];
 569                 srcPhys += 2 * sizeof(*src);
 570                 imageReadPhys = srcPhys;
 571                 break;
 572         }
 573
 574
 575         if (!count)
 576         {
 577             if (!stage)
 578                 break;
 579             stage--;
 580             srcPhys = 0;
 581             continue;
 582         }
 583
 584         for (page = 0; page < count; page++, ppnum++)
 585         {
 586             uint32_t tag;
 587             int conflicts;
 588
 589             src = (uint32_t *) pal_hib_map(IMAGE_AREA, srcPhys);
 590
 591             if (2 == stage) ppnum = gIOHibernateHandoffPages[page];
 592             else if (!stage)
 593             {
 594                 tag = *src++;
 595 //              debug_code(kIOHibernateRestoreCodeTag, (uintptr_t) tag);
 596                 srcPhys += sizeof(*src);
 597                 compressedSize = kIOHibernateTagLength & tag;
 598             }
 599
 600             conflicts = (ppnum >= atop_64(mapPhys)) && (ppnum <= lastMapPage);
 601
 602             conflicts |= ((ppnum >= atop_64(imageReadPhys)) && (ppnum <= lastImagePage));
 603
 604             if (stage >= 2)
 605                 conflicts |= ((ppnum >= atop_64(srcPhys)) && (ppnum <= (handoffPages + handoffPageCount - 1)));
 606
 607             if (stage >= 1)
 608                 conflicts |= ((ppnum >= atop_64(pageIndexPhys)) && (ppnum <= lastPageIndexPage));
 609
 610             if (!conflicts)
 611             {
 612                 pageSum = store_one_page(gIOHibernateCurrentHeader->processorFlags,
 613                                          src, compressedSize, 0, ppnum);
 614                 if (stage != 2)
 615                     sum += pageSum;
 616                 uncompressedPages++;
 617             }
 618             else
 619             {
 620                 uint32_t   bufferPage = 0;
 621                 uint32_t * dst;
 622
 623 //              debug_code(kIOHibernateRestoreCodeConflictPage,   ppnum);
 624 //              debug_code(kIOHibernateRestoreCodeConflictSource, (uintptr_t) src);
 625                 conflictCount++;
 626                 if (compressedSize)
 627                 {
 628                     // alloc new buffer page
 629                     bufferPage = hibernate_page_list_grab(map, &nextFree);
 630                     dst = (uint32_t *)pal_hib_map(DEST_COPY_AREA, ptoa_64(bufferPage));
 631                     __nosan_memcpy(dst, src, compressedSize);
 632                 }
 633                 if (copyPageIndex > ((PAGE_SIZE >> 2) - 3))
 634                 {
 635                     // alloc new copy list page
 636                     pageListPage = hibernate_page_list_grab(map, &nextFree);
 637                     // link to current
 638                     if (copyPageList) {
 639                             copyPageList[1] = pageListPage;
 640                     } else {
 641                             copyPageListHeadPage = pageListPage;
 642                     }
 643                     copyPageList = (uint32_t *)pal_hib_map(SRC_COPY_AREA,
 644                                     ptoa_64(pageListPage));
 645                     copyPageList[1] = 0;
 646                     copyPageIndex = 2;
 647                 }
 648                 copyPageList[copyPageIndex++] = ppnum;
 649                 copyPageList[copyPageIndex++] = bufferPage;
 650                 copyPageList[copyPageIndex++] = (compressedSize | (stage << 24));
 651                 copyPageList[0] = copyPageIndex;
 652             }
 653             srcPhys += ((compressedSize + 3) & ~3);
 654             src     += ((compressedSize + 3) >> 2);
 655         }
 656     }
 657
 658     /* src points to the last page restored, so we need to skip over that */
 659     hibernateRestorePALState(src);
 660
 661     // -- copy back conflicts
 662
 663     pageListPage = copyPageListHeadPage;
 664     while (pageListPage)
 665     {
 666         copyPageList = (uint32_t *)pal_hib_map(COPY_PAGE_AREA, ptoa_64(pageListPage));
 667         for (copyPageIndex = 2; copyPageIndex < copyPageList[0]; copyPageIndex += 3)
 668         {
 669             ppnum          = copyPageList[copyPageIndex + 0];
 670             srcPhys        = ptoa_64(copyPageList[copyPageIndex + 1]);
 671             src            = (uint32_t *) pal_hib_map(SRC_COPY_AREA, srcPhys);
 672             compressedSize = copyPageList[copyPageIndex + 2];
 673             stage          = compressedSize >> 24;
 674             compressedSize &= 0x1FFF;
 675             pageSum        = store_one_page(gIOHibernateCurrentHeader->processorFlags,
 676                                                 src, compressedSize, 0, ppnum);
 677             if (stage != 2)
 678                 sum += pageSum;
 679             uncompressedPages++;
 680         }
 681         pageListPage = copyPageList[1];
 682     }
 683
 684     pal_hib_patchup();
 685
 686     // -- image has been destroyed...
 687
 688     gIOHibernateCurrentHeader->actualImage1Sum         = sum;
 689     gIOHibernateCurrentHeader->actualUncompressedPages = uncompressedPages;
 690     gIOHibernateCurrentHeader->conflictCount           = conflictCount;
 691     gIOHibernateCurrentHeader->nextFree                = nextFree;
 692
 693     gIOHibernateState = kIOHibernateStateWakingFromHibernate;
 694
 695     gIOHibernateCurrentHeader->trampolineTime = (((rdtsc64() - timeStart)) >> 8);
 696
 697 //  debug_code('done', 0);
 698
 699 #if CONFIG_SLEEP
 700 #if defined(__i386__) || defined(__x86_64__)
 701     typedef void (*ResetProc)(void);
 702     ResetProc proc;
 703     proc = HIB_ENTRYPOINT;
 704     // flush caches
 705     __asm__("wbinvd");
 706     proc();
 707 #else
 708 // implement me
 709 #endif
 710 #endif
 711
 712     return -1;
 713 }
 714
 715 #if CONFIG_DEBUG
 716 /* standalone printf implementation */
 717 /*-
 718  * Copyright (c) 1986, 1988, 1991, 1993
 719  *      The Regents of the University of California.  All rights reserved.
 720  * (c) UNIX System Laboratories, Inc.
 721  * All or some portions of this file are derived from material licensed
 722  * to the University of California by American Telephone and Telegraph
 723  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
 724  * the permission of UNIX System Laboratories, Inc.
 725  *
 726  * Redistribution and use in source and binary forms, with or without
 727  * modification, are permitted provided that the following conditions
 728  * are met:
 729  * 1. Redistributions of source code must retain the above copyright
 730  *    notice, this list of conditions and the following disclaimer.
 731  * 2. Redistributions in binary form must reproduce the above copyright
 732  *    notice, this list of conditions and the following disclaimer in the
 733  *    documentation and/or other materials provided with the distribution.
 734  * 4. Neither the name of the University nor the names of its contributors
 735  *    may be used to endorse or promote products derived from this software
 736  *    without specific prior written permission.
 737  *
 738  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 739  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 740  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 741  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 742  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 743  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 744  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 745  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 746  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 747  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 748  * SUCH DAMAGE.
 749  *
 750  *      @(#)subr_prf.c  8.3 (Berkeley) 1/21/94
 751  */
 752
 753 typedef long ptrdiff_t;
 754 char const hibhex2ascii_data[] = "0123456789abcdefghijklmnopqrstuvwxyz";
 755 #define hibhex2ascii(hex)  (hibhex2ascii_data[hex])
 756 #define toupper(c)      ((c) - 0x20 * (((c) >= 'a') && ((c) <= 'z')))
 757 static size_t
 758 hibstrlen(const char *s)
 759 {
 760         size_t l = 0;
 761         while (*s++)
 762                 l++;
 763         return l;
 764 }
 765
 766 /* Max number conversion buffer length: a u_quad_t in base 2, plus NUL byte. */
 767 #define MAXNBUF (sizeof(intmax_t) * NBBY + 1)
 768
 769 /*
 770  * Put a NUL-terminated ASCII number (base <= 36) in a buffer in reverse
 771  * order; return an optional length and a pointer to the last character
 772  * written in the buffer (i.e., the first character of the string).
 773  * The buffer pointed to by `nbuf' must have length >= MAXNBUF.
 774  */
 775 static char *
 776 ksprintn(char *nbuf, uintmax_t num, int base, int *lenp, int upper)
 777 {
 778         char *p, c;
 779
 780         /* Truncate so we don't call umoddi3, which isn't in __HIB */
 781 #if !defined(__LP64__)
 782         uint32_t num2 = (uint32_t) num;
 783 #else
 784         uintmax_t num2 = num;
 785 #endif
 786
 787         p = nbuf;
 788         *p = '\0';
 789         do {
 790                 c = hibhex2ascii(num2 % base);
 791                 *++p = upper ? toupper(c) : c;
 792         } while (num2 /= base);
 793         if (lenp)
 794                 *lenp = (int)(p - nbuf);
 795         return (p);
 796 }
 797
 798 /*
 799  * Scaled down version of printf(3).
 800  *
 801  * Two additional formats:
 802  *
 803  * The format %b is supported to decode error registers.
 804  * Its usage is:
 805  *
 806  *      printf("reg=%b\n", regval, "*");
 807  *
 808  * where  is the output base expressed as a control character, e.g.
 809  * \10 gives octal; \20 gives hex.  Each arg is a sequence of characters,
 810  * the first of which gives the bit number to be inspected (origin 1), and
 811  * the next characters (up to a control character, i.e. a character <= 32),
 812  * give the name of the register.  Thus:
 813  *
 814  *      kvprintf("reg=%b\n", 3, "\10\2BITTWO\1BITONE\n");
 815  *
 816  * would produce output:
 817  *
 818  *      reg=3
 819  *
 820  * XXX:  %D  -- Hexdump, takes pointer and separator string:
 821  *              ("%6D", ptr, ":")   -> XX:XX:XX:XX:XX:XX
 822  *              ("%*D", len, ptr, " " -> XX XX XX XX ...
 823  */
 824 static int
 825 hibkvprintf(char const *fmt, void (*func)(int, void*), void *arg, int radix, va_list ap)
 826 {
 827 #define PCHAR(c) {int cc=(c); if (func) (*func)(cc,arg); else *d++ = cc; retval++; }
 828         char nbuf[MAXNBUF];
 829         char *d;
 830         const char *p, *percent, *q;
 831         u_char *up;
 832         int ch, n;
 833         uintmax_t num;
 834         int base, lflag, qflag, tmp, width, ladjust, sharpflag, neg, sign, dot;
 835         int cflag, hflag, jflag, tflag, zflag;
 836         int dwidth, upper;
 837         char padc;
 838         int stop = 0, retval = 0;
 839
 840         num = 0;
 841         if (!func)
 842                 d = (char *) arg;
 843         else
 844                 d = NULL;
 845
 846         if (fmt == NULL)
 847                 fmt = "(fmt null)\n";
 848
 849         if (radix < 2 || radix > 36)
 850                 radix = 10;
 851
 852         for (;;) {
 853                 padc = ' ';
 854                 width = 0;
 855                 while ((ch = (u_char)*fmt++) != '%' || stop) {
 856                         if (ch == '\0')
 857                                 return (retval);
 858                         PCHAR(ch);
 859                 }
 860                 percent = fmt - 1;
 861                 qflag = 0; lflag = 0; ladjust = 0; sharpflag = 0; neg = 0;
 862                 sign = 0; dot = 0; dwidth = 0; upper = 0;
 863                 cflag = 0; hflag = 0; jflag = 0; tflag = 0; zflag = 0;
 864 reswitch:       switch (ch = (u_char)*fmt++) {
 865                 case '.':
 866                         dot = 1;
 867                         goto reswitch;
 868                 case '#':
 869                         sharpflag = 1;
 870                         goto reswitch;
 871                 case '+':
 872                         sign = 1;
 873                         goto reswitch;
 874                 case '-':
 875                         ladjust = 1;
 876                         goto reswitch;
 877                 case '%':
 878                         PCHAR(ch);
 879                         break;
 880                 case '*':
 881                         if (!dot) {
 882                                 width = va_arg(ap, int);
 883                                 if (width < 0) {
 884                                         ladjust = !ladjust;
 885                                         width = -width;
 886                                 }
 887                         } else {
 888                                 dwidth = va_arg(ap, int);
 889                         }
 890                         goto reswitch;
 891                 case '0':
 892                         if (!dot) {
 893                                 padc = '0';
 894                                 goto reswitch;
 895                         }
 896                 case '1': case '2': case '3': case '4':
 897                 case '5': case '6': case '7': case '8': case '9':
 898                                 for (n = 0;; ++fmt) {
 899                                         n = n * 10 + ch - '0';
 900                                         ch = *fmt;
 901                                         if (ch < '0' || ch > '9')
 902                                                 break;
 903                                 }
 904                         if (dot)
 905                                 dwidth = n;
 906                         else
 907                                 width = n;
 908                         goto reswitch;
 909                 case 'b':
 910                         num = (u_int)va_arg(ap, int);
 911                         p = va_arg(ap, char *);
 912                         for (q = ksprintn(nbuf, num, *p++, NULL, 0); *q;)
 913                                 PCHAR(*q--);
 914
 915                         if (num == 0)
 916                                 break;
 917
 918                         for (tmp = 0; *p;) {
 919                                 n = *p++;
 920                                 if (num & (1 << (n - 1))) {
 921                                         PCHAR(tmp ? ',' : '<');
 922                                         for (; (n = *p) > ' '; ++p)
 923                                                 PCHAR(n);
 924                                         tmp = 1;
 925                                 } else
 926                                         for (; *p > ' '; ++p)
 927                                                 continue;
 928                         }
 929                         if (tmp)
 930                                 PCHAR('>');
 931                         break;
 932                 case 'c':
 933                         PCHAR(va_arg(ap, int));
 934                         break;
 935                 case 'D':
 936                         up = va_arg(ap, u_char *);
 937                         p = va_arg(ap, char *);
 938                         if (!width)
 939                                 width = 16;
 940                         while(width--) {
 941                                 PCHAR(hibhex2ascii(*up >> 4));
 942                                 PCHAR(hibhex2ascii(*up & 0x0f));
 943                                 up++;
 944                                 if (width)
 945                                         for (q=p;*q;q++)
 946                                                 PCHAR(*q);
 947                         }
 948                         break;
 949                 case 'd':
 950                 case 'i':
 951                         base = 10;
 952                         sign = 1;
 953                         goto handle_sign;
 954                 case 'h':
 955                         if (hflag) {
 956                                 hflag = 0;
 957                                 cflag = 1;
 958                         } else
 959                                 hflag = 1;
 960                         goto reswitch;
 961                 case 'j':
 962                         jflag = 1;
 963                         goto reswitch;
 964                 case 'l':
 965                         if (lflag) {
 966                                 lflag = 0;
 967                                 qflag = 1;
 968                         } else
 969                                 lflag = 1;
 970                         goto reswitch;
 971                 case 'n':
 972                         if (jflag)
 973                                 *(va_arg(ap, intmax_t *)) = retval;
 974                         else if (qflag)
 975                                 *(va_arg(ap, quad_t *)) = retval;
 976                         else if (lflag)
 977                                 *(va_arg(ap, long *)) = retval;
 978                         else if (zflag)
 979                                 *(va_arg(ap, size_t *)) = retval;
 980                         else if (hflag)
 981                                 *(va_arg(ap, short *)) = retval;
 982                         else if (cflag)
 983                                 *(va_arg(ap, char *)) = retval;
 984                         else
 985                                 *(va_arg(ap, int *)) = retval;
 986                         break;
 987                 case 'o':
 988                         base = 8;
 989                         goto handle_nosign;
 990                 case 'p':
 991                         base = 16;
 992                         sharpflag = (width == 0);
 993                         sign = 0;
 994                         num = (uintptr_t)va_arg(ap, void *);
 995                         goto number;
 996                 case 'q':
 997                         qflag = 1;
 998                         goto reswitch;
 999                 case 'r':
1000                         base = radix;
1001                         if (sign)
1002                                 goto handle_sign;
1003                         goto handle_nosign;
1004                 case 's':
1005                         p = va_arg(ap, char *);
1006                         if (p == NULL)
1007                                 p = "(null)";
1008                         if (!dot)
1009                                 n = (typeof(n))hibstrlen (p);
1010                         else
1011                                 for (n = 0; n < dwidth && p[n]; n++)
1012                                         continue;
1013
1014                         width -= n;
1015
1016                         if (!ladjust && width > 0)
1017                                 while (width--)
1018                                         PCHAR(padc);
1019                         while (n--)
1020                                 PCHAR(*p++);
1021                         if (ladjust && width > 0)
1022                                 while (width--)
1023                                         PCHAR(padc);
1024                         break;
1025                 case 't':
1026                         tflag = 1;
1027                         goto reswitch;
1028                 case 'u':
1029                         base = 10;
1030                         goto handle_nosign;
1031                 case 'X':
1032                         upper = 1;
1033                 case 'x':
1034                         base = 16;
1035                         goto handle_nosign;
1036                 case 'y':
1037                         base = 16;
1038                         sign = 1;
1039                         goto handle_sign;
1040                 case 'z':
1041                         zflag = 1;
1042                         goto reswitch;
1043 handle_nosign:
1044                         sign = 0;
1045                         if (jflag)
1046                                 num = va_arg(ap, uintmax_t);
1047                         else if (qflag)
1048                                 num = va_arg(ap, u_quad_t);
1049                         else if (tflag)
1050                                 num = va_arg(ap, ptrdiff_t);
1051                         else if (lflag)
1052                                 num = va_arg(ap, u_long);
1053                         else if (zflag)
1054                                 num = va_arg(ap, size_t);
1055                         else if (hflag)
1056                                 num = (u_short)va_arg(ap, int);
1057                         else if (cflag)
1058                                 num = (u_char)va_arg(ap, int);
1059                         else
1060                                 num = va_arg(ap, u_int);
1061                         goto number;
1062 handle_sign:
1063                         if (jflag)
1064                                 num = va_arg(ap, intmax_t);
1065                         else if (qflag)
1066                                 num = va_arg(ap, quad_t);
1067                         else if (tflag)
1068                                 num = va_arg(ap, ptrdiff_t);
1069                         else if (lflag)
1070                                 num = va_arg(ap, long);
1071                         else if (zflag)
1072                                 num = va_arg(ap, ssize_t);
1073                         else if (hflag)
1074                                 num = (short)va_arg(ap, int);
1075                         else if (cflag)
1076                                 num = (char)va_arg(ap, int);
1077                         else
1078                                 num = va_arg(ap, int);
1079 number:
1080                         if (sign && (intmax_t)num < 0) {
1081                                 neg = 1;
1082                                 num = -(intmax_t)num;
1083                         }
1084                         p = ksprintn(nbuf, num, base, &tmp, upper);
1085                         if (sharpflag && num != 0) {
1086                                 if (base == 8)
1087                                         tmp++;
1088                                 else if (base == 16)
1089                                         tmp += 2;
1090                         }
1091                         if (neg)
1092                                 tmp++;
1093
1094                         if (!ladjust && padc != '0' && width
1095                             && (width -= tmp) > 0)
1096                                 while (width--)
1097                                         PCHAR(padc);
1098                         if (neg)
1099                                 PCHAR('-');
1100                         if (sharpflag && num != 0) {
1101                                 if (base == 8) {
1102                                         PCHAR('0');
1103                                 } else if (base == 16) {
1104                                         PCHAR('0');
1105                                         PCHAR('x');
1106                                 }
1107                         }
1108                         if (!ladjust && width && (width -= tmp) > 0)
1109                                 while (width--)
1110                                         PCHAR(padc);
1111
1112                         while (*p)
1113                                 PCHAR(*p--);
1114
1115                         if (ladjust && width && (width -= tmp) > 0)
1116                                 while (width--)
1117                                         PCHAR(padc);
1118
1119                         break;
1120                 default:
1121                         while (percent < fmt)
1122                                 PCHAR(*percent++);
1123                         /*
1124                          * Since we ignore an formatting argument it is no
1125                          * longer safe to obey the remaining formatting
1126                          * arguments as the arguments will no longer match
1127                          * the format specs.
1128                          */
1129                         stop = 1;
1130                         break;
1131                 }
1132         }
1133 #undef PCHAR
1134 }
1135
1136
1137 static void
1138 putchar(int c, void *arg)
1139 {
1140         (void)arg;
1141         uart_putc(c);
1142 }
1143
1144 void
1145 hibprintf(const char *fmt, ...)
1146 {
1147         /* http://www.pagetable.com/?p=298 */
1148         va_list ap;
1149
1150         va_start(ap, fmt);
1151         hibkvprintf(fmt, putchar, NULL, 10, ap);
1152         va_end(ap);
1153 }
1154 #endif /* CONFIG_DEBUG */
1155