/*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
*
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License"). You may not use this file except in compliance with the
- * License. Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
*
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
*
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
#include <stdint.h>
#include <mach/mach_types.h>
#include <mach/vm_param.h>
#include <IOKit/IOHibernatePrivate.h>
+#include <IOKit/IOLib.h>
#include <pexpert/boot.h>
-#include <crypto/aes.h>
+#include <libkern/libkern.h>
-#include "WKdm.h"
+#include <libkern/WKdm.h>
#include "IOHibernateInternal.h"
+#if defined(__i386__) || defined(__x86_64__)
+#include <i386/pal_hibernate.h>
+#endif
+
/*
This code is linked into the kernel but part of the "__HIB" section, which means
its used by code running in the special context of restoring the kernel text and data
uint32_t gIOHibernateState;
+uint32_t gIOHibernateDebugFlags;
+
static IOHibernateImageHeader _hibernateHeader;
IOHibernateImageHeader * gIOHibernateCurrentHeader = &_hibernateHeader;
-static hibernate_graphics_t _hibernateGraphics;
-hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
+ppnum_t gIOHibernateHandoffPages[64];
+uint32_t gIOHibernateHandoffPageCount = sizeof(gIOHibernateHandoffPages)
+ / sizeof(gIOHibernateHandoffPages[0]);
-static hibernate_cryptwakevars_t _cryptWakeVars;
-hibernate_cryptwakevars_t * gIOHibernateCryptWakeVars = &_cryptWakeVars;
-
-#if __i386__
-extern void acpi_wake_prot_entry(void);
+#if CONFIG_DEBUG
+void hibprintf(const char *fmt, ...);
+#else
+#define hibprintf(x...)
#endif
+#if CONFIG_SLEEP
+#if defined(__i386__) || defined(__x86_64__)
+extern void acpi_wake_prot_entry(void);
+#endif
+#endif
+
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-#define BASE 65521L /* largest prime smaller than 65536 */
-#define NMAX 5000
-// NMAX (was 5521) the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1
+#if defined(__i386__) || defined(__x86_64__)
+
+#define DBGLOG 1
+
+#include <architecture/i386/pio.h>
+
+/* standard port addresses */
+enum {
+ COM1_PORT_ADDR = 0x3f8,
+ COM2_PORT_ADDR = 0x2f8
+};
+
+/* UART register offsets */
+enum {
+ UART_RBR = 0, /* receive buffer Register (R) */
+ UART_THR = 0, /* transmit holding register (W) */
+ UART_DLL = 0, /* DLAB = 1, divisor latch (LSB) */
+ UART_IER = 1, /* interrupt enable register */
+ UART_DLM = 1, /* DLAB = 1, divisor latch (MSB) */
+ UART_IIR = 2, /* interrupt ident register (R) */
+ UART_FCR = 2, /* fifo control register (W) */
+ UART_LCR = 3, /* line control register */
+ UART_MCR = 4, /* modem control register */
+ UART_LSR = 5, /* line status register */
+ UART_MSR = 6, /* modem status register */
+ UART_SCR = 7 /* scratch register */
+};
+
+enum {
+ UART_LCR_8BITS = 0x03,
+ UART_LCR_DLAB = 0x80
+};
+
+enum {
+ UART_MCR_DTR = 0x01,
+ UART_MCR_RTS = 0x02,
+ UART_MCR_OUT1 = 0x04,
+ UART_MCR_OUT2 = 0x08,
+ UART_MCR_LOOP = 0x10
+};
+
+enum {
+ UART_LSR_DR = 0x01,
+ UART_LSR_OE = 0x02,
+ UART_LSR_PE = 0x04,
+ UART_LSR_FE = 0x08,
+ UART_LSR_THRE = 0x20
+};
+
+static void uart_putc(char c)
+{
+ while (!(inb(COM1_PORT_ADDR + UART_LSR) & UART_LSR_THRE))
+ {}
+ outb(COM1_PORT_ADDR + UART_THR, c);
+}
-#define DO1(buf,i) {s1 += buf[i]; s2 += s1;}
-#define DO2(buf,i) DO1(buf,i); DO1(buf,i+1);
-#define DO4(buf,i) DO2(buf,i); DO2(buf,i+2);
-#define DO8(buf,i) DO4(buf,i); DO4(buf,i+4);
-#define DO16(buf) DO8(buf,0); DO8(buf,8);
+static int debug_probe( void )
+{
+ /* Verify that the Scratch Register is accessible */
+ outb(COM1_PORT_ADDR + UART_SCR, 0x5a);
+ if (inb(COM1_PORT_ADDR + UART_SCR) != 0x5a) return false;
+ outb(COM1_PORT_ADDR + UART_SCR, 0xa5);
+ if (inb(COM1_PORT_ADDR + UART_SCR) != 0xa5) return false;
+ uart_putc('\n');
+ return true;
+}
-uint32_t
-hibernate_sum(uint8_t *buf, int32_t len)
+static void uart_puthex(uint64_t num)
{
- unsigned long s1 = 1; // adler & 0xffff;
- unsigned long s2 = 0; // (adler >> 16) & 0xffff;
- int k;
-
- while (len > 0) {
- k = len < NMAX ? len : NMAX;
- len -= k;
- while (k >= 16) {
- DO16(buf);
- buf += 16;
- k -= 16;
- }
- if (k != 0) do {
- s1 += *buf++;
- s2 += s1;
- } while (--k);
- s1 %= BASE;
- s2 %= BASE;
+ int bit;
+ char c;
+ bool leading = true;
+
+ for (bit = 60; bit >= 0; bit -= 4)
+ {
+ c = 0xf & (num >> bit);
+ if (c)
+ leading = false;
+ else if (leading && bit)
+ continue;
+ if (c <= 9)
+ c += '0';
+ else
+ c+= 'a' - 10;
+ uart_putc(c);
}
- return (s2 << 16) | s1;
}
+static void debug_code(uint32_t code, uint64_t value)
+{
+ int bit;
+ char c;
+
+ if (!(kIOHibernateDebugRestoreLogs & gIOHibernateDebugFlags))
+ return;
+
+ for (bit = 24; bit >= 0; bit -= 8)
+ {
+ c = 0xFF & (code >> bit);
+ if (c)
+ uart_putc(c);
+ }
+ uart_putc('=');
+ uart_puthex(value);
+ uart_putc('\n');
+ uart_putc('\r');
+}
+
+#endif /* defined(__i386__) || defined(__x86_64__) */
+
+#if !defined(DBGLOG)
+#define debug_probe() (false)
+#define debug_code(c, v) {}
+#endif
+
+enum
+{
+ kIOHibernateRestoreCodeImageStart = 'imgS',
+ kIOHibernateRestoreCodeImageEnd = 'imgE',
+ kIOHibernateRestoreCodePageIndexStart = 'pgiS',
+ kIOHibernateRestoreCodePageIndexEnd = 'pgiE',
+ kIOHibernateRestoreCodeMapStart = 'mapS',
+ kIOHibernateRestoreCodeMapEnd = 'mapE',
+ kIOHibernateRestoreCodeWakeMapSize = 'wkms',
+ kIOHibernateRestoreCodeConflictPage = 'cfpg',
+ kIOHibernateRestoreCodeConflictSource = 'cfsr',
+ kIOHibernateRestoreCodeNoMemory = 'nomm',
+ kIOHibernateRestoreCodeTag = 'tag ',
+ kIOHibernateRestoreCodeSignature = 'sign',
+ kIOHibernateRestoreCodeMapVirt = 'mapV',
+ kIOHibernateRestoreCodeHandoffPages = 'hand',
+ kIOHibernateRestoreCodeHandoffCount = 'hndc',
+};
+
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-#if __ppc__
-static __inline__ unsigned int cntlzw(unsigned int num)
+
+static void fatal(void)
{
- unsigned int result;
- __asm__ volatile("cntlzw %0, %1" : "=r" (result) : "r" (num));
- return result;
+#if defined(__i386__) || defined(__x86_64__)
+ outb(0xcf9, 6);
+#else
+ while (true) {}
+#endif
}
-#elif __i386__
-static __inline__ unsigned int cntlzw(unsigned int num)
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+uint32_t
+hibernate_sum_page(uint8_t *buf, uint32_t ppnum)
{
- unsigned int result;
- __asm__ volatile( "bsrl %1, %0\n\t"
- "cmovel %2, %0"
- : "=r" (result)
- : "rm" (num), "r" (63));
- return 31 ^ result;
+ return (((uint32_t *)buf)[((PAGE_SIZE >> 2) - 1) & ppnum]);
}
-#else
-#error arch
-#endif
-void
-hibernate_page_bitset(hibernate_page_list_t * list, boolean_t set, uint32_t page)
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+static hibernate_bitmap_t *
+hibernate_page_bitmap(hibernate_page_list_t * list, uint32_t page)
{
uint32_t bank;
hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
for (bank = 0; bank < list->bank_count; bank++)
{
if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
- {
- page -= bitmap->first_page;
- if (set)
- bitmap->bitmap[page >> 5] |= (0x80000000 >> (page & 31));
- //setbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
- else
- bitmap->bitmap[page >> 5] &= ~(0x80000000 >> (page & 31));
- //clrbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
break;
- }
bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
}
+ if (bank == list->bank_count)
+ bitmap = NULL;
+
+ return (bitmap);
}
-boolean_t
-hibernate_page_bittst(hibernate_page_list_t * list, uint32_t page)
+hibernate_bitmap_t *
+hibernate_page_bitmap_pin(hibernate_page_list_t * list, uint32_t * pPage)
{
- boolean_t result = TRUE;
- uint32_t bank;
+ uint32_t bank, page = *pPage;
hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
for (bank = 0; bank < list->bank_count; bank++)
{
- if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
+ if (page <= bitmap->first_page)
{
- page -= bitmap->first_page;
- result = (0 != (bitmap->bitmap[page >> 5] & (0x80000000 >> (page & 31))));
+ *pPage = bitmap->first_page;
break;
}
+ if (page <= bitmap->last_page)
+ break;
bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
}
+ if (bank == list->bank_count)
+ bitmap = NULL;
+
+ return (bitmap);
+}
+
+void
+hibernate_page_bitset(hibernate_page_list_t * list, boolean_t set, uint32_t page)
+{
+ hibernate_bitmap_t * bitmap;
+
+ bitmap = hibernate_page_bitmap(list, page);
+ if (bitmap)
+ {
+ page -= bitmap->first_page;
+ if (set)
+ bitmap->bitmap[page >> 5] |= (0x80000000 >> (page & 31));
+ //setbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
+ else
+ bitmap->bitmap[page >> 5] &= ~(0x80000000 >> (page & 31));
+ //clrbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
+ }
+}
+
+boolean_t
+hibernate_page_bittst(hibernate_page_list_t * list, uint32_t page)
+{
+ boolean_t result = TRUE;
+ hibernate_bitmap_t * bitmap;
+
+ bitmap = hibernate_page_bitmap(list, page);
+ if (bitmap)
+ {
+ page -= bitmap->first_page;
+ result = (0 != (bitmap->bitmap[page >> 5] & (0x80000000 >> (page & 31))));
+ }
return (result);
}
-// count bits clear or set (set == TRUE) starting at index page.
+// count bits clear or set (set == TRUE) starting at page.
uint32_t
-hibernate_page_list_count(hibernate_page_list_t * list, uint32_t set, uint32_t page)
+hibernate_page_bitmap_count(hibernate_bitmap_t * bitmap, uint32_t set, uint32_t page)
{
- uint32_t bank, count;
- hibernate_bitmap_t * bitmap;
+ uint32_t index, bit, bits;
+ uint32_t count;
- bitmap = &list->bank_bitmap[0];
- count = 0;
+ count = 0;
- for (bank = 0; bank < list->bank_count; bank++)
+ index = (page - bitmap->first_page) >> 5;
+ bit = (page - bitmap->first_page) & 31;
+
+ bits = bitmap->bitmap[index];
+ if (set)
+ bits = ~bits;
+ bits = (bits << bit);
+ if (bits)
+ count += __builtin_clz(bits);
+ else
{
- // bits between banks are "set"
- if (set && (page < bitmap->first_page))
+ count += 32 - bit;
+ while (++index < bitmap->bitmapwords)
{
- count += bitmap->first_page - page;
- page = bitmap->first_page;
- }
- if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
- {
- uint32_t index, bit, bits;
-
- index = (page - bitmap->first_page) >> 5;
- bit = (page - bitmap->first_page) & 31;
-
- while (TRUE)
+ bits = bitmap->bitmap[index];
+ if (set)
+ bits = ~bits;
+ if (bits)
{
- bits = bitmap->bitmap[index];
- if (set)
- bits = ~bits;
- bits = (bits << bit);
- count += cntlzw(bits);
- if (bits)
- break;
- count -= bit;
-
- while (++index < bitmap->bitmapwords)
- {
- bits = bitmap->bitmap[index];
- if (set)
- bits = ~bits;
- count += cntlzw(bits);
- if (bits)
- break;
- }
- if (bits)
- break;
- if (!set)
- break;
- // bits between banks are "set"
- bank++;
- if (bank >= list->bank_count)
- break;
- count -= (bitmap->last_page + 1);
- bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
- count += bitmap->first_page;
- index = 0;
- bit = 0;
+ count += __builtin_clz(bits);
+ break;
}
- break;
+ count += 32;
}
- bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
}
+ if ((page + count) > (bitmap->last_page + 1)) count = (bitmap->last_page + 1) - page;
+
return (count);
}
-
-static uint32_t
-hibernate_page_list_grab(hibernate_page_list_t * map, uint32_t * _nextFree)
+static ppnum_t
+hibernate_page_list_grab(hibernate_page_list_t * list, uint32_t * pNextFree)
{
- uint32_t nextFree = *_nextFree;
+ uint32_t nextFree = *pNextFree;
+ uint32_t nextFreeInBank;
+ hibernate_bitmap_t * bitmap;
- if (!nextFree)
- nextFree = hibernate_page_list_count(map, 0, 0);
+ nextFreeInBank = nextFree + 1;
+ while ((bitmap = hibernate_page_bitmap_pin(list, &nextFreeInBank)))
+ {
+ nextFreeInBank += hibernate_page_bitmap_count(bitmap, FALSE, nextFreeInBank);
+ if (nextFreeInBank <= bitmap->last_page)
+ {
+ *pNextFree = nextFreeInBank;
+ break;
+ }
+ }
- *_nextFree = nextFree + 1 + hibernate_page_list_count(map, 0, nextFree + 1);
+ if (!bitmap)
+ {
+ debug_code(kIOHibernateRestoreCodeNoMemory, nextFree);
+ fatal();
+ nextFree = 0;
+ }
return (nextFree);
}
store_one_page(uint32_t procFlags, uint32_t * src, uint32_t compressedSize,
uint32_t * buffer, uint32_t ppnum)
{
- uint64_t dst;
- uint32_t sum;
-
- dst = ptoa_64(ppnum);
-#if __ppc__
- if (ppnum < 0x00100000)
- buffer = (uint32_t *) (uint32_t) dst;
-#elif __i386__
- if (ppnum < atop_32(0xC0000000)) {
- buffer = (uint32_t *) (uint32_t) dst;
- }
-#endif
-
- if (compressedSize != PAGE_SIZE)
- {
- WKdm_decompress((WK_word*) src, (WK_word*) buffer, PAGE_SIZE >> 2);
- src = buffer;
- }
+ uint64_t dst = ptoa_64(ppnum);
- sum = hibernate_sum((uint8_t *) src, PAGE_SIZE);
-
- if (((uint64_t) (uint32_t) src) == dst)
- src = 0;
-
- hibernate_restore_phys_page((uint64_t) (uint32_t) src, dst, PAGE_SIZE, procFlags);
+ if (compressedSize != PAGE_SIZE)
+ {
+ dst = pal_hib_map(DEST_COPY_AREA, dst);
+ WKdm_decompress((WK_word*) src, (WK_word*)(uintptr_t)dst, PAGE_SIZE >> 2);
+ }
+ else
+ {
+ dst = hibernate_restore_phys_page((uint64_t) (uintptr_t) src, dst, PAGE_SIZE, procFlags);
+ }
- return (sum);
+ return hibernate_sum_page((uint8_t *)(uintptr_t)dst, ppnum);
}
+// used only for small struct copies
static void
bcopy_internal(const void *src, void *dst, uint32_t len)
{
}
}
+#define C_ASSERT(e) typedef char __C_ASSERT__[(e) ? 1 : -1]
+
long
-hibernate_kernel_entrypoint(IOHibernateImageHeader * header,
- void * p2, void * p3, __unused void * p4)
+hibernate_kernel_entrypoint(uint32_t p1,
+ uint32_t p2, uint32_t p3, uint32_t p4)
{
- typedef void (*ResetProc)(void);
+ uint64_t headerPhys;
+ uint64_t mapPhys;
+ uint64_t srcPhys;
+ uint64_t imageReadPhys;
+ uint64_t pageIndexPhys;
uint32_t idx;
- uint32_t * src;
- uint32_t * buffer;
uint32_t * pageIndexSource;
hibernate_page_list_t * map;
+ uint32_t stage;
uint32_t count;
uint32_t ppnum;
uint32_t page;
uint32_t conflictCount;
uint32_t compressedSize;
uint32_t uncompressedPages;
- uint32_t copyPageListHead;
+ uint32_t copyPageListHeadPage;
+ uint32_t pageListPage;
uint32_t * copyPageList;
+ uint32_t * src;
uint32_t copyPageIndex;
uint32_t sum;
+ uint32_t pageSum;
uint32_t nextFree;
uint32_t lastImagePage;
uint32_t lastMapPage;
uint32_t lastPageIndexPage;
+ uint32_t handoffPages;
+ uint32_t handoffPageCount;
+ C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
- bcopy_internal(header,
- gIOHibernateCurrentHeader,
- sizeof(IOHibernateImageHeader));
+ headerPhys = ptoa_64(p1);
- if (p2)
- bcopy_internal(p2,
- gIOHibernateGraphicsInfo,
- sizeof(hibernate_graphics_t));
- else
- gIOHibernateGraphicsInfo->physicalAddress = gIOHibernateGraphicsInfo->depth = 0;
+ if ((kIOHibernateDebugRestoreLogs & gIOHibernateDebugFlags) && !debug_probe())
+ gIOHibernateDebugFlags &= ~kIOHibernateDebugRestoreLogs;
- if (p3)
- bcopy_internal(p3,
- gIOHibernateCryptWakeVars,
- sizeof(hibernate_cryptvars_t));
+ debug_code(kIOHibernateRestoreCodeImageStart, headerPhys);
- src = (uint32_t *)
- (((uint32_t) &header->fileExtentMap[0])
- + header->fileExtentMapSize
- + ptoa_32(header->restore1PageCount));
+ bcopy_internal((void *) pal_hib_map(IMAGE_AREA, headerPhys),
+ gIOHibernateCurrentHeader,
+ sizeof(IOHibernateImageHeader));
- if (header->previewSize)
- {
- pageIndexSource = src;
- map = (hibernate_page_list_t *)(((uint32_t) pageIndexSource) + header->previewSize);
- src = (uint32_t *) (((uint32_t) pageIndexSource) + header->previewPageListSize);
- }
- else
- {
- pageIndexSource = 0;
- map = (hibernate_page_list_t *) src;
- src = (uint32_t *) (((uint32_t) map) + header->bitmapSize);
- }
+ debug_code(kIOHibernateRestoreCodeSignature, gIOHibernateCurrentHeader->signature);
+
+ mapPhys = headerPhys
+ + (offsetof(IOHibernateImageHeader, fileExtentMap)
+ + gIOHibernateCurrentHeader->fileExtentMapSize
+ + ptoa_32(gIOHibernateCurrentHeader->restore1PageCount)
+ + gIOHibernateCurrentHeader->previewSize);
+
+ map = (hibernate_page_list_t *) pal_hib_map(BITMAP_AREA, mapPhys);
- lastPageIndexPage = atop_32(src);
+ lastImagePage = atop_64(headerPhys + gIOHibernateCurrentHeader->image1Size);
+ lastMapPage = atop_64(mapPhys + gIOHibernateCurrentHeader->bitmapSize);
- lastImagePage = atop_32(((uint32_t) header) + header->image1Size);
+ handoffPages = gIOHibernateCurrentHeader->handoffPages;
+ handoffPageCount = gIOHibernateCurrentHeader->handoffPageCount;
- lastMapPage = atop_32(((uint32_t) map) + header->bitmapSize);
+ debug_code(kIOHibernateRestoreCodeImageEnd, ptoa_64(lastImagePage));
+ debug_code(kIOHibernateRestoreCodeMapStart, mapPhys);
+ debug_code(kIOHibernateRestoreCodeMapEnd, ptoa_64(lastMapPage));
+
+ debug_code(kIOHibernateRestoreCodeMapVirt, (uintptr_t) map);
+ debug_code(kIOHibernateRestoreCodeHandoffPages, ptoa_64(handoffPages));
+ debug_code(kIOHibernateRestoreCodeHandoffCount, handoffPageCount);
// knock all the image pages to be used out of free map
- for (ppnum = atop_32(header); ppnum <= lastImagePage; ppnum++)
+ for (ppnum = atop_64(headerPhys); ppnum <= lastImagePage; ppnum++)
+ {
+ hibernate_page_bitset(map, FALSE, ppnum);
+ }
+ // knock all the handoff pages to be used out of free map
+ for (ppnum = handoffPages; ppnum < (handoffPages + handoffPageCount); ppnum++)
{
hibernate_page_bitset(map, FALSE, ppnum);
}
nextFree = 0;
- buffer = (uint32_t *) ptoa_32(hibernate_page_list_grab(map, &nextFree));
+ hibernate_page_list_grab(map, &nextFree);
sum = gIOHibernateCurrentHeader->actualRestore1Sum;
- gIOHibernateCurrentHeader->diag[0] = (uint32_t) header;
+ gIOHibernateCurrentHeader->diag[0] = atop_64(headerPhys);
gIOHibernateCurrentHeader->diag[1] = sum;
- uncompressedPages = 0;
- conflictCount = 0;
- copyPageListHead = 0;
- copyPageList = 0;
- copyPageIndex = PAGE_SIZE >> 2;
+ uncompressedPages = 0;
+ conflictCount = 0;
+ copyPageListHeadPage = 0;
+ copyPageList = 0;
+ copyPageIndex = PAGE_SIZE >> 2;
- compressedSize = PAGE_SIZE;
+ compressedSize = PAGE_SIZE;
+ stage = 2;
+ count = 0;
+ srcPhys = 0;
+
+ if (gIOHibernateCurrentHeader->previewSize)
+ {
+ pageIndexPhys = headerPhys
+ + (offsetof(IOHibernateImageHeader, fileExtentMap)
+ + gIOHibernateCurrentHeader->fileExtentMapSize
+ + ptoa_32(gIOHibernateCurrentHeader->restore1PageCount));
+ imageReadPhys = (pageIndexPhys + gIOHibernateCurrentHeader->previewPageListSize);
+ lastPageIndexPage = atop_64(imageReadPhys);
+ pageIndexSource = (uint32_t *) pal_hib_map(IMAGE2_AREA, pageIndexPhys);
+ }
+ else
+ {
+ pageIndexPhys = 0;
+ lastPageIndexPage = 0;
+ imageReadPhys = (mapPhys + gIOHibernateCurrentHeader->bitmapSize);
+ }
+
+ debug_code(kIOHibernateRestoreCodePageIndexStart, pageIndexPhys);
+ debug_code(kIOHibernateRestoreCodePageIndexEnd, ptoa_64(lastPageIndexPage));
while (1)
{
- if (pageIndexSource)
- {
- ppnum = pageIndexSource[0];
- count = pageIndexSource[1];
- pageIndexSource += 2;
- if (!count)
- {
- pageIndexSource = 0;
- src = (uint32_t *) (((uint32_t) map) + gIOHibernateCurrentHeader->bitmapSize);
- ppnum = src[0];
- count = src[1];
- src += 2;
- }
- }
- else
- {
- ppnum = src[0];
- count = src[1];
- if (!count)
- break;
- src += 2;
+ switch (stage)
+ {
+ case 2:
+ // copy handoff data
+ count = srcPhys ? 0 : handoffPageCount;
+ if (!count)
+ break;
+ if (count > gIOHibernateHandoffPageCount) count = gIOHibernateHandoffPageCount;
+ srcPhys = ptoa_64(handoffPages);
+ break;
+
+ case 1:
+ // copy pageIndexSource pages == preview image data
+ if (!srcPhys)
+ {
+ if (!pageIndexPhys) break;
+ srcPhys = imageReadPhys;
+ }
+ ppnum = pageIndexSource[0];
+ count = pageIndexSource[1];
+ pageIndexSource += 2;
+ pageIndexPhys += 2 * sizeof(pageIndexSource[0]);
+ imageReadPhys = srcPhys;
+ break;
+
+ case 0:
+ // copy pages
+ if (!srcPhys) srcPhys = (mapPhys + gIOHibernateCurrentHeader->bitmapSize);
+ src = (uint32_t *) pal_hib_map(IMAGE_AREA, srcPhys);
+ ppnum = src[0];
+ count = src[1];
+ srcPhys += 2 * sizeof(*src);
+ imageReadPhys = srcPhys;
+ break;
+ }
+
+
+ if (!count)
+ {
+ if (!stage)
+ break;
+ stage--;
+ srcPhys = 0;
+ continue;
}
for (page = 0; page < count; page++, ppnum++)
{
- uint32_t tag;
+ uint32_t tag;
int conflicts;
- if (!pageIndexSource)
- {
- tag = *src++;
- compressedSize = kIOHibernateTagLength & tag;
- }
+ src = (uint32_t *) pal_hib_map(IMAGE_AREA, srcPhys);
+
+ if (2 == stage) ppnum = gIOHibernateHandoffPages[page];
+ else if (!stage)
+ {
+ tag = *src++;
+// debug_code(kIOHibernateRestoreCodeTag, (uintptr_t) tag);
+ srcPhys += sizeof(*src);
+ compressedSize = kIOHibernateTagLength & tag;
+ }
+
+ conflicts = (ppnum >= atop_64(mapPhys)) && (ppnum <= lastMapPage);
- conflicts = (((ppnum >= atop_32(map)) && (ppnum <= lastMapPage))
- || ((ppnum >= atop_32(src)) && (ppnum <= lastImagePage)));
+ conflicts |= ((ppnum >= atop_64(imageReadPhys)) && (ppnum <= lastImagePage));
- if (pageIndexSource)
- conflicts |= ((ppnum >= atop_32(pageIndexSource)) && (ppnum <= lastPageIndexPage));
+ if (stage >= 2)
+ conflicts |= ((ppnum >= atop_64(srcPhys)) && (ppnum <= (handoffPages + handoffPageCount - 1)));
+
+ if (stage >= 1)
+ conflicts |= ((ppnum >= atop_64(pageIndexPhys)) && (ppnum <= lastPageIndexPage));
if (!conflicts)
{
- if (compressedSize)
- sum += store_one_page(gIOHibernateCurrentHeader->processorFlags,
- src, compressedSize, buffer, ppnum);
+// if (compressedSize)
+ pageSum = store_one_page(gIOHibernateCurrentHeader->processorFlags,
+ src, compressedSize, 0, ppnum);
+ if (stage != 2)
+ sum += pageSum;
uncompressedPages++;
}
else
uint32_t bufferPage;
uint32_t * dst;
+// debug_code(kIOHibernateRestoreCodeConflictPage, ppnum);
+// debug_code(kIOHibernateRestoreCodeConflictSource, (uintptr_t) src);
+
conflictCount++;
// alloc new buffer page
if (copyPageIndex > ((PAGE_SIZE >> 2) - 3))
{
// alloc new copy list page
- uint32_t pageListPage = hibernate_page_list_grab(map, &nextFree);
+ pageListPage = hibernate_page_list_grab(map, &nextFree);
// link to current
- if (copyPageList)
- copyPageList[1] = pageListPage;
- else
- copyPageListHead = pageListPage;
- copyPageList = (uint32_t *) ptoa_32(pageListPage);
+ if (copyPageList) {
+ copyPageList[1] = pageListPage;
+ } else {
+ copyPageListHeadPage = pageListPage;
+ }
+ copyPageList = (uint32_t *)pal_hib_map(SRC_COPY_AREA,
+ ptoa_64(pageListPage));
copyPageList[1] = 0;
copyPageIndex = 2;
}
copyPageList[copyPageIndex++] = ppnum;
copyPageList[copyPageIndex++] = bufferPage;
- copyPageList[copyPageIndex++] = compressedSize;
+ copyPageList[copyPageIndex++] = (compressedSize | (stage << 24));
copyPageList[0] = copyPageIndex;
- dst = (uint32_t *) ptoa_32(bufferPage);
+ dst = (uint32_t *)pal_hib_map(DEST_COPY_AREA, ptoa_64(bufferPage));
for (idx = 0; idx < ((compressedSize + 3) >> 2); idx++)
- dst[idx] = src[idx];
+ dst[idx] = src[idx];
}
- src += ((compressedSize + 3) >> 2);
+ srcPhys += ((compressedSize + 3) & ~3);
+ src += ((compressedSize + 3) >> 2);
}
}
+ /* src points to the last page restored, so we need to skip over that */
+ hibernateRestorePALState(src);
+
// -- copy back conflicts
- copyPageList = (uint32_t *) ptoa_32(copyPageListHead);
- while (copyPageList)
+ pageListPage = copyPageListHeadPage;
+ while (pageListPage)
{
+ copyPageList = (uint32_t *)pal_hib_map(COPY_PAGE_AREA, ptoa_64(pageListPage));
for (copyPageIndex = 2; copyPageIndex < copyPageList[0]; copyPageIndex += 3)
{
- ppnum = copyPageList[copyPageIndex + 0];
- src = (uint32_t *) ptoa_32(copyPageList[copyPageIndex + 1]);
- compressedSize = copyPageList[copyPageIndex + 2];
-
- sum += store_one_page(gIOHibernateCurrentHeader->processorFlags,
- src, compressedSize, buffer, ppnum);
+ ppnum = copyPageList[copyPageIndex + 0];
+ srcPhys = ptoa_64(copyPageList[copyPageIndex + 1]);
+ src = (uint32_t *) pal_hib_map(SRC_COPY_AREA, srcPhys);
+ compressedSize = copyPageList[copyPageIndex + 2];
+ stage = compressedSize >> 24;
+ compressedSize &= 0x1FFF;
+ pageSum = store_one_page(gIOHibernateCurrentHeader->processorFlags,
+ src, compressedSize, 0, ppnum);
+ if (stage != 2)
+ sum += pageSum;
uncompressedPages++;
}
- copyPageList = (uint32_t *) ptoa_32(copyPageList[1]);
+ pageListPage = copyPageList[1];
}
+ pal_hib_patchup();
+
// -- image has been destroyed...
gIOHibernateCurrentHeader->actualImage1Sum = sum;
gIOHibernateState = kIOHibernateStateWakingFromHibernate;
-#if __ppc__
- ResetProc proc;
- proc = (ResetProc) 0x100;
- __asm__ volatile("ori 0, 0, 0" : : );
- proc();
-#elif __i386__
+#if CONFIG_SLEEP
+#if defined(__i386__) || defined(__x86_64__)
+ typedef void (*ResetProc)(void);
ResetProc proc;
- proc = (ResetProc) acpi_wake_prot_entry;
-
+ proc = HIB_ENTRYPOINT;
+ // flush caches
+ __asm__("wbinvd");
proc();
+#else
+// implement me
+#endif
#endif
-
+
return -1;
}
+#if CONFIG_DEBUG
+/* standalone printf implementation */
+/*-
+ * Copyright (c) 1986, 1988, 1991, 1993
+ * The Regents of the University of California. All rights reserved.
+ * (c) UNIX System Laboratories, Inc.
+ * All or some portions of this file are derived from material licensed
+ * to the University of California by American Telephone and Telegraph
+ * Co. or Unix System Laboratories, Inc. and are reproduced herein with
+ * the permission of UNIX System Laboratories, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * @(#)subr_prf.c 8.3 (Berkeley) 1/21/94
+ */
+
+typedef long ptrdiff_t;
+char const hibhex2ascii_data[] = "0123456789abcdefghijklmnopqrstuvwxyz";
+#define hibhex2ascii(hex) (hibhex2ascii_data[hex])
+#define toupper(c) ((c) - 0x20 * (((c) >= 'a') && ((c) <= 'z')))
+static size_t
+hibstrlen(const char *s)
+{
+ size_t l = 0;
+ while (*s++)
+ l++;
+ return l;
+}
+
+/* Max number conversion buffer length: a u_quad_t in base 2, plus NUL byte. */
+#define MAXNBUF (sizeof(intmax_t) * NBBY + 1)
+
+/*
+ * Put a NUL-terminated ASCII number (base <= 36) in a buffer in reverse
+ * order; return an optional length and a pointer to the last character
+ * written in the buffer (i.e., the first character of the string).
+ * The buffer pointed to by `nbuf' must have length >= MAXNBUF.
+ */
+static char *
+ksprintn(char *nbuf, uintmax_t num, int base, int *lenp, int upper)
+{
+ char *p, c;
+
+ /* Truncate so we don't call umoddi3, which isn't in __HIB */
+#if !defined(__LP64__)
+ uint32_t num2 = (uint32_t) num;
+#else
+ uintmax_t num2 = num;
+#endif
+
+ p = nbuf;
+ *p = '\0';
+ do {
+ c = hibhex2ascii(num2 % base);
+ *++p = upper ? toupper(c) : c;
+ } while (num2 /= base);
+ if (lenp)
+ *lenp = (int)(p - nbuf);
+ return (p);
+}
+
+/*
+ * Scaled down version of printf(3).
+ *
+ * Two additional formats:
+ *
+ * The format %b is supported to decode error registers.
+ * Its usage is:
+ *
+ * printf("reg=%b\n", regval, "*");
+ *
+ * where is the output base expressed as a control character, e.g.
+ * \10 gives octal; \20 gives hex. Each arg is a sequence of characters,
+ * the first of which gives the bit number to be inspected (origin 1), and
+ * the next characters (up to a control character, i.e. a character <= 32),
+ * give the name of the register. Thus:
+ *
+ * kvprintf("reg=%b\n", 3, "\10\2BITTWO\1BITONE\n");
+ *
+ * would produce output:
+ *
+ * reg=3
+ *
+ * XXX: %D -- Hexdump, takes pointer and separator string:
+ * ("%6D", ptr, ":") -> XX:XX:XX:XX:XX:XX
+ * ("%*D", len, ptr, " " -> XX XX XX XX ...
+ */
+static int
+hibkvprintf(char const *fmt, void (*func)(int, void*), void *arg, int radix, va_list ap)
+{
+#define PCHAR(c) {int cc=(c); if (func) (*func)(cc,arg); else *d++ = cc; retval++; }
+ char nbuf[MAXNBUF];
+ char *d;
+ const char *p, *percent, *q;
+ u_char *up;
+ int ch, n;
+ uintmax_t num;
+ int base, lflag, qflag, tmp, width, ladjust, sharpflag, neg, sign, dot;
+ int cflag, hflag, jflag, tflag, zflag;
+ int dwidth, upper;
+ char padc;
+ int stop = 0, retval = 0;
+
+ num = 0;
+ if (!func)
+ d = (char *) arg;
+ else
+ d = NULL;
+
+ if (fmt == NULL)
+ fmt = "(fmt null)\n";
+
+ if (radix < 2 || radix > 36)
+ radix = 10;
+
+ for (;;) {
+ padc = ' ';
+ width = 0;
+ while ((ch = (u_char)*fmt++) != '%' || stop) {
+ if (ch == '\0')
+ return (retval);
+ PCHAR(ch);
+ }
+ percent = fmt - 1;
+ qflag = 0; lflag = 0; ladjust = 0; sharpflag = 0; neg = 0;
+ sign = 0; dot = 0; dwidth = 0; upper = 0;
+ cflag = 0; hflag = 0; jflag = 0; tflag = 0; zflag = 0;
+reswitch: switch (ch = (u_char)*fmt++) {
+ case '.':
+ dot = 1;
+ goto reswitch;
+ case '#':
+ sharpflag = 1;
+ goto reswitch;
+ case '+':
+ sign = 1;
+ goto reswitch;
+ case '-':
+ ladjust = 1;
+ goto reswitch;
+ case '%':
+ PCHAR(ch);
+ break;
+ case '*':
+ if (!dot) {
+ width = va_arg(ap, int);
+ if (width < 0) {
+ ladjust = !ladjust;
+ width = -width;
+ }
+ } else {
+ dwidth = va_arg(ap, int);
+ }
+ goto reswitch;
+ case '0':
+ if (!dot) {
+ padc = '0';
+ goto reswitch;
+ }
+ case '1': case '2': case '3': case '4':
+ case '5': case '6': case '7': case '8': case '9':
+ for (n = 0;; ++fmt) {
+ n = n * 10 + ch - '0';
+ ch = *fmt;
+ if (ch < '0' || ch > '9')
+ break;
+ }
+ if (dot)
+ dwidth = n;
+ else
+ width = n;
+ goto reswitch;
+ case 'b':
+ num = (u_int)va_arg(ap, int);
+ p = va_arg(ap, char *);
+ for (q = ksprintn(nbuf, num, *p++, NULL, 0); *q;)
+ PCHAR(*q--);
+
+ if (num == 0)
+ break;
+
+ for (tmp = 0; *p;) {
+ n = *p++;
+ if (num & (1 << (n - 1))) {
+ PCHAR(tmp ? ',' : '<');
+ for (; (n = *p) > ' '; ++p)
+ PCHAR(n);
+ tmp = 1;
+ } else
+ for (; *p > ' '; ++p)
+ continue;
+ }
+ if (tmp)
+ PCHAR('>');
+ break;
+ case 'c':
+ PCHAR(va_arg(ap, int));
+ break;
+ case 'D':
+ up = va_arg(ap, u_char *);
+ p = va_arg(ap, char *);
+ if (!width)
+ width = 16;
+ while(width--) {
+ PCHAR(hibhex2ascii(*up >> 4));
+ PCHAR(hibhex2ascii(*up & 0x0f));
+ up++;
+ if (width)
+ for (q=p;*q;q++)
+ PCHAR(*q);
+ }
+ break;
+ case 'd':
+ case 'i':
+ base = 10;
+ sign = 1;
+ goto handle_sign;
+ case 'h':
+ if (hflag) {
+ hflag = 0;
+ cflag = 1;
+ } else
+ hflag = 1;
+ goto reswitch;
+ case 'j':
+ jflag = 1;
+ goto reswitch;
+ case 'l':
+ if (lflag) {
+ lflag = 0;
+ qflag = 1;
+ } else
+ lflag = 1;
+ goto reswitch;
+ case 'n':
+ if (jflag)
+ *(va_arg(ap, intmax_t *)) = retval;
+ else if (qflag)
+ *(va_arg(ap, quad_t *)) = retval;
+ else if (lflag)
+ *(va_arg(ap, long *)) = retval;
+ else if (zflag)
+ *(va_arg(ap, size_t *)) = retval;
+ else if (hflag)
+ *(va_arg(ap, short *)) = retval;
+ else if (cflag)
+ *(va_arg(ap, char *)) = retval;
+ else
+ *(va_arg(ap, int *)) = retval;
+ break;
+ case 'o':
+ base = 8;
+ goto handle_nosign;
+ case 'p':
+ base = 16;
+ sharpflag = (width == 0);
+ sign = 0;
+ num = (uintptr_t)va_arg(ap, void *);
+ goto number;
+ case 'q':
+ qflag = 1;
+ goto reswitch;
+ case 'r':
+ base = radix;
+ if (sign)
+ goto handle_sign;
+ goto handle_nosign;
+ case 's':
+ p = va_arg(ap, char *);
+ if (p == NULL)
+ p = "(null)";
+ if (!dot)
+ n = (typeof(n))hibstrlen (p);
+ else
+ for (n = 0; n < dwidth && p[n]; n++)
+ continue;
+
+ width -= n;
+
+ if (!ladjust && width > 0)
+ while (width--)
+ PCHAR(padc);
+ while (n--)
+ PCHAR(*p++);
+ if (ladjust && width > 0)
+ while (width--)
+ PCHAR(padc);
+ break;
+ case 't':
+ tflag = 1;
+ goto reswitch;
+ case 'u':
+ base = 10;
+ goto handle_nosign;
+ case 'X':
+ upper = 1;
+ case 'x':
+ base = 16;
+ goto handle_nosign;
+ case 'y':
+ base = 16;
+ sign = 1;
+ goto handle_sign;
+ case 'z':
+ zflag = 1;
+ goto reswitch;
+handle_nosign:
+ sign = 0;
+ if (jflag)
+ num = va_arg(ap, uintmax_t);
+ else if (qflag)
+ num = va_arg(ap, u_quad_t);
+ else if (tflag)
+ num = va_arg(ap, ptrdiff_t);
+ else if (lflag)
+ num = va_arg(ap, u_long);
+ else if (zflag)
+ num = va_arg(ap, size_t);
+ else if (hflag)
+ num = (u_short)va_arg(ap, int);
+ else if (cflag)
+ num = (u_char)va_arg(ap, int);
+ else
+ num = va_arg(ap, u_int);
+ goto number;
+handle_sign:
+ if (jflag)
+ num = va_arg(ap, intmax_t);
+ else if (qflag)
+ num = va_arg(ap, quad_t);
+ else if (tflag)
+ num = va_arg(ap, ptrdiff_t);
+ else if (lflag)
+ num = va_arg(ap, long);
+ else if (zflag)
+ num = va_arg(ap, ssize_t);
+ else if (hflag)
+ num = (short)va_arg(ap, int);
+ else if (cflag)
+ num = (char)va_arg(ap, int);
+ else
+ num = va_arg(ap, int);
+number:
+ if (sign && (intmax_t)num < 0) {
+ neg = 1;
+ num = -(intmax_t)num;
+ }
+ p = ksprintn(nbuf, num, base, &tmp, upper);
+ if (sharpflag && num != 0) {
+ if (base == 8)
+ tmp++;
+ else if (base == 16)
+ tmp += 2;
+ }
+ if (neg)
+ tmp++;
+
+ if (!ladjust && padc != '0' && width
+ && (width -= tmp) > 0)
+ while (width--)
+ PCHAR(padc);
+ if (neg)
+ PCHAR('-');
+ if (sharpflag && num != 0) {
+ if (base == 8) {
+ PCHAR('0');
+ } else if (base == 16) {
+ PCHAR('0');
+ PCHAR('x');
+ }
+ }
+ if (!ladjust && width && (width -= tmp) > 0)
+ while (width--)
+ PCHAR(padc);
+
+ while (*p)
+ PCHAR(*p--);
+
+ if (ladjust && width && (width -= tmp) > 0)
+ while (width--)
+ PCHAR(padc);
+
+ break;
+ default:
+ while (percent < fmt)
+ PCHAR(*percent++);
+ /*
+ * Since we ignore an formatting argument it is no
+ * longer safe to obey the remaining formatting
+ * arguments as the arguments will no longer match
+ * the format specs.
+ */
+ stop = 1;
+ break;
+ }
+ }
+#undef PCHAR
+}
+
+
+static void
+putchar(int c, void *arg)
+{
+ (void)arg;
+ uart_putc(c);
+}
+
+void
+hibprintf(const char *fmt, ...)
+{
+ /* http://www.pagetable.com/?p=298 */
+ va_list ap;
+
+ va_start(ap, fmt);
+ hibkvprintf(fmt, putchar, NULL, 10, ap);
+ va_end(ap);
+}
+#endif /* CONFIG_DEBUG */
+
projects / apple / xnu.git / blobdiff