]>
Commit | Line | Data |
---|---|---|
3a60a9f5 | 1 | /* |
2d21ac55 | 2 | * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved. |
3a60a9f5 | 3 | * |
2d21ac55 | 4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
3a60a9f5 | 5 | * |
2d21ac55 A |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License | |
8 | * Version 2.0 (the 'License'). You may not use this file except in | |
9 | * compliance with the License. The rights granted to you under the License | |
10 | * may not be used to create, or enable the creation or redistribution of, | |
11 | * unlawful or unlicensed copies of an Apple operating system, or to | |
12 | * circumvent, violate, or enable the circumvention or violation of, any | |
13 | * terms of an Apple operating system software license agreement. | |
8f6c56a5 | 14 | * |
2d21ac55 A |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. | |
17 | * | |
18 | * The Original Code and all software distributed under the License are | |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER | |
8f6c56a5 A |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, | |
2d21ac55 A |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and | |
24 | * limitations under the License. | |
8f6c56a5 | 25 | * |
2d21ac55 | 26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
3a60a9f5 A |
27 | */ |
28 | ||
29 | #include <stdint.h> | |
30 | #include <mach/mach_types.h> | |
31 | #include <mach/vm_param.h> | |
32 | #include <IOKit/IOHibernatePrivate.h> | |
0c530ab8 | 33 | #include <IOKit/IOLib.h> |
3a60a9f5 | 34 | #include <pexpert/boot.h> |
2d21ac55 | 35 | #include <libkern/libkern.h> |
3a60a9f5 | 36 | |
39236c6e | 37 | #include <vm/WKdm_new.h> |
3a60a9f5 A |
38 | #include "IOHibernateInternal.h" |
39 | ||
bd504ef0 | 40 | #include <machine/pal_hibernate.h> |
6d2010ae | 41 | |
3a60a9f5 A |
42 | /* |
43 | This code is linked into the kernel but part of the "__HIB" section, which means | |
44 | its used by code running in the special context of restoring the kernel text and data | |
45 | from the hibernation image read by the booter. hibernate_kernel_entrypoint() and everything | |
46 | it calls or references needs to be careful to only touch memory also in the "__HIB" section. | |
47 | */ | |
48 | ||
49 | uint32_t gIOHibernateState; | |
50 | ||
b0d623f7 A |
51 | uint32_t gIOHibernateDebugFlags; |
52 | ||
3a60a9f5 A |
53 | static IOHibernateImageHeader _hibernateHeader; |
54 | IOHibernateImageHeader * gIOHibernateCurrentHeader = &_hibernateHeader; | |
55 | ||
6d2010ae A |
56 | ppnum_t gIOHibernateHandoffPages[64]; |
57 | uint32_t gIOHibernateHandoffPageCount = sizeof(gIOHibernateHandoffPages) | |
58 | / sizeof(gIOHibernateHandoffPages[0]); | |
3a60a9f5 | 59 | |
6d2010ae A |
60 | #if CONFIG_DEBUG |
61 | void hibprintf(const char *fmt, ...); | |
62 | #else | |
63 | #define hibprintf(x...) | |
64 | #endif | |
0c530ab8 | 65 | |
b0d623f7 A |
66 | |
67 | #if CONFIG_SLEEP | |
68 | #if defined(__i386__) || defined(__x86_64__) | |
69 | extern void acpi_wake_prot_entry(void); | |
70 | #endif | |
71 | #endif | |
72 | ||
73 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ | |
74 | ||
75 | #if defined(__i386__) || defined(__x86_64__) | |
5ba3f43e | 76 | #include <i386/proc_reg.h> |
bd504ef0 A |
77 | #else |
78 | ||
79 | static inline uint64_t rdtsc64(void) | |
80 | { | |
81 | return (0); | |
82 | } | |
83 | ||
84 | #endif /* defined(__i386__) || defined(__x86_64__) */ | |
85 | ||
86 | #if defined(__i386__) || defined(__x86_64__) | |
87 | ||
b0d623f7 A |
88 | #define DBGLOG 1 |
89 | ||
90 | #include <architecture/i386/pio.h> | |
91 | ||
92 | /* standard port addresses */ | |
93 | enum { | |
94 | COM1_PORT_ADDR = 0x3f8, | |
95 | COM2_PORT_ADDR = 0x2f8 | |
96 | }; | |
97 | ||
98 | /* UART register offsets */ | |
99 | enum { | |
100 | UART_RBR = 0, /* receive buffer Register (R) */ | |
101 | UART_THR = 0, /* transmit holding register (W) */ | |
102 | UART_DLL = 0, /* DLAB = 1, divisor latch (LSB) */ | |
103 | UART_IER = 1, /* interrupt enable register */ | |
104 | UART_DLM = 1, /* DLAB = 1, divisor latch (MSB) */ | |
105 | UART_IIR = 2, /* interrupt ident register (R) */ | |
106 | UART_FCR = 2, /* fifo control register (W) */ | |
107 | UART_LCR = 3, /* line control register */ | |
108 | UART_MCR = 4, /* modem control register */ | |
109 | UART_LSR = 5, /* line status register */ | |
110 | UART_MSR = 6, /* modem status register */ | |
111 | UART_SCR = 7 /* scratch register */ | |
112 | }; | |
113 | ||
114 | enum { | |
115 | UART_LCR_8BITS = 0x03, | |
116 | UART_LCR_DLAB = 0x80 | |
117 | }; | |
118 | ||
119 | enum { | |
120 | UART_MCR_DTR = 0x01, | |
121 | UART_MCR_RTS = 0x02, | |
122 | UART_MCR_OUT1 = 0x04, | |
123 | UART_MCR_OUT2 = 0x08, | |
124 | UART_MCR_LOOP = 0x10 | |
125 | }; | |
126 | ||
127 | enum { | |
128 | UART_LSR_DR = 0x01, | |
129 | UART_LSR_OE = 0x02, | |
130 | UART_LSR_PE = 0x04, | |
131 | UART_LSR_FE = 0x08, | |
132 | UART_LSR_THRE = 0x20 | |
133 | }; | |
134 | ||
135 | static void uart_putc(char c) | |
136 | { | |
137 | while (!(inb(COM1_PORT_ADDR + UART_LSR) & UART_LSR_THRE)) | |
138 | {} | |
139 | outb(COM1_PORT_ADDR + UART_THR, c); | |
140 | } | |
141 | ||
142 | static int debug_probe( void ) | |
143 | { | |
144 | /* Verify that the Scratch Register is accessible */ | |
145 | outb(COM1_PORT_ADDR + UART_SCR, 0x5a); | |
146 | if (inb(COM1_PORT_ADDR + UART_SCR) != 0x5a) return false; | |
147 | outb(COM1_PORT_ADDR + UART_SCR, 0xa5); | |
148 | if (inb(COM1_PORT_ADDR + UART_SCR) != 0xa5) return false; | |
149 | uart_putc('\n'); | |
150 | return true; | |
151 | } | |
152 | ||
153 | static void uart_puthex(uint64_t num) | |
154 | { | |
155 | int bit; | |
156 | char c; | |
157 | bool leading = true; | |
158 | ||
159 | for (bit = 60; bit >= 0; bit -= 4) | |
160 | { | |
161 | c = 0xf & (num >> bit); | |
162 | if (c) | |
163 | leading = false; | |
6d2010ae | 164 | else if (leading && bit) |
b0d623f7 A |
165 | continue; |
166 | if (c <= 9) | |
167 | c += '0'; | |
168 | else | |
169 | c+= 'a' - 10; | |
170 | uart_putc(c); | |
171 | } | |
172 | } | |
173 | ||
174 | static void debug_code(uint32_t code, uint64_t value) | |
175 | { | |
176 | int bit; | |
177 | char c; | |
178 | ||
179 | if (!(kIOHibernateDebugRestoreLogs & gIOHibernateDebugFlags)) | |
180 | return; | |
181 | ||
182 | for (bit = 24; bit >= 0; bit -= 8) | |
183 | { | |
184 | c = 0xFF & (code >> bit); | |
185 | if (c) | |
186 | uart_putc(c); | |
187 | } | |
188 | uart_putc('='); | |
189 | uart_puthex(value); | |
190 | uart_putc('\n'); | |
191 | uart_putc('\r'); | |
192 | } | |
193 | ||
194 | #endif /* defined(__i386__) || defined(__x86_64__) */ | |
195 | ||
196 | #if !defined(DBGLOG) | |
197 | #define debug_probe() (false) | |
198 | #define debug_code(c, v) {} | |
3a60a9f5 A |
199 | #endif |
200 | ||
b0d623f7 A |
201 | enum |
202 | { | |
203 | kIOHibernateRestoreCodeImageStart = 'imgS', | |
204 | kIOHibernateRestoreCodeImageEnd = 'imgE', | |
205 | kIOHibernateRestoreCodePageIndexStart = 'pgiS', | |
206 | kIOHibernateRestoreCodePageIndexEnd = 'pgiE', | |
207 | kIOHibernateRestoreCodeMapStart = 'mapS', | |
208 | kIOHibernateRestoreCodeMapEnd = 'mapE', | |
209 | kIOHibernateRestoreCodeWakeMapSize = 'wkms', | |
210 | kIOHibernateRestoreCodeConflictPage = 'cfpg', | |
211 | kIOHibernateRestoreCodeConflictSource = 'cfsr', | |
7ddcb079 A |
212 | kIOHibernateRestoreCodeNoMemory = 'nomm', |
213 | kIOHibernateRestoreCodeTag = 'tag ', | |
214 | kIOHibernateRestoreCodeSignature = 'sign', | |
215 | kIOHibernateRestoreCodeMapVirt = 'mapV', | |
216 | kIOHibernateRestoreCodeHandoffPages = 'hand', | |
217 | kIOHibernateRestoreCodeHandoffCount = 'hndc', | |
b0d623f7 A |
218 | }; |
219 | ||
220 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ | |
221 | ||
222 | ||
223 | static void fatal(void) | |
224 | { | |
225 | #if defined(__i386__) || defined(__x86_64__) | |
226 | outb(0xcf9, 6); | |
227 | #else | |
228 | while (true) {} | |
229 | #endif | |
230 | } | |
231 | ||
3a60a9f5 A |
232 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
233 | ||
3a60a9f5 | 234 | uint32_t |
0b4c1975 | 235 | hibernate_sum_page(uint8_t *buf, uint32_t ppnum) |
3a60a9f5 | 236 | { |
0b4c1975 | 237 | return (((uint32_t *)buf)[((PAGE_SIZE >> 2) - 1) & ppnum]); |
3a60a9f5 A |
238 | } |
239 | ||
240 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ | |
241 | ||
0c530ab8 A |
242 | static hibernate_bitmap_t * |
243 | hibernate_page_bitmap(hibernate_page_list_t * list, uint32_t page) | |
89b3af67 | 244 | { |
6601e61a | 245 | uint32_t bank; |
21362eb3 | 246 | hibernate_bitmap_t * bitmap = &list->bank_bitmap[0]; |
89b3af67 | 247 | |
21362eb3 | 248 | for (bank = 0; bank < list->bank_count; bank++) |
89b3af67 | 249 | { |
6601e61a | 250 | if ((page >= bitmap->first_page) && (page <= bitmap->last_page)) |
21362eb3 | 251 | break; |
21362eb3 | 252 | bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords]; |
89b3af67 | 253 | } |
0c530ab8 | 254 | if (bank == list->bank_count) |
2d21ac55 | 255 | bitmap = NULL; |
0c530ab8 A |
256 | |
257 | return (bitmap); | |
4452a7af A |
258 | } |
259 | ||
0c530ab8 A |
260 | hibernate_bitmap_t * |
261 | hibernate_page_bitmap_pin(hibernate_page_list_t * list, uint32_t * pPage) | |
4452a7af | 262 | { |
0c530ab8 | 263 | uint32_t bank, page = *pPage; |
6601e61a | 264 | hibernate_bitmap_t * bitmap = &list->bank_bitmap[0]; |
4452a7af | 265 | |
6601e61a | 266 | for (bank = 0; bank < list->bank_count; bank++) |
4452a7af | 267 | { |
0c530ab8 | 268 | if (page <= bitmap->first_page) |
6601e61a | 269 | { |
0c530ab8 | 270 | *pPage = bitmap->first_page; |
6601e61a A |
271 | break; |
272 | } | |
0c530ab8 A |
273 | if (page <= bitmap->last_page) |
274 | break; | |
6601e61a | 275 | bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords]; |
4452a7af | 276 | } |
0c530ab8 | 277 | if (bank == list->bank_count) |
2d21ac55 | 278 | bitmap = NULL; |
0c530ab8 A |
279 | |
280 | return (bitmap); | |
281 | } | |
282 | ||
283 | void | |
284 | hibernate_page_bitset(hibernate_page_list_t * list, boolean_t set, uint32_t page) | |
285 | { | |
286 | hibernate_bitmap_t * bitmap; | |
287 | ||
288 | bitmap = hibernate_page_bitmap(list, page); | |
289 | if (bitmap) | |
290 | { | |
291 | page -= bitmap->first_page; | |
292 | if (set) | |
293 | bitmap->bitmap[page >> 5] |= (0x80000000 >> (page & 31)); | |
294 | //setbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]); | |
295 | else | |
296 | bitmap->bitmap[page >> 5] &= ~(0x80000000 >> (page & 31)); | |
297 | //clrbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]); | |
298 | } | |
299 | } | |
300 | ||
301 | boolean_t | |
302 | hibernate_page_bittst(hibernate_page_list_t * list, uint32_t page) | |
303 | { | |
304 | boolean_t result = TRUE; | |
305 | hibernate_bitmap_t * bitmap; | |
306 | ||
307 | bitmap = hibernate_page_bitmap(list, page); | |
308 | if (bitmap) | |
309 | { | |
310 | page -= bitmap->first_page; | |
311 | result = (0 != (bitmap->bitmap[page >> 5] & (0x80000000 >> (page & 31)))); | |
312 | } | |
3a60a9f5 A |
313 | return (result); |
314 | } | |
315 | ||
0c530ab8 | 316 | // count bits clear or set (set == TRUE) starting at page. |
3a60a9f5 | 317 | uint32_t |
0c530ab8 | 318 | hibernate_page_bitmap_count(hibernate_bitmap_t * bitmap, uint32_t set, uint32_t page) |
3a60a9f5 | 319 | { |
0c530ab8 A |
320 | uint32_t index, bit, bits; |
321 | uint32_t count; | |
3a60a9f5 | 322 | |
0c530ab8 | 323 | count = 0; |
4452a7af | 324 | |
0c530ab8 A |
325 | index = (page - bitmap->first_page) >> 5; |
326 | bit = (page - bitmap->first_page) & 31; | |
327 | ||
328 | bits = bitmap->bitmap[index]; | |
329 | if (set) | |
330 | bits = ~bits; | |
331 | bits = (bits << bit); | |
332 | if (bits) | |
333 | count += __builtin_clz(bits); | |
334 | else | |
3a60a9f5 | 335 | { |
0c530ab8 A |
336 | count += 32 - bit; |
337 | while (++index < bitmap->bitmapwords) | |
3a60a9f5 | 338 | { |
0c530ab8 A |
339 | bits = bitmap->bitmap[index]; |
340 | if (set) | |
341 | bits = ~bits; | |
342 | if (bits) | |
3a60a9f5 | 343 | { |
0c530ab8 A |
344 | count += __builtin_clz(bits); |
345 | break; | |
3a60a9f5 | 346 | } |
0c530ab8 | 347 | count += 32; |
3a60a9f5 | 348 | } |
3a60a9f5 A |
349 | } |
350 | ||
7ddcb079 A |
351 | if ((page + count) > (bitmap->last_page + 1)) count = (bitmap->last_page + 1) - page; |
352 | ||
3a60a9f5 A |
353 | return (count); |
354 | } | |
355 | ||
6d2010ae | 356 | static ppnum_t |
0c530ab8 | 357 | hibernate_page_list_grab(hibernate_page_list_t * list, uint32_t * pNextFree) |
3a60a9f5 | 358 | { |
0c530ab8 A |
359 | uint32_t nextFree = *pNextFree; |
360 | uint32_t nextFreeInBank; | |
361 | hibernate_bitmap_t * bitmap; | |
3a60a9f5 | 362 | |
0c530ab8 A |
363 | nextFreeInBank = nextFree + 1; |
364 | while ((bitmap = hibernate_page_bitmap_pin(list, &nextFreeInBank))) | |
365 | { | |
366 | nextFreeInBank += hibernate_page_bitmap_count(bitmap, FALSE, nextFreeInBank); | |
367 | if (nextFreeInBank <= bitmap->last_page) | |
368 | { | |
369 | *pNextFree = nextFreeInBank; | |
370 | break; | |
371 | } | |
372 | } | |
3a60a9f5 | 373 | |
b0d623f7 A |
374 | if (!bitmap) |
375 | { | |
376 | debug_code(kIOHibernateRestoreCodeNoMemory, nextFree); | |
377 | fatal(); | |
378 | nextFree = 0; | |
379 | } | |
380 | ||
3a60a9f5 A |
381 | return (nextFree); |
382 | } | |
383 | ||
384 | static uint32_t | |
385 | store_one_page(uint32_t procFlags, uint32_t * src, uint32_t compressedSize, | |
386 | uint32_t * buffer, uint32_t ppnum) | |
387 | { | |
6d2010ae | 388 | uint64_t dst = ptoa_64(ppnum); |
39037602 | 389 | uint8_t scratch[WKdm_SCRATCH_BUF_SIZE_INTERNAL] __attribute__ ((aligned (16))); |
3a60a9f5 | 390 | |
6d2010ae A |
391 | if (compressedSize != PAGE_SIZE) |
392 | { | |
393 | dst = pal_hib_map(DEST_COPY_AREA, dst); | |
3e170ce0 | 394 | if (compressedSize != 4) WKdm_decompress_new((WK_word*) src, (WK_word*)(uintptr_t)dst, (WK_word*) &scratch[0], compressedSize); |
743345f9 A |
395 | else |
396 | { | |
397 | size_t i; | |
398 | uint32_t s, *d; | |
3e170ce0 | 399 | |
743345f9 A |
400 | s = *src; |
401 | d = (uint32_t *)(uintptr_t)dst; | |
5ba3f43e | 402 | if (!s) __nosan_bzero((void *) dst, PAGE_SIZE); |
743345f9 | 403 | else for (i = 0; i < (PAGE_SIZE / sizeof(int32_t)); i++) *d++ = s; |
3e170ce0 | 404 | } |
6d2010ae A |
405 | } |
406 | else | |
407 | { | |
408 | dst = hibernate_restore_phys_page((uint64_t) (uintptr_t) src, dst, PAGE_SIZE, procFlags); | |
409 | } | |
3a60a9f5 | 410 | |
6d2010ae | 411 | return hibernate_sum_page((uint8_t *)(uintptr_t)dst, ppnum); |
3a60a9f5 A |
412 | } |
413 | ||
3a60a9f5 | 414 | long |
7ddcb079 A |
415 | hibernate_kernel_entrypoint(uint32_t p1, |
416 | uint32_t p2, uint32_t p3, uint32_t p4) | |
3a60a9f5 | 417 | { |
7ddcb079 A |
418 | uint64_t headerPhys; |
419 | uint64_t mapPhys; | |
420 | uint64_t srcPhys; | |
421 | uint64_t imageReadPhys; | |
422 | uint64_t pageIndexPhys; | |
3a60a9f5 A |
423 | uint32_t * pageIndexSource; |
424 | hibernate_page_list_t * map; | |
6d2010ae | 425 | uint32_t stage; |
3a60a9f5 A |
426 | uint32_t count; |
427 | uint32_t ppnum; | |
428 | uint32_t page; | |
429 | uint32_t conflictCount; | |
430 | uint32_t compressedSize; | |
431 | uint32_t uncompressedPages; | |
7ddcb079 A |
432 | uint32_t copyPageListHeadPage; |
433 | uint32_t pageListPage; | |
3a60a9f5 | 434 | uint32_t * copyPageList; |
7ddcb079 | 435 | uint32_t * src; |
3a60a9f5 A |
436 | uint32_t copyPageIndex; |
437 | uint32_t sum; | |
6d2010ae | 438 | uint32_t pageSum; |
3a60a9f5 A |
439 | uint32_t nextFree; |
440 | uint32_t lastImagePage; | |
441 | uint32_t lastMapPage; | |
442 | uint32_t lastPageIndexPage; | |
6d2010ae A |
443 | uint32_t handoffPages; |
444 | uint32_t handoffPageCount; | |
3a60a9f5 | 445 | |
39236c6e | 446 | uint64_t timeStart; |
bd504ef0 A |
447 | timeStart = rdtsc64(); |
448 | ||
39037602 | 449 | static_assert(sizeof(IOHibernateImageHeader) == 512); |
3a60a9f5 | 450 | |
7ddcb079 A |
451 | headerPhys = ptoa_64(p1); |
452 | ||
b0d623f7 A |
453 | if ((kIOHibernateDebugRestoreLogs & gIOHibernateDebugFlags) && !debug_probe()) |
454 | gIOHibernateDebugFlags &= ~kIOHibernateDebugRestoreLogs; | |
455 | ||
7ddcb079 A |
456 | debug_code(kIOHibernateRestoreCodeImageStart, headerPhys); |
457 | ||
5ba3f43e | 458 | __nosan_memcpy(gIOHibernateCurrentHeader, |
39236c6e A |
459 | (void *) pal_hib_map(IMAGE_AREA, headerPhys), |
460 | sizeof(IOHibernateImageHeader)); | |
b0d623f7 | 461 | |
7ddcb079 | 462 | debug_code(kIOHibernateRestoreCodeSignature, gIOHibernateCurrentHeader->signature); |
3a60a9f5 | 463 | |
7ddcb079 A |
464 | mapPhys = headerPhys |
465 | + (offsetof(IOHibernateImageHeader, fileExtentMap) | |
466 | + gIOHibernateCurrentHeader->fileExtentMapSize | |
467 | + ptoa_32(gIOHibernateCurrentHeader->restore1PageCount) | |
468 | + gIOHibernateCurrentHeader->previewSize); | |
b0d623f7 | 469 | |
7ddcb079 | 470 | map = (hibernate_page_list_t *) pal_hib_map(BITMAP_AREA, mapPhys); |
3a60a9f5 | 471 | |
7ddcb079 A |
472 | lastImagePage = atop_64(headerPhys + gIOHibernateCurrentHeader->image1Size); |
473 | lastMapPage = atop_64(mapPhys + gIOHibernateCurrentHeader->bitmapSize); | |
3a60a9f5 | 474 | |
7ddcb079 A |
475 | handoffPages = gIOHibernateCurrentHeader->handoffPages; |
476 | handoffPageCount = gIOHibernateCurrentHeader->handoffPageCount; | |
6d2010ae | 477 | |
b0d623f7 | 478 | debug_code(kIOHibernateRestoreCodeImageEnd, ptoa_64(lastImagePage)); |
7ddcb079 | 479 | debug_code(kIOHibernateRestoreCodeMapStart, mapPhys); |
b0d623f7 | 480 | debug_code(kIOHibernateRestoreCodeMapEnd, ptoa_64(lastMapPage)); |
3a60a9f5 | 481 | |
7ddcb079 A |
482 | debug_code(kIOHibernateRestoreCodeMapVirt, (uintptr_t) map); |
483 | debug_code(kIOHibernateRestoreCodeHandoffPages, ptoa_64(handoffPages)); | |
484 | debug_code(kIOHibernateRestoreCodeHandoffCount, handoffPageCount); | |
6d2010ae | 485 | |
3a60a9f5 | 486 | // knock all the image pages to be used out of free map |
7ddcb079 | 487 | for (ppnum = atop_64(headerPhys); ppnum <= lastImagePage; ppnum++) |
3a60a9f5 A |
488 | { |
489 | hibernate_page_bitset(map, FALSE, ppnum); | |
490 | } | |
6d2010ae A |
491 | // knock all the handoff pages to be used out of free map |
492 | for (ppnum = handoffPages; ppnum < (handoffPages + handoffPageCount); ppnum++) | |
493 | { | |
494 | hibernate_page_bitset(map, FALSE, ppnum); | |
495 | } | |
3a60a9f5 A |
496 | |
497 | nextFree = 0; | |
0c530ab8 | 498 | hibernate_page_list_grab(map, &nextFree); |
3a60a9f5 | 499 | |
7ddcb079 A |
500 | sum = gIOHibernateCurrentHeader->actualRestore1Sum; |
501 | gIOHibernateCurrentHeader->diag[0] = atop_64(headerPhys); | |
3a60a9f5 | 502 | gIOHibernateCurrentHeader->diag[1] = sum; |
39236c6e | 503 | gIOHibernateCurrentHeader->trampolineTime = 0; |
3a60a9f5 | 504 | |
7ddcb079 A |
505 | uncompressedPages = 0; |
506 | conflictCount = 0; | |
507 | copyPageListHeadPage = 0; | |
508 | copyPageList = 0; | |
509 | copyPageIndex = PAGE_SIZE >> 2; | |
3a60a9f5 | 510 | |
7ddcb079 A |
511 | compressedSize = PAGE_SIZE; |
512 | stage = 2; | |
513 | count = 0; | |
514 | srcPhys = 0; | |
6d2010ae A |
515 | |
516 | if (gIOHibernateCurrentHeader->previewSize) | |
517 | { | |
7ddcb079 A |
518 | pageIndexPhys = headerPhys |
519 | + (offsetof(IOHibernateImageHeader, fileExtentMap) | |
520 | + gIOHibernateCurrentHeader->fileExtentMapSize | |
521 | + ptoa_32(gIOHibernateCurrentHeader->restore1PageCount)); | |
522 | imageReadPhys = (pageIndexPhys + gIOHibernateCurrentHeader->previewPageListSize); | |
523 | lastPageIndexPage = atop_64(imageReadPhys); | |
524 | pageIndexSource = (uint32_t *) pal_hib_map(IMAGE2_AREA, pageIndexPhys); | |
6d2010ae A |
525 | } |
526 | else | |
527 | { | |
7ddcb079 | 528 | pageIndexPhys = 0; |
6d2010ae | 529 | lastPageIndexPage = 0; |
7ddcb079 | 530 | imageReadPhys = (mapPhys + gIOHibernateCurrentHeader->bitmapSize); |
6d2010ae A |
531 | } |
532 | ||
7ddcb079 | 533 | debug_code(kIOHibernateRestoreCodePageIndexStart, pageIndexPhys); |
6d2010ae | 534 | debug_code(kIOHibernateRestoreCodePageIndexEnd, ptoa_64(lastPageIndexPage)); |
3a60a9f5 A |
535 | |
536 | while (1) | |
537 | { | |
6d2010ae A |
538 | switch (stage) |
539 | { | |
540 | case 2: | |
541 | // copy handoff data | |
7ddcb079 | 542 | count = srcPhys ? 0 : handoffPageCount; |
6d2010ae A |
543 | if (!count) |
544 | break; | |
7ddcb079 A |
545 | if (count > gIOHibernateHandoffPageCount) count = gIOHibernateHandoffPageCount; |
546 | srcPhys = ptoa_64(handoffPages); | |
6d2010ae A |
547 | break; |
548 | ||
549 | case 1: | |
550 | // copy pageIndexSource pages == preview image data | |
7ddcb079 | 551 | if (!srcPhys) |
6d2010ae | 552 | { |
7ddcb079 A |
553 | if (!pageIndexPhys) break; |
554 | srcPhys = imageReadPhys; | |
6d2010ae A |
555 | } |
556 | ppnum = pageIndexSource[0]; | |
557 | count = pageIndexSource[1]; | |
558 | pageIndexSource += 2; | |
7ddcb079 A |
559 | pageIndexPhys += 2 * sizeof(pageIndexSource[0]); |
560 | imageReadPhys = srcPhys; | |
6d2010ae A |
561 | break; |
562 | ||
563 | case 0: | |
564 | // copy pages | |
7ddcb079 A |
565 | if (!srcPhys) srcPhys = (mapPhys + gIOHibernateCurrentHeader->bitmapSize); |
566 | src = (uint32_t *) pal_hib_map(IMAGE_AREA, srcPhys); | |
6d2010ae A |
567 | ppnum = src[0]; |
568 | count = src[1]; | |
7ddcb079 A |
569 | srcPhys += 2 * sizeof(*src); |
570 | imageReadPhys = srcPhys; | |
6d2010ae A |
571 | break; |
572 | } | |
573 | ||
574 | ||
575 | if (!count) | |
576 | { | |
577 | if (!stage) | |
578 | break; | |
579 | stage--; | |
7ddcb079 | 580 | srcPhys = 0; |
6d2010ae | 581 | continue; |
3a60a9f5 A |
582 | } |
583 | ||
584 | for (page = 0; page < count; page++, ppnum++) | |
585 | { | |
6d2010ae | 586 | uint32_t tag; |
3a60a9f5 A |
587 | int conflicts; |
588 | ||
7ddcb079 A |
589 | src = (uint32_t *) pal_hib_map(IMAGE_AREA, srcPhys); |
590 | ||
591 | if (2 == stage) ppnum = gIOHibernateHandoffPages[page]; | |
6d2010ae A |
592 | else if (!stage) |
593 | { | |
594 | tag = *src++; | |
7ddcb079 A |
595 | // debug_code(kIOHibernateRestoreCodeTag, (uintptr_t) tag); |
596 | srcPhys += sizeof(*src); | |
6d2010ae A |
597 | compressedSize = kIOHibernateTagLength & tag; |
598 | } | |
599 | ||
7ddcb079 | 600 | conflicts = (ppnum >= atop_64(mapPhys)) && (ppnum <= lastMapPage); |
3a60a9f5 | 601 | |
7ddcb079 | 602 | conflicts |= ((ppnum >= atop_64(imageReadPhys)) && (ppnum <= lastImagePage)); |
3a60a9f5 | 603 | |
6d2010ae | 604 | if (stage >= 2) |
7ddcb079 | 605 | conflicts |= ((ppnum >= atop_64(srcPhys)) && (ppnum <= (handoffPages + handoffPageCount - 1))); |
6d2010ae A |
606 | |
607 | if (stage >= 1) | |
7ddcb079 | 608 | conflicts |= ((ppnum >= atop_64(pageIndexPhys)) && (ppnum <= lastPageIndexPage)); |
3a60a9f5 A |
609 | |
610 | if (!conflicts) | |
611 | { | |
6d2010ae A |
612 | pageSum = store_one_page(gIOHibernateCurrentHeader->processorFlags, |
613 | src, compressedSize, 0, ppnum); | |
614 | if (stage != 2) | |
615 | sum += pageSum; | |
3a60a9f5 A |
616 | uncompressedPages++; |
617 | } | |
618 | else | |
619 | { | |
39236c6e | 620 | uint32_t bufferPage = 0; |
3a60a9f5 A |
621 | uint32_t * dst; |
622 | ||
b0d623f7 A |
623 | // debug_code(kIOHibernateRestoreCodeConflictPage, ppnum); |
624 | // debug_code(kIOHibernateRestoreCodeConflictSource, (uintptr_t) src); | |
3a60a9f5 | 625 | conflictCount++; |
39236c6e A |
626 | if (compressedSize) |
627 | { | |
628 | // alloc new buffer page | |
629 | bufferPage = hibernate_page_list_grab(map, &nextFree); | |
630 | dst = (uint32_t *)pal_hib_map(DEST_COPY_AREA, ptoa_64(bufferPage)); | |
5ba3f43e | 631 | __nosan_memcpy(dst, src, compressedSize); |
39236c6e | 632 | } |
3a60a9f5 A |
633 | if (copyPageIndex > ((PAGE_SIZE >> 2) - 3)) |
634 | { | |
635 | // alloc new copy list page | |
7ddcb079 | 636 | pageListPage = hibernate_page_list_grab(map, &nextFree); |
3a60a9f5 | 637 | // link to current |
6d2010ae A |
638 | if (copyPageList) { |
639 | copyPageList[1] = pageListPage; | |
640 | } else { | |
7ddcb079 | 641 | copyPageListHeadPage = pageListPage; |
6d2010ae A |
642 | } |
643 | copyPageList = (uint32_t *)pal_hib_map(SRC_COPY_AREA, | |
7ddcb079 | 644 | ptoa_64(pageListPage)); |
3a60a9f5 A |
645 | copyPageList[1] = 0; |
646 | copyPageIndex = 2; | |
647 | } | |
3a60a9f5 A |
648 | copyPageList[copyPageIndex++] = ppnum; |
649 | copyPageList[copyPageIndex++] = bufferPage; | |
6d2010ae | 650 | copyPageList[copyPageIndex++] = (compressedSize | (stage << 24)); |
3a60a9f5 | 651 | copyPageList[0] = copyPageIndex; |
3a60a9f5 | 652 | } |
7ddcb079 A |
653 | srcPhys += ((compressedSize + 3) & ~3); |
654 | src += ((compressedSize + 3) >> 2); | |
3a60a9f5 A |
655 | } |
656 | } | |
657 | ||
6d2010ae A |
658 | /* src points to the last page restored, so we need to skip over that */ |
659 | hibernateRestorePALState(src); | |
660 | ||
3a60a9f5 A |
661 | // -- copy back conflicts |
662 | ||
7ddcb079 A |
663 | pageListPage = copyPageListHeadPage; |
664 | while (pageListPage) | |
3a60a9f5 | 665 | { |
7ddcb079 | 666 | copyPageList = (uint32_t *)pal_hib_map(COPY_PAGE_AREA, ptoa_64(pageListPage)); |
3a60a9f5 A |
667 | for (copyPageIndex = 2; copyPageIndex < copyPageList[0]; copyPageIndex += 3) |
668 | { | |
6d2010ae | 669 | ppnum = copyPageList[copyPageIndex + 0]; |
7ddcb079 A |
670 | srcPhys = ptoa_64(copyPageList[copyPageIndex + 1]); |
671 | src = (uint32_t *) pal_hib_map(SRC_COPY_AREA, srcPhys); | |
6d2010ae A |
672 | compressedSize = copyPageList[copyPageIndex + 2]; |
673 | stage = compressedSize >> 24; | |
674 | compressedSize &= 0x1FFF; | |
675 | pageSum = store_one_page(gIOHibernateCurrentHeader->processorFlags, | |
676 | src, compressedSize, 0, ppnum); | |
677 | if (stage != 2) | |
678 | sum += pageSum; | |
3a60a9f5 A |
679 | uncompressedPages++; |
680 | } | |
7ddcb079 | 681 | pageListPage = copyPageList[1]; |
3a60a9f5 A |
682 | } |
683 | ||
6d2010ae A |
684 | pal_hib_patchup(); |
685 | ||
3a60a9f5 A |
686 | // -- image has been destroyed... |
687 | ||
688 | gIOHibernateCurrentHeader->actualImage1Sum = sum; | |
689 | gIOHibernateCurrentHeader->actualUncompressedPages = uncompressedPages; | |
690 | gIOHibernateCurrentHeader->conflictCount = conflictCount; | |
691 | gIOHibernateCurrentHeader->nextFree = nextFree; | |
692 | ||
693 | gIOHibernateState = kIOHibernateStateWakingFromHibernate; | |
694 | ||
39236c6e | 695 | gIOHibernateCurrentHeader->trampolineTime = (((rdtsc64() - timeStart)) >> 8); |
bd504ef0 | 696 | |
3e170ce0 A |
697 | // debug_code('done', 0); |
698 | ||
b0d623f7 | 699 | #if CONFIG_SLEEP |
6d2010ae | 700 | #if defined(__i386__) || defined(__x86_64__) |
b0d623f7 A |
701 | typedef void (*ResetProc)(void); |
702 | ResetProc proc; | |
6d2010ae | 703 | proc = HIB_ENTRYPOINT; |
0c530ab8 A |
704 | // flush caches |
705 | __asm__("wbinvd"); | |
3a60a9f5 | 706 | proc(); |
b0d623f7 A |
707 | #else |
708 | // implement me | |
709 | #endif | |
3a60a9f5 | 710 | #endif |
b0d623f7 | 711 | |
3a60a9f5 A |
712 | return -1; |
713 | } | |
6d2010ae A |
714 | |
715 | #if CONFIG_DEBUG | |
716 | /* standalone printf implementation */ | |
717 | /*- | |
718 | * Copyright (c) 1986, 1988, 1991, 1993 | |
719 | * The Regents of the University of California. All rights reserved. | |
720 | * (c) UNIX System Laboratories, Inc. | |
721 | * All or some portions of this file are derived from material licensed | |
722 | * to the University of California by American Telephone and Telegraph | |
723 | * Co. or Unix System Laboratories, Inc. and are reproduced herein with | |
724 | * the permission of UNIX System Laboratories, Inc. | |
725 | * | |
726 | * Redistribution and use in source and binary forms, with or without | |
727 | * modification, are permitted provided that the following conditions | |
728 | * are met: | |
729 | * 1. Redistributions of source code must retain the above copyright | |
730 | * notice, this list of conditions and the following disclaimer. | |
731 | * 2. Redistributions in binary form must reproduce the above copyright | |
732 | * notice, this list of conditions and the following disclaimer in the | |
733 | * documentation and/or other materials provided with the distribution. | |
734 | * 4. Neither the name of the University nor the names of its contributors | |
735 | * may be used to endorse or promote products derived from this software | |
736 | * without specific prior written permission. | |
737 | * | |
738 | * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND | |
739 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |
740 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |
741 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE | |
742 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | |
743 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS | |
744 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) | |
745 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT | |
746 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY | |
747 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | |
748 | * SUCH DAMAGE. | |
749 | * | |
750 | * @(#)subr_prf.c 8.3 (Berkeley) 1/21/94 | |
751 | */ | |
752 | ||
753 | typedef long ptrdiff_t; | |
754 | char const hibhex2ascii_data[] = "0123456789abcdefghijklmnopqrstuvwxyz"; | |
755 | #define hibhex2ascii(hex) (hibhex2ascii_data[hex]) | |
756 | #define toupper(c) ((c) - 0x20 * (((c) >= 'a') && ((c) <= 'z'))) | |
757 | static size_t | |
758 | hibstrlen(const char *s) | |
759 | { | |
760 | size_t l = 0; | |
761 | while (*s++) | |
762 | l++; | |
763 | return l; | |
764 | } | |
765 | ||
766 | /* Max number conversion buffer length: a u_quad_t in base 2, plus NUL byte. */ | |
767 | #define MAXNBUF (sizeof(intmax_t) * NBBY + 1) | |
768 | ||
769 | /* | |
770 | * Put a NUL-terminated ASCII number (base <= 36) in a buffer in reverse | |
771 | * order; return an optional length and a pointer to the last character | |
772 | * written in the buffer (i.e., the first character of the string). | |
773 | * The buffer pointed to by `nbuf' must have length >= MAXNBUF. | |
774 | */ | |
775 | static char * | |
776 | ksprintn(char *nbuf, uintmax_t num, int base, int *lenp, int upper) | |
777 | { | |
778 | char *p, c; | |
779 | ||
780 | /* Truncate so we don't call umoddi3, which isn't in __HIB */ | |
781 | #if !defined(__LP64__) | |
782 | uint32_t num2 = (uint32_t) num; | |
783 | #else | |
784 | uintmax_t num2 = num; | |
785 | #endif | |
786 | ||
787 | p = nbuf; | |
788 | *p = '\0'; | |
789 | do { | |
790 | c = hibhex2ascii(num2 % base); | |
791 | *++p = upper ? toupper(c) : c; | |
792 | } while (num2 /= base); | |
793 | if (lenp) | |
794 | *lenp = (int)(p - nbuf); | |
795 | return (p); | |
796 | } | |
797 | ||
798 | /* | |
799 | * Scaled down version of printf(3). | |
800 | * | |
801 | * Two additional formats: | |
802 | * | |
803 | * The format %b is supported to decode error registers. | |
804 | * Its usage is: | |
805 | * | |
806 | * printf("reg=%b\n", regval, "*"); | |
807 | * | |
808 | * where is the output base expressed as a control character, e.g. | |
809 | * \10 gives octal; \20 gives hex. Each arg is a sequence of characters, | |
810 | * the first of which gives the bit number to be inspected (origin 1), and | |
811 | * the next characters (up to a control character, i.e. a character <= 32), | |
812 | * give the name of the register. Thus: | |
813 | * | |
814 | * kvprintf("reg=%b\n", 3, "\10\2BITTWO\1BITONE\n"); | |
815 | * | |
816 | * would produce output: | |
817 | * | |
818 | * reg=3 | |
819 | * | |
820 | * XXX: %D -- Hexdump, takes pointer and separator string: | |
821 | * ("%6D", ptr, ":") -> XX:XX:XX:XX:XX:XX | |
822 | * ("%*D", len, ptr, " " -> XX XX XX XX ... | |
823 | */ | |
824 | static int | |
825 | hibkvprintf(char const *fmt, void (*func)(int, void*), void *arg, int radix, va_list ap) | |
826 | { | |
827 | #define PCHAR(c) {int cc=(c); if (func) (*func)(cc,arg); else *d++ = cc; retval++; } | |
828 | char nbuf[MAXNBUF]; | |
829 | char *d; | |
830 | const char *p, *percent, *q; | |
831 | u_char *up; | |
832 | int ch, n; | |
833 | uintmax_t num; | |
834 | int base, lflag, qflag, tmp, width, ladjust, sharpflag, neg, sign, dot; | |
835 | int cflag, hflag, jflag, tflag, zflag; | |
836 | int dwidth, upper; | |
837 | char padc; | |
838 | int stop = 0, retval = 0; | |
839 | ||
840 | num = 0; | |
841 | if (!func) | |
842 | d = (char *) arg; | |
843 | else | |
844 | d = NULL; | |
845 | ||
846 | if (fmt == NULL) | |
847 | fmt = "(fmt null)\n"; | |
848 | ||
849 | if (radix < 2 || radix > 36) | |
850 | radix = 10; | |
851 | ||
852 | for (;;) { | |
853 | padc = ' '; | |
854 | width = 0; | |
855 | while ((ch = (u_char)*fmt++) != '%' || stop) { | |
856 | if (ch == '\0') | |
857 | return (retval); | |
858 | PCHAR(ch); | |
859 | } | |
860 | percent = fmt - 1; | |
861 | qflag = 0; lflag = 0; ladjust = 0; sharpflag = 0; neg = 0; | |
862 | sign = 0; dot = 0; dwidth = 0; upper = 0; | |
863 | cflag = 0; hflag = 0; jflag = 0; tflag = 0; zflag = 0; | |
864 | reswitch: switch (ch = (u_char)*fmt++) { | |
865 | case '.': | |
866 | dot = 1; | |
867 | goto reswitch; | |
868 | case '#': | |
869 | sharpflag = 1; | |
870 | goto reswitch; | |
871 | case '+': | |
872 | sign = 1; | |
873 | goto reswitch; | |
874 | case '-': | |
875 | ladjust = 1; | |
876 | goto reswitch; | |
877 | case '%': | |
878 | PCHAR(ch); | |
879 | break; | |
880 | case '*': | |
881 | if (!dot) { | |
882 | width = va_arg(ap, int); | |
883 | if (width < 0) { | |
884 | ladjust = !ladjust; | |
885 | width = -width; | |
886 | } | |
887 | } else { | |
888 | dwidth = va_arg(ap, int); | |
889 | } | |
890 | goto reswitch; | |
891 | case '0': | |
892 | if (!dot) { | |
893 | padc = '0'; | |
894 | goto reswitch; | |
895 | } | |
896 | case '1': case '2': case '3': case '4': | |
897 | case '5': case '6': case '7': case '8': case '9': | |
898 | for (n = 0;; ++fmt) { | |
899 | n = n * 10 + ch - '0'; | |
900 | ch = *fmt; | |
901 | if (ch < '0' || ch > '9') | |
902 | break; | |
903 | } | |
904 | if (dot) | |
905 | dwidth = n; | |
906 | else | |
907 | width = n; | |
908 | goto reswitch; | |
909 | case 'b': | |
910 | num = (u_int)va_arg(ap, int); | |
911 | p = va_arg(ap, char *); | |
912 | for (q = ksprintn(nbuf, num, *p++, NULL, 0); *q;) | |
913 | PCHAR(*q--); | |
914 | ||
915 | if (num == 0) | |
916 | break; | |
917 | ||
918 | for (tmp = 0; *p;) { | |
919 | n = *p++; | |
920 | if (num & (1 << (n - 1))) { | |
921 | PCHAR(tmp ? ',' : '<'); | |
922 | for (; (n = *p) > ' '; ++p) | |
923 | PCHAR(n); | |
924 | tmp = 1; | |
925 | } else | |
926 | for (; *p > ' '; ++p) | |
927 | continue; | |
928 | } | |
929 | if (tmp) | |
930 | PCHAR('>'); | |
931 | break; | |
932 | case 'c': | |
933 | PCHAR(va_arg(ap, int)); | |
934 | break; | |
935 | case 'D': | |
936 | up = va_arg(ap, u_char *); | |
937 | p = va_arg(ap, char *); | |
938 | if (!width) | |
939 | width = 16; | |
940 | while(width--) { | |
941 | PCHAR(hibhex2ascii(*up >> 4)); | |
942 | PCHAR(hibhex2ascii(*up & 0x0f)); | |
943 | up++; | |
944 | if (width) | |
945 | for (q=p;*q;q++) | |
946 | PCHAR(*q); | |
947 | } | |
948 | break; | |
949 | case 'd': | |
950 | case 'i': | |
951 | base = 10; | |
952 | sign = 1; | |
953 | goto handle_sign; | |
954 | case 'h': | |
955 | if (hflag) { | |
956 | hflag = 0; | |
957 | cflag = 1; | |
958 | } else | |
959 | hflag = 1; | |
960 | goto reswitch; | |
961 | case 'j': | |
962 | jflag = 1; | |
963 | goto reswitch; | |
964 | case 'l': | |
965 | if (lflag) { | |
966 | lflag = 0; | |
967 | qflag = 1; | |
968 | } else | |
969 | lflag = 1; | |
970 | goto reswitch; | |
971 | case 'n': | |
972 | if (jflag) | |
973 | *(va_arg(ap, intmax_t *)) = retval; | |
974 | else if (qflag) | |
975 | *(va_arg(ap, quad_t *)) = retval; | |
976 | else if (lflag) | |
977 | *(va_arg(ap, long *)) = retval; | |
978 | else if (zflag) | |
979 | *(va_arg(ap, size_t *)) = retval; | |
980 | else if (hflag) | |
981 | *(va_arg(ap, short *)) = retval; | |
982 | else if (cflag) | |
983 | *(va_arg(ap, char *)) = retval; | |
984 | else | |
985 | *(va_arg(ap, int *)) = retval; | |
986 | break; | |
987 | case 'o': | |
988 | base = 8; | |
989 | goto handle_nosign; | |
990 | case 'p': | |
991 | base = 16; | |
992 | sharpflag = (width == 0); | |
993 | sign = 0; | |
994 | num = (uintptr_t)va_arg(ap, void *); | |
995 | goto number; | |
996 | case 'q': | |
997 | qflag = 1; | |
998 | goto reswitch; | |
999 | case 'r': | |
1000 | base = radix; | |
1001 | if (sign) | |
1002 | goto handle_sign; | |
1003 | goto handle_nosign; | |
1004 | case 's': | |
1005 | p = va_arg(ap, char *); | |
1006 | if (p == NULL) | |
1007 | p = "(null)"; | |
1008 | if (!dot) | |
1009 | n = (typeof(n))hibstrlen (p); | |
1010 | else | |
1011 | for (n = 0; n < dwidth && p[n]; n++) | |
1012 | continue; | |
1013 | ||
1014 | width -= n; | |
1015 | ||
1016 | if (!ladjust && width > 0) | |
1017 | while (width--) | |
1018 | PCHAR(padc); | |
1019 | while (n--) | |
1020 | PCHAR(*p++); | |
1021 | if (ladjust && width > 0) | |
1022 | while (width--) | |
1023 | PCHAR(padc); | |
1024 | break; | |
1025 | case 't': | |
1026 | tflag = 1; | |
1027 | goto reswitch; | |
1028 | case 'u': | |
1029 | base = 10; | |
1030 | goto handle_nosign; | |
1031 | case 'X': | |
1032 | upper = 1; | |
1033 | case 'x': | |
1034 | base = 16; | |
1035 | goto handle_nosign; | |
1036 | case 'y': | |
1037 | base = 16; | |
1038 | sign = 1; | |
1039 | goto handle_sign; | |
1040 | case 'z': | |
1041 | zflag = 1; | |
1042 | goto reswitch; | |
1043 | handle_nosign: | |
1044 | sign = 0; | |
1045 | if (jflag) | |
1046 | num = va_arg(ap, uintmax_t); | |
1047 | else if (qflag) | |
1048 | num = va_arg(ap, u_quad_t); | |
1049 | else if (tflag) | |
1050 | num = va_arg(ap, ptrdiff_t); | |
1051 | else if (lflag) | |
1052 | num = va_arg(ap, u_long); | |
1053 | else if (zflag) | |
1054 | num = va_arg(ap, size_t); | |
1055 | else if (hflag) | |
1056 | num = (u_short)va_arg(ap, int); | |
1057 | else if (cflag) | |
1058 | num = (u_char)va_arg(ap, int); | |
1059 | else | |
1060 | num = va_arg(ap, u_int); | |
1061 | goto number; | |
1062 | handle_sign: | |
1063 | if (jflag) | |
1064 | num = va_arg(ap, intmax_t); | |
1065 | else if (qflag) | |
1066 | num = va_arg(ap, quad_t); | |
1067 | else if (tflag) | |
1068 | num = va_arg(ap, ptrdiff_t); | |
1069 | else if (lflag) | |
1070 | num = va_arg(ap, long); | |
1071 | else if (zflag) | |
1072 | num = va_arg(ap, ssize_t); | |
1073 | else if (hflag) | |
1074 | num = (short)va_arg(ap, int); | |
1075 | else if (cflag) | |
1076 | num = (char)va_arg(ap, int); | |
1077 | else | |
1078 | num = va_arg(ap, int); | |
1079 | number: | |
1080 | if (sign && (intmax_t)num < 0) { | |
1081 | neg = 1; | |
1082 | num = -(intmax_t)num; | |
1083 | } | |
1084 | p = ksprintn(nbuf, num, base, &tmp, upper); | |
1085 | if (sharpflag && num != 0) { | |
1086 | if (base == 8) | |
1087 | tmp++; | |
1088 | else if (base == 16) | |
1089 | tmp += 2; | |
1090 | } | |
1091 | if (neg) | |
1092 | tmp++; | |
1093 | ||
1094 | if (!ladjust && padc != '0' && width | |
1095 | && (width -= tmp) > 0) | |
1096 | while (width--) | |
1097 | PCHAR(padc); | |
1098 | if (neg) | |
1099 | PCHAR('-'); | |
1100 | if (sharpflag && num != 0) { | |
1101 | if (base == 8) { | |
1102 | PCHAR('0'); | |
1103 | } else if (base == 16) { | |
1104 | PCHAR('0'); | |
1105 | PCHAR('x'); | |
1106 | } | |
1107 | } | |
1108 | if (!ladjust && width && (width -= tmp) > 0) | |
1109 | while (width--) | |
1110 | PCHAR(padc); | |
1111 | ||
1112 | while (*p) | |
1113 | PCHAR(*p--); | |
1114 | ||
1115 | if (ladjust && width && (width -= tmp) > 0) | |
1116 | while (width--) | |
1117 | PCHAR(padc); | |
1118 | ||
1119 | break; | |
1120 | default: | |
1121 | while (percent < fmt) | |
1122 | PCHAR(*percent++); | |
1123 | /* | |
1124 | * Since we ignore an formatting argument it is no | |
1125 | * longer safe to obey the remaining formatting | |
1126 | * arguments as the arguments will no longer match | |
1127 | * the format specs. | |
1128 | */ | |
1129 | stop = 1; | |
1130 | break; | |
1131 | } | |
1132 | } | |
1133 | #undef PCHAR | |
1134 | } | |
1135 | ||
1136 | ||
1137 | static void | |
1138 | putchar(int c, void *arg) | |
1139 | { | |
1140 | (void)arg; | |
1141 | uart_putc(c); | |
1142 | } | |
1143 | ||
1144 | void | |
1145 | hibprintf(const char *fmt, ...) | |
1146 | { | |
1147 | /* http://www.pagetable.com/?p=298 */ | |
1148 | va_list ap; | |
1149 | ||
1150 | va_start(ap, fmt); | |
1151 | hibkvprintf(fmt, putchar, NULL, 10, ap); | |
1152 | va_end(ap); | |
1153 | } | |
1154 | #endif /* CONFIG_DEBUG */ | |
1155 |