]> git.saurik.com Git - apple/xnu.git/blob - osfmk/i386/trap.c
projects / apple / xnu.git / blob
? search:


55be4fc7585cced87da54cf6238ed6e7fbce11a5
[apple/xnu.git] / osfmk / i386 / trap.c
1 /*
2 * Copyright (c) 2000-2010 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58
59 /*
60 * Hardware trap/fault handler.
61 */
62
63 #include <mach_kdb.h>
64 #include <mach_kgdb.h>
65 #include <mach_kdp.h>
66 #include <mach_ldebug.h>
67
68 #include <types.h>
69 #include <i386/eflags.h>
70 #include <i386/trap.h>
71 #include <i386/pmap.h>
72 #include <i386/fpu.h>
73 #include <i386/misc_protos.h> /* panic_io_port_read() */
74 #include <i386/lapic.h>
75
76 #include <mach/exception.h>
77 #include <mach/kern_return.h>
78 #include <mach/vm_param.h>
79 #include <mach/i386/thread_status.h>
80
81 #include <vm/vm_kern.h>
82 #include <vm/vm_fault.h>
83
84 #include <kern/kern_types.h>
85 #include <kern/processor.h>
86 #include <kern/thread.h>
87 #include <kern/task.h>
88 #include <kern/sched.h>
89 #include <kern/sched_prim.h>
90 #include <kern/exception.h>
91 #include <kern/spl.h>
92 #include <kern/misc_protos.h>
93 #include <kern/debug.h>
94
95 #include <sys/kdebug.h>
96
97 #if MACH_KGDB
98 #include <kgdb/kgdb_defs.h>
99 #endif /* MACH_KGDB */
100
101 #if MACH_KDB
102 #include <debug.h>
103 #include <ddb/db_watch.h>
104 #include <ddb/db_run.h>
105 #include <ddb/db_break.h>
106 #include <ddb/db_trap.h>
107 #endif /* MACH_KDB */
108
109 #include <string.h>
110
111 #include <i386/postcode.h>
112 #include <i386/mp_desc.h>
113 #include <i386/proc_reg.h>
114 #if CONFIG_MCA
115 #include <i386/machine_check.h>
116 #endif
117 #include <mach/i386/syscall_sw.h>
118
119 #include <libkern/OSDebug.h>
120
121 #include <machine/pal_routines.h>
122
123 extern void throttle_lowpri_io(int);
124 extern void kprint_state(x86_saved_state64_t *saved_state);
125
126 /*
127 * Forward declarations
128 */
129 static void user_page_fault_continue(kern_return_t kret);
130 #ifdef __i386__
131 static void panic_trap(x86_saved_state32_t *saved_state);
132 static void set_recovery_ip(x86_saved_state32_t *saved_state, vm_offset_t ip);
133 extern void panic_64(x86_saved_state_t *, int, const char *, boolean_t);
134 #else
135 static void panic_trap(x86_saved_state64_t *saved_state);
136 static void set_recovery_ip(x86_saved_state64_t *saved_state, vm_offset_t ip);
137 #endif
138
139 volatile perfCallback perfTrapHook = NULL; /* Pointer to CHUD trap hook routine */
140
141 #if CONFIG_DTRACE
142 /* See <rdar://problem/4613924> */
143 perfCallback tempDTraceTrapHook = NULL; /* Pointer to DTrace fbt trap hook routine */
144
145 extern boolean_t dtrace_tally_fault(user_addr_t);
146 #endif
147
148 void
149 thread_syscall_return(
150 kern_return_t ret)
151 {
152 thread_t thr_act = current_thread();
153 boolean_t is_mach;
154 int code;
155
156 pal_register_cache_state(thr_act, DIRTY);
157
158 if (thread_is_64bit(thr_act)) {
159 x86_saved_state64_t *regs;
160
161 regs = USER_REGS64(thr_act);
162
163 code = (int) (regs->rax & SYSCALL_NUMBER_MASK);
164 is_mach = (regs->rax & SYSCALL_CLASS_MASK)
165 == (SYSCALL_CLASS_MACH << SYSCALL_CLASS_SHIFT);
166 if (kdebug_enable && is_mach) {
167 /* Mach trap */
168 KERNEL_DEBUG_CONSTANT(
169 MACHDBG_CODE(DBG_MACH_EXCP_SC,code)|DBG_FUNC_END,
170 ret, 0, 0, 0, 0);
171 }
172 regs->rax = ret;
173 #if DEBUG
174 if (is_mach)
175 DEBUG_KPRINT_SYSCALL_MACH(
176 "thread_syscall_return: 64-bit mach ret=%u\n",
177 ret);
178 else
179 DEBUG_KPRINT_SYSCALL_UNIX(
180 "thread_syscall_return: 64-bit unix ret=%u\n",
181 ret);
182 #endif
183 } else {
184 x86_saved_state32_t *regs;
185
186 regs = USER_REGS32(thr_act);
187
188 code = ((int) regs->eax);
189 is_mach = (code < 0);
190 if (kdebug_enable && is_mach) {
191 /* Mach trap */
192 KERNEL_DEBUG_CONSTANT(
193 MACHDBG_CODE(DBG_MACH_EXCP_SC,-code)|DBG_FUNC_END,
194 ret, 0, 0, 0, 0);
195 }
196 regs->eax = ret;
197 #if DEBUG
198 if (is_mach)
199 DEBUG_KPRINT_SYSCALL_MACH(
200 "thread_syscall_return: 32-bit mach ret=%u\n",
201 ret);
202 else
203 DEBUG_KPRINT_SYSCALL_UNIX(
204 "thread_syscall_return: 32-bit unix ret=%u\n",
205 ret);
206 #endif
207 }
208 throttle_lowpri_io(TRUE);
209
210 thread_exception_return();
211 /*NOTREACHED*/
212 }
213
214
215 #if MACH_KDB
216 boolean_t debug_all_traps_with_kdb = FALSE;
217 extern struct db_watchpoint *db_watchpoint_list;
218 extern boolean_t db_watchpoints_inserted;
219 extern boolean_t db_breakpoints_inserted;
220
221 void
222 thread_kdb_return(void)
223 {
224 thread_t thr_act = current_thread();
225 x86_saved_state_t *iss = USER_STATE(thr_act);
226
227 pal_register_cache_state(thr_act, DIRTY);
228
229 if (is_saved_state64(iss)) {
230 x86_saved_state64_t *regs;
231
232 regs = saved_state64(iss);
233
234 if (kdb_trap(regs->isf.trapno, (int)regs->isf.err, (void *)regs)) {
235 thread_exception_return();
236 /*NOTREACHED*/
237 }
238
239 } else {
240 x86_saved_state32_t *regs;
241
242 regs = saved_state32(iss);
243
244 if (kdb_trap(regs->trapno, regs->err, (void *)regs)) {
245 thread_exception_return();
246 /*NOTREACHED*/
247 }
248 }
249 }
250
251 #endif /* MACH_KDB */
252
253 static inline void
254 user_page_fault_continue(
255 kern_return_t kr)
256 {
257 thread_t thread = current_thread();
258 user_addr_t vaddr;
259
260 #if MACH_KDB
261 x86_saved_state_t *regs = USER_STATE(thread);
262 int err;
263 int trapno;
264
265 assert((is_saved_state32(regs) && !thread_is_64bit(thread)) ||
266 (is_saved_state64(regs) && thread_is_64bit(thread)));
267 #endif
268
269 if (thread_is_64bit(thread)) {
270 x86_saved_state64_t *uregs;
271
272 uregs = USER_REGS64(thread);
273
274 #if MACH_KDB
275 trapno = uregs->isf.trapno;
276 err = (int)uregs->isf.err;
277 #endif
278 vaddr = (user_addr_t)uregs->cr2;
279 } else {
280 x86_saved_state32_t *uregs;
281
282 uregs = USER_REGS32(thread);
283
284 #if MACH_KDB
285 trapno = uregs->trapno;
286 err = uregs->err;
287 #endif
288 vaddr = uregs->cr2;
289 }
290
291 if (__probable((kr == KERN_SUCCESS) || (kr == KERN_ABORTED))) {
292 #if MACH_KDB
293 if (!db_breakpoints_inserted) {
294 db_set_breakpoints();
295 }
296 if (db_watchpoint_list &&
297 db_watchpoints_inserted &&
298 (err & T_PF_WRITE) &&
299 db_find_watchpoint(thread->map,
300 (vm_offset_t)vaddr,
301 saved_state32(regs)))
302 kdb_trap(T_WATCHPOINT, 0, saved_state32(regs));
303 #endif /* MACH_KDB */
304 thread_exception_return();
305 /*NOTREACHED*/
306 }
307
308 #if MACH_KDB
309 if (debug_all_traps_with_kdb &&
310 kdb_trap(trapno, err, saved_state32(regs))) {
311 thread_exception_return();
312 /*NOTREACHED*/
313 }
314 #endif /* MACH_KDB */
315
316 /* PAL debug hook */
317 pal_dbg_page_fault( thread, vaddr, kr );
318
319 i386_exception(EXC_BAD_ACCESS, kr, vaddr);
320 /*NOTREACHED*/
321 }
322
323 /*
324 * Fault recovery in copyin/copyout routines.
325 */
326 struct recovery {
327 uintptr_t fault_addr;
328 uintptr_t recover_addr;
329 };
330
331 extern struct recovery recover_table[];
332 extern struct recovery recover_table_end[];
333
334 const char * trap_type[] = {TRAP_NAMES};
335 unsigned TRAP_TYPES = sizeof(trap_type)/sizeof(trap_type[0]);
336
337 extern void PE_incoming_interrupt(int interrupt);
338
339 #if defined(__x86_64__) && DEBUG
340 void
341 kprint_state(x86_saved_state64_t *saved_state)
342 {
343 kprintf("current_cpu_datap() 0x%lx\n", (uintptr_t)current_cpu_datap());
344 kprintf("Current GS base MSR 0x%llx\n", rdmsr64(MSR_IA32_GS_BASE));
345 kprintf("Kernel GS base MSR 0x%llx\n", rdmsr64(MSR_IA32_KERNEL_GS_BASE));
346 kprintf("state at 0x%lx:\n", (uintptr_t) saved_state);
347
348 kprintf(" rdi 0x%llx\n", saved_state->rdi);
349 kprintf(" rsi 0x%llx\n", saved_state->rsi);
350 kprintf(" rdx 0x%llx\n", saved_state->rdx);
351 kprintf(" r10 0x%llx\n", saved_state->r10);
352 kprintf(" r8 0x%llx\n", saved_state->r8);
353 kprintf(" r9 0x%llx\n", saved_state->r9);
354 kprintf(" v_arg6 0x%llx\n", saved_state->v_arg6);
355 kprintf(" v_arg7 0x%llx\n", saved_state->v_arg7);
356 kprintf(" v_arg8 0x%llx\n", saved_state->v_arg8);
357
358 kprintf(" cr2 0x%llx\n", saved_state->cr2);
359 kprintf("real cr2 0x%lx\n", get_cr2());
360 kprintf(" r15 0x%llx\n", saved_state->r15);
361 kprintf(" r14 0x%llx\n", saved_state->r14);
362 kprintf(" r13 0x%llx\n", saved_state->r13);
363 kprintf(" r12 0x%llx\n", saved_state->r12);
364 kprintf(" r11 0x%llx\n", saved_state->r11);
365 kprintf(" rbp 0x%llx\n", saved_state->rbp);
366 kprintf(" rbx 0x%llx\n", saved_state->rbx);
367 kprintf(" rcx 0x%llx\n", saved_state->rcx);
368 kprintf(" rax 0x%llx\n", saved_state->rax);
369
370 kprintf(" gs 0x%x\n", saved_state->gs);
371 kprintf(" fs 0x%x\n", saved_state->fs);
372
373 kprintf(" isf.trapno 0x%x\n", saved_state->isf.trapno);
374 kprintf(" isf._pad 0x%x\n", saved_state->isf._pad);
375 kprintf(" isf.trapfn 0x%llx\n", saved_state->isf.trapfn);
376 kprintf(" isf.err 0x%llx\n", saved_state->isf.err);
377 kprintf(" isf.rip 0x%llx\n", saved_state->isf.rip);
378 kprintf(" isf.cs 0x%llx\n", saved_state->isf.cs);
379 kprintf(" isf.rflags 0x%llx\n", saved_state->isf.rflags);
380 kprintf(" isf.rsp 0x%llx\n", saved_state->isf.rsp);
381 kprintf(" isf.ss 0x%llx\n", saved_state->isf.ss);
382 }
383 #endif
384
385
386 /*
387 * Non-zero indicates latency assert is enabled and capped at valued
388 * absolute time units.
389 */
390
391 uint64_t interrupt_latency_cap = 0;
392 boolean_t ilat_assert = FALSE;
393
394 void
395 interrupt_latency_tracker_setup(void) {
396 uint32_t ilat_cap_us;
397 if (PE_parse_boot_argn("interrupt_latency_cap_us", &ilat_cap_us, sizeof(ilat_cap_us))) {
398 interrupt_latency_cap = ilat_cap_us * NSEC_PER_USEC;
399 nanoseconds_to_absolutetime(interrupt_latency_cap, &interrupt_latency_cap);
400 } else {
401 interrupt_latency_cap = LockTimeOut;
402 }
403 PE_parse_boot_argn("-interrupt_latency_assert_enable", &ilat_assert, sizeof(ilat_assert));
404 }
405
406 void interrupt_reset_latency_stats(void) {
407 uint32_t i;
408 for (i = 0; i < real_ncpus; i++) {
409 cpu_data_ptr[i]->cpu_max_observed_int_latency =
410 cpu_data_ptr[i]->cpu_max_observed_int_latency_vector = 0;
411 }
412 }
413
414 void interrupt_populate_latency_stats(char *buf, unsigned bufsize) {
415 uint32_t i, tcpu = ~0;
416 uint64_t cur_max = 0;
417
418 for (i = 0; i < real_ncpus; i++) {
419 if (cur_max < cpu_data_ptr[i]->cpu_max_observed_int_latency) {
420 cur_max = cpu_data_ptr[i]->cpu_max_observed_int_latency;
421 tcpu = i;
422 }
423 }
424
425 if (tcpu < real_ncpus)
426 snprintf(buf, bufsize, "0x%x 0x%x 0x%llx", tcpu, cpu_data_ptr[tcpu]->cpu_max_observed_int_latency_vector, cpu_data_ptr[tcpu]->cpu_max_observed_int_latency);
427 }
428
429 /*
430 * Handle interrupts:
431 * - local APIC interrupts (IPIs, timers, etc) are handled by the kernel,
432 * - device interrupts go to the platform expert.
433 */
434 void
435 interrupt(x86_saved_state_t *state)
436 {
437 uint64_t rip;
438 uint64_t rsp;
439 int interrupt_num;
440 boolean_t user_mode = FALSE;
441 int ipl;
442 int cnum = cpu_number();
443
444 if (is_saved_state64(state) == TRUE) {
445 x86_saved_state64_t *state64;
446
447 state64 = saved_state64(state);
448 rip = state64->isf.rip;
449 rsp = state64->isf.rsp;
450 interrupt_num = state64->isf.trapno;
451 #ifdef __x86_64__
452 if(state64->isf.cs & 0x03)
453 #endif
454 user_mode = TRUE;
455 } else {
456 x86_saved_state32_t *state32;
457
458 state32 = saved_state32(state);
459 if (state32->cs & 0x03)
460 user_mode = TRUE;
461 rip = state32->eip;
462 rsp = state32->uesp;
463 interrupt_num = state32->trapno;
464 }
465
466 KERNEL_DEBUG_CONSTANT(
467 MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_START,
468 interrupt_num, rip, user_mode, 0, 0);
469
470 SCHED_STATS_INTERRUPT(current_processor());
471
472 ipl = get_preemption_level();
473
474 /*
475 * Handle local APIC interrupts
476 * else call platform expert for devices.
477 */
478 if (!lapic_interrupt(interrupt_num, state))
479 PE_incoming_interrupt(interrupt_num);
480
481 if (__improbable(get_preemption_level() != ipl)) {
482 panic("Preemption level altered by interrupt vector 0x%x: initial 0x%x, final: 0x%x\n", interrupt_num, ipl, get_preemption_level());
483 }
484
485 KERNEL_DEBUG_CONSTANT(
486 MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_END,
487 interrupt_num, 0, 0, 0, 0);
488
489 if (cpu_data_ptr[cnum]->cpu_nested_istack) {
490 cpu_data_ptr[cnum]->cpu_nested_istack_events++;
491 }
492 else {
493 uint64_t int_latency = mach_absolute_time() - cpu_data_ptr[cnum]->cpu_int_event_time;
494 if (ilat_assert && (int_latency > interrupt_latency_cap) && !machine_timeout_suspended()) {
495 panic("Interrupt vector 0x%x exceeded interrupt latency threshold, 0x%llx absolute time delta, prior signals: 0x%x, current signals: 0x%x", interrupt_num, int_latency, cpu_data_ptr[cnum]->cpu_prior_signals, cpu_data_ptr[cnum]->cpu_signals);
496 }
497 if (int_latency > cpu_data_ptr[cnum]->cpu_max_observed_int_latency) {
498 cpu_data_ptr[cnum]->cpu_max_observed_int_latency = int_latency;
499 cpu_data_ptr[cnum]->cpu_max_observed_int_latency_vector = interrupt_num;
500 }
501 }
502
503 /*
504 * Having serviced the interrupt first, look at the interrupted stack depth.
505 */
506 if (!user_mode) {
507 uint64_t depth = cpu_data_ptr[cnum]->cpu_kernel_stack
508 + sizeof(struct x86_kernel_state)
509 + sizeof(struct i386_exception_link *)
510 - rsp;
511 if (depth > kernel_stack_depth_max) {
512 kernel_stack_depth_max = (vm_offset_t)depth;
513 KERNEL_DEBUG_CONSTANT(
514 MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_DEPTH),
515 (long) depth, (long) rip, 0, 0, 0);
516 }
517 }
518 }
519
520 static inline void
521 reset_dr7(void)
522 {
523 long dr7 = 0x400; /* magic dr7 reset value; 32 bit on i386, 64 bit on x86_64 */
524 __asm__ volatile("mov %0,%%dr7" : : "r" (dr7));
525 }
526 #if MACH_KDP
527 unsigned kdp_has_active_watchpoints = 0;
528 #define NO_WATCHPOINTS (!kdp_has_active_watchpoints)
529 #else
530 #define NO_WATCHPOINTS 1
531 #endif
532 /*
533 * Trap from kernel mode. Only page-fault errors are recoverable,
534 * and then only in special circumstances. All other errors are
535 * fatal. Return value indicates if trap was handled.
536 */
537
538 void
539 kernel_trap(
540 x86_saved_state_t *state,
541 uintptr_t *lo_spp)
542 {
543 #ifdef __i386__
544 x86_saved_state32_t *saved_state;
545 #else
546 x86_saved_state64_t *saved_state;
547 #endif
548 int code;
549 user_addr_t vaddr;
550 int type;
551 vm_map_t map = 0; /* protected by T_PAGE_FAULT */
552 kern_return_t result = KERN_FAILURE;
553 thread_t thread;
554 ast_t *myast;
555 boolean_t intr;
556 vm_prot_t prot;
557 struct recovery *rp;
558 vm_offset_t kern_ip;
559 #if NCOPY_WINDOWS > 0
560 int fault_in_copy_window = -1;
561 #endif
562 int is_user = 0;
563 #if MACH_KDB
564 pt_entry_t *pte;
565 #endif /* MACH_KDB */
566
567 thread = current_thread();
568
569 #ifdef __i386__
570 if (__improbable(is_saved_state64(state))) {
571 panic_64(state, 0, "Kernel trap with 64-bit state", FALSE);
572 }
573
574 saved_state = saved_state32(state);
575
576 /* Record cpu where state was captured (trampolines don't set this) */
577 saved_state->cpu = cpu_number();
578
579 vaddr = (user_addr_t)saved_state->cr2;
580 type = saved_state->trapno;
581 code = saved_state->err & 0xffff;
582 intr = (saved_state->efl & EFL_IF) != 0; /* state of ints at trap */
583 kern_ip = (vm_offset_t)saved_state->eip;
584 #else
585 if (__improbable(is_saved_state32(state)))
586 panic("kernel_trap(%p) with 32-bit state", state);
587 saved_state = saved_state64(state);
588
589 /* Record cpu where state was captured */
590 saved_state->isf.cpu = cpu_number();
591
592 vaddr = (user_addr_t)saved_state->cr2;
593 type = saved_state->isf.trapno;
594 code = (int)(saved_state->isf.err & 0xffff);
595 intr = (saved_state->isf.rflags & EFL_IF) != 0; /* state of ints at trap */
596 kern_ip = (vm_offset_t)saved_state->isf.rip;
597 #endif
598
599 myast = ast_pending();
600
601 perfASTCallback astfn = perfASTHook;
602 if (__improbable(astfn != NULL)) {
603 if (*myast & AST_CHUD_ALL)
604 astfn(AST_CHUD_ALL, myast);
605 } else
606 *myast &= ~AST_CHUD_ALL;
607
608 /*
609 * Is there a hook?
610 */
611 perfCallback fn = perfTrapHook;
612 if (__improbable(fn != NULL)) {
613 if (fn(type, NULL, 0, 0) == KERN_SUCCESS) {
614 /*
615 * If it succeeds, we are done...
616 */
617 return;
618 }
619 }
620
621 #if CONFIG_DTRACE
622 if (__improbable(tempDTraceTrapHook != NULL)) {
623 if (tempDTraceTrapHook(type, state, lo_spp, 0) == KERN_SUCCESS) {
624 /*
625 * If it succeeds, we are done...
626 */
627 return;
628 }
629 }
630 #endif /* CONFIG_DTRACE */
631
632 /*
633 * we come here with interrupts off as we don't want to recurse
634 * on preemption below. but we do want to re-enable interrupts
635 * as soon we possibly can to hold latency down
636 */
637 if (__improbable(T_PREEMPT == type)) {
638 ast_taken(AST_PREEMPTION, FALSE);
639
640 KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_EXCP_KTRAP_x86, type)) | DBG_FUNC_NONE,
641 0, 0, 0, kern_ip, 0);
642 return;
643 }
644
645 if (T_PAGE_FAULT == type) {
646 /*
647 * assume we're faulting in the kernel map
648 */
649 map = kernel_map;
650
651 if (__probable(thread != THREAD_NULL && thread->map != kernel_map)) {
652 #if NCOPY_WINDOWS > 0
653 vm_offset_t copy_window_base;
654 vm_offset_t kvaddr;
655 int window_index;
656
657 kvaddr = (vm_offset_t)vaddr;
658 /*
659 * must determine if fault occurred in
660 * the copy window while pre-emption is
661 * disabled for this processor so that
662 * we only need to look at the window
663 * associated with this processor
664 */
665 copy_window_base = current_cpu_datap()->cpu_copywindow_base;
666
667 if (kvaddr >= copy_window_base && kvaddr < (copy_window_base + (NBPDE * NCOPY_WINDOWS)) ) {
668
669 window_index = (int)((kvaddr - copy_window_base) / NBPDE);
670
671 if (thread->machine.copy_window[window_index].user_base != (user_addr_t)-1) {
672
673 kvaddr -= (copy_window_base + (NBPDE * window_index));
674 vaddr = thread->machine.copy_window[window_index].user_base + kvaddr;
675
676 map = thread->map;
677 fault_in_copy_window = window_index;
678 }
679 is_user = -1;
680 }
681 #else
682 if (vaddr < VM_MAX_USER_PAGE_ADDRESS) {
683 /* fault occurred in userspace */
684 map = thread->map;
685 is_user = -1;
686 /*
687 * If we're not sharing cr3 with the user
688 * and we faulted in copyio,
689 * then switch cr3 here and dismiss the fault.
690 */
691 if (no_shared_cr3 &&
692 (thread->machine.specFlags&CopyIOActive) &&
693 map->pmap->pm_cr3 != get_cr3_base()) {
694 pmap_assert(current_cpu_datap()->cpu_pmap_pcid_enabled == FALSE);
695 set_cr3_raw(map->pmap->pm_cr3);
696 return;
697 }
698 }
699 #endif
700 }
701 }
702 KERNEL_DEBUG_CONSTANT(
703 (MACHDBG_CODE(DBG_MACH_EXCP_KTRAP_x86, type)) | DBG_FUNC_NONE,
704 (unsigned)(vaddr >> 32), (unsigned)vaddr, is_user, kern_ip, 0);
705
706
707 (void) ml_set_interrupts_enabled(intr);
708
709 switch (type) {
710
711 case T_NO_FPU:
712 fpnoextflt();
713 return;
714
715 case T_FPU_FAULT:
716 fpextovrflt();
717 return;
718
719 case T_FLOATING_POINT_ERROR:
720 fpexterrflt();
721 return;
722
723 case T_SSE_FLOAT_ERROR:
724 fpSSEexterrflt();
725 return;
726 case T_DEBUG:
727 #ifdef __i386__
728 if ((saved_state->efl & EFL_TF) == 0 && NO_WATCHPOINTS)
729 #else
730 if ((saved_state->isf.rflags & EFL_TF) == 0 && NO_WATCHPOINTS)
731 #endif
732 {
733 /* We've somehow encountered a debug
734 * register match that does not belong
735 * to the kernel debugger.
736 * This isn't supposed to happen.
737 */
738 reset_dr7();
739 return;
740 }
741 goto debugger_entry;
742 #ifdef __x86_64__
743 case T_INT3:
744 goto debugger_entry;
745 #endif
746 case T_PAGE_FAULT:
747 /*
748 * If the current map is a submap of the kernel map,
749 * and the address is within that map, fault on that
750 * map. If the same check is done in vm_fault
751 * (vm_map_lookup), we may deadlock on the kernel map
752 * lock.
753 */
754
755 prot = VM_PROT_READ;
756
757 if (code & T_PF_WRITE)
758 prot |= VM_PROT_WRITE;
759 #if PAE
760 if (code & T_PF_EXECUTE)
761 prot |= VM_PROT_EXECUTE;
762 #endif
763
764 #if MACH_KDB
765 /*
766 * Check for watchpoint on kernel static data.
767 * vm_fault would fail in this case
768 */
769 if (map == kernel_map && db_watchpoint_list && db_watchpoints_inserted &&
770 (code & T_PF_WRITE) && vaddr < vm_map_max(map) &&
771 ((*(pte = pmap_pte(kernel_pmap, (vm_map_offset_t)vaddr))) & INTEL_PTE_WRITE) == 0) {
772 pmap_store_pte(
773 pte,
774 *pte | INTEL_PTE_VALID | INTEL_PTE_WRITE);
775 /* XXX need invltlb here? */
776
777 result = KERN_SUCCESS;
778 goto look_for_watchpoints;
779 }
780 #endif /* MACH_KDB */
781
782 #if CONFIG_DTRACE
783 if (thread != THREAD_NULL && thread->options & TH_OPT_DTRACE) { /* Executing under dtrace_probe? */
784 if (dtrace_tally_fault(vaddr)) { /* Should a fault under dtrace be ignored? */
785 /*
786 * DTrace has "anticipated" the possibility of this fault, and has
787 * established the suitable recovery state. Drop down now into the
788 * recovery handling code in "case T_GENERAL_PROTECTION:".
789 */
790 goto FALL_THROUGH;
791 }
792 }
793 #endif /* CONFIG_DTRACE */
794
795 result = vm_fault(map,
796 vm_map_trunc_page(vaddr),
797 prot,
798 FALSE,
799 THREAD_UNINT, NULL, 0);
800
801 #if MACH_KDB
802 if (result == KERN_SUCCESS) {
803 /*
804 * Look for watchpoints
805 */
806 look_for_watchpoints:
807 if (map == kernel_map && db_watchpoint_list && db_watchpoints_inserted && (code & T_PF_WRITE) &&
808 db_find_watchpoint(map, vaddr, saved_state))
809 kdb_trap(T_WATCHPOINT, 0, saved_state);
810 }
811 #endif /* MACH_KDB */
812
813 if (result == KERN_SUCCESS) {
814 #if NCOPY_WINDOWS > 0
815 if (fault_in_copy_window != -1) {
816 ml_set_interrupts_enabled(FALSE);
817 copy_window_fault(thread, map,
818 fault_in_copy_window);
819 (void) ml_set_interrupts_enabled(intr);
820 }
821 #endif /* NCOPY_WINDOWS > 0 */
822 return;
823 }
824 /*
825 * fall through
826 */
827 #if CONFIG_DTRACE
828 FALL_THROUGH:
829 #endif /* CONFIG_DTRACE */
830
831 case T_GENERAL_PROTECTION:
832 /*
833 * If there is a failure recovery address
834 * for this fault, go there.
835 */
836 for (rp = recover_table; rp < recover_table_end; rp++) {
837 if (kern_ip == rp->fault_addr) {
838 set_recovery_ip(saved_state, rp->recover_addr);
839 return;
840 }
841 }
842
843 /*
844 * Check thread recovery address also.
845 */
846 if (thread != THREAD_NULL && thread->recover) {
847 set_recovery_ip(saved_state, thread->recover);
848 thread->recover = 0;
849 return;
850 }
851 /*
852 * Unanticipated page-fault errors in kernel
853 * should not happen.
854 *
855 * fall through...
856 */
857 default:
858 /*
859 * Exception 15 is reserved but some chips may generate it
860 * spuriously. Seen at startup on AMD Athlon-64.
861 */
862 if (type == 15) {
863 kprintf("kernel_trap() ignoring spurious trap 15\n");
864 return;
865 }
866 #if defined(__x86_64__) && DEBUG
867 kprint_state(saved_state);
868 #endif
869 debugger_entry:
870 /* Ensure that the i386_kernel_state at the base of the
871 * current thread's stack (if any) is synchronized with the
872 * context at the moment of the trap, to facilitate
873 * access through the debugger.
874 */
875 sync_iss_to_iks(state);
876 #if MACH_KDB
877 restart_debugger:
878 #endif /* MACH_KDB */
879 #if MACH_KDP
880 if (current_debugger != KDB_CUR_DB) {
881 if (kdp_i386_trap(type, saved_state, result, (vm_offset_t)vaddr))
882 return;
883 } else {
884 #endif /* MACH_KDP */
885 #if MACH_KDB
886 if (kdb_trap(type, code, saved_state)) {
887 if (switch_debugger) {
888 current_debugger = KDP_CUR_DB;
889 switch_debugger = 0;
890 goto restart_debugger;
891 }
892 return;
893 }
894 #endif /* MACH_KDB */
895 #if MACH_KDP
896 }
897 #endif
898 }
899 __asm__ volatile("cli":::"cc");
900 panic_trap(saved_state);
901 /*
902 * NO RETURN
903 */
904 }
905
906
907 #ifdef __i386__
908 static void
909 set_recovery_ip(x86_saved_state32_t *saved_state, vm_offset_t ip)
910 {
911 saved_state->eip = ip;
912 }
913 #else
914 static void
915 set_recovery_ip(x86_saved_state64_t *saved_state, vm_offset_t ip)
916 {
917 saved_state->isf.rip = ip;
918 }
919 #endif
920
921
922 #ifdef __i386__
923 static void
924 panic_trap(x86_saved_state32_t *regs)
925 {
926 const char *trapname = "Unknown";
927 pal_cr_t cr0, cr2, cr3, cr4;
928
929 pal_get_control_registers( &cr0, &cr2, &cr3, &cr4 );
930
931 /*
932 * Issue an I/O port read if one has been requested - this is an
933 * event logic analyzers can use as a trigger point.
934 */
935 panic_io_port_read();
936
937 kprintf("panic trap number 0x%x, eip 0x%x\n", regs->trapno, regs->eip);
938 kprintf("cr0 0x%08x cr2 0x%08x cr3 0x%08x cr4 0x%08x\n",
939 cr0, cr2, cr3, cr4);
940
941 if (regs->trapno < TRAP_TYPES)
942 trapname = trap_type[regs->trapno];
943 #undef panic
944 panic("Kernel trap at 0x%08x, type %d=%s, registers:\n"
945 "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n"
946 "EAX: 0x%08x, EBX: 0x%08x, ECX: 0x%08x, EDX: 0x%08x\n"
947 "CR2: 0x%08x, EBP: 0x%08x, ESI: 0x%08x, EDI: 0x%08x\n"
948 "EFL: 0x%08x, EIP: 0x%08x, CS: 0x%08x, DS: 0x%08x\n"
949 "Error code: 0x%08x\n",
950 regs->eip, regs->trapno, trapname, cr0, cr2, cr3, cr4,
951 regs->eax,regs->ebx,regs->ecx,regs->edx,
952 regs->cr2,regs->ebp,regs->esi,regs->edi,
953 regs->efl,regs->eip,regs->cs & 0xFFFF, regs->ds & 0xFFFF, regs->err);
954 /*
955 * This next statement is not executed,
956 * but it's needed to stop the compiler using tail call optimization
957 * for the panic call - which confuses the subsequent backtrace.
958 */
959 cr0 = 0;
960 }
961 #else
962 static void
963 panic_trap(x86_saved_state64_t *regs)
964 {
965 const char *trapname = "Unknown";
966 pal_cr_t cr0, cr2, cr3, cr4;
967
968 pal_get_control_registers( &cr0, &cr2, &cr3, &cr4 );
969 assert(ml_get_interrupts_enabled() == FALSE);
970 current_cpu_datap()->cpu_fatal_trap_state = regs;
971 /*
972 * Issue an I/O port read if one has been requested - this is an
973 * event logic analyzers can use as a trigger point.
974 */
975 panic_io_port_read();
976
977 kprintf("panic trap number 0x%x, rip 0x%016llx\n",
978 regs->isf.trapno, regs->isf.rip);
979 kprintf("cr0 0x%016llx cr2 0x%016llx cr3 0x%016llx cr4 0x%016llx\n",
980 cr0, cr2, cr3, cr4);
981
982 if (regs->isf.trapno < TRAP_TYPES)
983 trapname = trap_type[regs->isf.trapno];
984 #undef panic
985 panic("Kernel trap at 0x%016llx, type %d=%s, registers:\n"
986 "CR0: 0x%016llx, CR2: 0x%016llx, CR3: 0x%016llx, CR4: 0x%016llx\n"
987 "RAX: 0x%016llx, RBX: 0x%016llx, RCX: 0x%016llx, RDX: 0x%016llx\n"
988 "RSP: 0x%016llx, RBP: 0x%016llx, RSI: 0x%016llx, RDI: 0x%016llx\n"
989 "R8: 0x%016llx, R9: 0x%016llx, R10: 0x%016llx, R11: 0x%016llx\n"
990 "R12: 0x%016llx, R13: 0x%016llx, R14: 0x%016llx, R15: 0x%016llx\n"
991 "RFL: 0x%016llx, RIP: 0x%016llx, CS: 0x%016llx, SS: 0x%016llx\n"
992 "CR2: 0x%016llx, Error code: 0x%016llx, Faulting CPU: 0x%x\n",
993 regs->isf.rip, regs->isf.trapno, trapname,
994 cr0, cr2, cr3, cr4,
995 regs->rax, regs->rbx, regs->rcx, regs->rdx,
996 regs->isf.rsp, regs->rbp, regs->rsi, regs->rdi,
997 regs->r8, regs->r9, regs->r10, regs->r11,
998 regs->r12, regs->r13, regs->r14, regs->r15,
999 regs->isf.rflags, regs->isf.rip, regs->isf.cs & 0xFFFF,
1000 regs->isf.ss & 0xFFFF,regs->cr2, regs->isf.err, regs->isf.cpu);
1001 /*
1002 * This next statement is not executed,
1003 * but it's needed to stop the compiler using tail call optimization
1004 * for the panic call - which confuses the subsequent backtrace.
1005 */
1006 cr0 = 0;
1007 }
1008 #endif
1009
1010 #if CONFIG_DTRACE
1011 extern kern_return_t dtrace_user_probe(x86_saved_state_t *);
1012 #endif
1013
1014 /*
1015 * Trap from user mode.
1016 */
1017 void
1018 user_trap(
1019 x86_saved_state_t *saved_state)
1020 {
1021 int exc;
1022 int err;
1023 mach_exception_code_t code;
1024 mach_exception_subcode_t subcode;
1025 int type;
1026 user_addr_t vaddr;
1027 vm_prot_t prot;
1028 thread_t thread = current_thread();
1029 ast_t *myast;
1030 kern_return_t kret;
1031 user_addr_t rip;
1032 unsigned long dr6 = 0; /* 32 bit for i386, 64 bit for x86_64 */
1033
1034 assert((is_saved_state32(saved_state) && !thread_is_64bit(thread)) ||
1035 (is_saved_state64(saved_state) && thread_is_64bit(thread)));
1036
1037 if (is_saved_state64(saved_state)) {
1038 x86_saved_state64_t *regs;
1039
1040 regs = saved_state64(saved_state);
1041
1042 /* Record cpu where state was captured */
1043 regs->isf.cpu = cpu_number();
1044
1045 type = regs->isf.trapno;
1046 err = (int)regs->isf.err & 0xffff;
1047 vaddr = (user_addr_t)regs->cr2;
1048 rip = (user_addr_t)regs->isf.rip;
1049 } else {
1050 x86_saved_state32_t *regs;
1051
1052 regs = saved_state32(saved_state);
1053
1054 /* Record cpu where state was captured */
1055 regs->cpu = cpu_number();
1056
1057 type = regs->trapno;
1058 err = regs->err & 0xffff;
1059 vaddr = (user_addr_t)regs->cr2;
1060 rip = (user_addr_t)regs->eip;
1061 }
1062
1063 if ((type == T_DEBUG) && thread->machine.ids) {
1064 unsigned long clear = 0;
1065 /* Stash and clear this processor's DR6 value, in the event
1066 * this was a debug register match
1067 */
1068 __asm__ volatile ("mov %%db6, %0" : "=r" (dr6));
1069 __asm__ volatile ("mov %0, %%db6" : : "r" (clear));
1070 }
1071
1072 pal_sti();
1073
1074 KERNEL_DEBUG_CONSTANT(
1075 (MACHDBG_CODE(DBG_MACH_EXCP_UTRAP_x86, type)) | DBG_FUNC_NONE,
1076 (unsigned)(vaddr>>32), (unsigned)vaddr,
1077 (unsigned)(rip>>32), (unsigned)rip, 0);
1078
1079 code = 0;
1080 subcode = 0;
1081 exc = 0;
1082
1083 #if DEBUG_TRACE
1084 kprintf("user_trap(0x%08x) type=%d vaddr=0x%016llx\n",
1085 saved_state, type, vaddr);
1086 #endif
1087
1088 perfASTCallback astfn = perfASTHook;
1089 if (__improbable(astfn != NULL)) {
1090 myast = ast_pending();
1091 if (*myast & AST_CHUD_ALL) {
1092 astfn(AST_CHUD_ALL, myast);
1093 }
1094 }
1095
1096 /* Is there a hook? */
1097 perfCallback fn = perfTrapHook;
1098 if (__improbable(fn != NULL)) {
1099 if (fn(type, saved_state, 0, 0) == KERN_SUCCESS)
1100 return; /* If it succeeds, we are done... */
1101 }
1102
1103 /*
1104 * DTrace does not consume all user traps, only INT_3's for now.
1105 * Avoid needlessly calling tempDTraceTrapHook here, and let the
1106 * INT_3 case handle them.
1107 */
1108 DEBUG_KPRINT_SYSCALL_MASK(1,
1109 "user_trap: type=0x%x(%s) err=0x%x cr2=%p rip=%p\n",
1110 type, trap_type[type], err, (void *)(long) vaddr, (void *)(long) rip);
1111
1112 switch (type) {
1113
1114 case T_DIVIDE_ERROR:
1115 exc = EXC_ARITHMETIC;
1116 code = EXC_I386_DIV;
1117 break;
1118
1119 case T_DEBUG:
1120 {
1121 pcb_t pcb;
1122 /*
1123 * Update the PCB with this processor's DR6 value
1124 * in the event this was a debug register match.
1125 */
1126 pcb = THREAD_TO_PCB(thread);
1127 if (pcb->ids) {
1128 /*
1129 * We can get and set the status register
1130 * in 32-bit mode even on a 64-bit thread
1131 * because the high order bits are not
1132 * used on x86_64
1133 */
1134 if (thread_is_64bit(thread)) {
1135 x86_debug_state64_t *ids = pcb->ids;
1136 ids->dr6 = dr6;
1137 } else { /* 32 bit thread */
1138 x86_debug_state32_t *ids = pcb->ids;
1139 ids->dr6 = (uint32_t) dr6;
1140 }
1141 }
1142 exc = EXC_BREAKPOINT;
1143 code = EXC_I386_SGL;
1144 break;
1145 }
1146 case T_INT3:
1147 #if CONFIG_DTRACE
1148 if (dtrace_user_probe(saved_state) == KERN_SUCCESS)
1149 return; /* If it succeeds, we are done... */
1150 #endif
1151 exc = EXC_BREAKPOINT;
1152 code = EXC_I386_BPT;
1153 break;
1154
1155 case T_OVERFLOW:
1156 exc = EXC_ARITHMETIC;
1157 code = EXC_I386_INTO;
1158 break;
1159
1160 case T_OUT_OF_BOUNDS:
1161 exc = EXC_SOFTWARE;
1162 code = EXC_I386_BOUND;
1163 break;
1164
1165 case T_INVALID_OPCODE:
1166 exc = EXC_BAD_INSTRUCTION;
1167 code = EXC_I386_INVOP;
1168 break;
1169
1170 case T_NO_FPU:
1171 fpnoextflt();
1172 return;
1173
1174 case T_FPU_FAULT:
1175 fpextovrflt(); /* Propagates exception directly, doesn't return */
1176 return;
1177
1178 case T_INVALID_TSS: /* invalid TSS == iret with NT flag set */
1179 exc = EXC_BAD_INSTRUCTION;
1180 code = EXC_I386_INVTSSFLT;
1181 subcode = err;
1182 break;
1183
1184 case T_SEGMENT_NOT_PRESENT:
1185 exc = EXC_BAD_INSTRUCTION;
1186 code = EXC_I386_SEGNPFLT;
1187 subcode = err;
1188 break;
1189
1190 case T_STACK_FAULT:
1191 exc = EXC_BAD_INSTRUCTION;
1192 code = EXC_I386_STKFLT;
1193 subcode = err;
1194 break;
1195
1196 case T_GENERAL_PROTECTION:
1197 /*
1198 * There's a wide range of circumstances which generate this
1199 * class of exception. From user-space, many involve bad
1200 * addresses (such as a non-canonical 64-bit address).
1201 * So we map this to EXC_BAD_ACCESS (and thereby SIGSEGV).
1202 * The trouble is cr2 doesn't contain the faulting address;
1203 * we'd need to decode the faulting instruction to really
1204 * determine this. We'll leave that to debuggers.
1205 * However, attempted execution of privileged instructions
1206 * (e.g. cli) also generate GP faults and so we map these to
1207 * to EXC_BAD_ACCESS (and thence SIGSEGV) also - rather than
1208 * EXC_BAD_INSTRUCTION which is more accurate. We just can't
1209 * win!
1210 */
1211 exc = EXC_BAD_ACCESS;
1212 code = EXC_I386_GPFLT;
1213 subcode = err;
1214 break;
1215
1216 case T_PAGE_FAULT:
1217 prot = VM_PROT_READ;
1218
1219 if (err & T_PF_WRITE)
1220 prot |= VM_PROT_WRITE;
1221 #if PAE
1222 if (__improbable(err & T_PF_EXECUTE))
1223 prot |= VM_PROT_EXECUTE;
1224 #endif
1225 kret = vm_fault(thread->map, vm_map_trunc_page(vaddr),
1226 prot, FALSE,
1227 THREAD_ABORTSAFE, NULL, 0);
1228
1229 user_page_fault_continue(kret);
1230
1231 /* NOTREACHED */
1232 break;
1233
1234 case T_SSE_FLOAT_ERROR:
1235 fpSSEexterrflt(); /* Propagates exception directly, doesn't return */
1236 return;
1237
1238
1239 case T_FLOATING_POINT_ERROR:
1240 fpexterrflt(); /* Propagates exception directly, doesn't return */
1241 return;
1242
1243 case T_DTRACE_RET:
1244 #if CONFIG_DTRACE
1245 if (dtrace_user_probe(saved_state) == KERN_SUCCESS)
1246 return; /* If it succeeds, we are done... */
1247 #endif
1248 /*
1249 * If we get an INT 0x7f when we do not expect to,
1250 * treat it as an illegal instruction
1251 */
1252 exc = EXC_BAD_INSTRUCTION;
1253 code = EXC_I386_INVOP;
1254 break;
1255
1256 default:
1257 #if MACH_KGDB
1258 Debugger("Unanticipated user trap");
1259 return;
1260 #endif /* MACH_KGDB */
1261 #if MACH_KDB
1262 if (kdb_trap(type, err, saved_state32(saved_state)))
1263 return;
1264 #endif /* MACH_KDB */
1265 panic("Unexpected user trap, type %d", type);
1266 return;
1267 }
1268 /* Note: Codepaths that directly return from user_trap() have pending
1269 * ASTs processed in locore
1270 */
1271 i386_exception(exc, code, subcode);
1272 /* NOTREACHED */
1273 }
1274
1275
1276 /*
1277 * Handle AST traps for i386.
1278 */
1279
1280 extern void log_thread_action (thread_t, char *);
1281
1282 void
1283 i386_astintr(int preemption)
1284 {
1285 ast_t mask = AST_ALL;
1286 spl_t s;
1287
1288 if (preemption)
1289 mask = AST_PREEMPTION;
1290
1291 s = splsched();
1292
1293 ast_taken(mask, s);
1294
1295 splx(s);
1296 }
1297
1298 /*
1299 * Handle exceptions for i386.
1300 *
1301 * If we are an AT bus machine, we must turn off the AST for a
1302 * delayed floating-point exception.
1303 *
1304 * If we are providing floating-point emulation, we may have
1305 * to retrieve the real register values from the floating point
1306 * emulator.
1307 */
1308 void
1309 i386_exception(
1310 int exc,
1311 mach_exception_code_t code,
1312 mach_exception_subcode_t subcode)
1313 {
1314 mach_exception_data_type_t codes[EXCEPTION_CODE_MAX];
1315
1316 DEBUG_KPRINT_SYSCALL_MACH("i386_exception: exc=%d code=0x%llx subcode=0x%llx\n",
1317 exc, code, subcode);
1318 codes[0] = code; /* new exception interface */
1319 codes[1] = subcode;
1320 exception_triage(exc, codes, 2);
1321 /*NOTREACHED*/
1322 }
1323
1324
1325 #if MACH_KDB
1326
1327 extern void db_i386_state(x86_saved_state32_t *regs);
1328
1329 #include <ddb/db_output.h>
1330
1331 void
1332 db_i386_state(
1333 x86_saved_state32_t *regs)
1334 {
1335 db_printf("eip %8x\n", regs->eip);
1336 db_printf("trap %8x\n", regs->trapno);
1337 db_printf("err %8x\n", regs->err);
1338 db_printf("efl %8x\n", regs->efl);
1339 db_printf("ebp %8x\n", regs->ebp);
1340 db_printf("esp %8x\n", regs->cr2);
1341 db_printf("uesp %8x\n", regs->uesp);
1342 db_printf("cs %8x\n", regs->cs & 0xff);
1343 db_printf("ds %8x\n", regs->ds & 0xff);
1344 db_printf("es %8x\n", regs->es & 0xff);
1345 db_printf("fs %8x\n", regs->fs & 0xff);
1346 db_printf("gs %8x\n", regs->gs & 0xff);
1347 db_printf("ss %8x\n", regs->ss & 0xff);
1348 db_printf("eax %8x\n", regs->eax);
1349 db_printf("ebx %8x\n", regs->ebx);
1350 db_printf("ecx %8x\n", regs->ecx);
1351 db_printf("edx %8x\n", regs->edx);
1352 db_printf("esi %8x\n", regs->esi);
1353 db_printf("edi %8x\n", regs->edi);
1354 }
1355
1356 #endif /* MACH_KDB */
1357
1358 /* Synchronize a thread's i386_kernel_state (if any) with the given
1359 * i386_saved_state_t obtained from the trap/IPI handler; called in
1360 * kernel_trap() prior to entering the debugger, and when receiving
1361 * an "MP_KDP" IPI.
1362 */
1363
1364 void
1365 sync_iss_to_iks(x86_saved_state_t *saved_state)
1366 {
1367 struct x86_kernel_state *iks;
1368 vm_offset_t kstack;
1369 boolean_t record_active_regs = FALSE;
1370
1371 /* The PAL may have a special way to sync registers */
1372 if( saved_state->flavor == THREAD_STATE_NONE )
1373 pal_get_kern_regs( saved_state );
1374
1375 if ((kstack = current_thread()->kernel_stack) != 0) {
1376 #ifdef __i386__
1377 x86_saved_state32_t *regs = saved_state32(saved_state);
1378 #else
1379 x86_saved_state64_t *regs = saved_state64(saved_state);
1380 #endif
1381
1382 iks = STACK_IKS(kstack);
1383
1384 /* Did we take the trap/interrupt in kernel mode? */
1385 #ifdef __i386__
1386 if (regs == USER_REGS32(current_thread()))
1387 record_active_regs = TRUE;
1388 else {
1389 iks->k_ebx = regs->ebx;
1390 iks->k_esp = (int)regs;
1391 iks->k_ebp = regs->ebp;
1392 iks->k_edi = regs->edi;
1393 iks->k_esi = regs->esi;
1394 iks->k_eip = regs->eip;
1395 }
1396 #else
1397 if (regs == USER_REGS64(current_thread()))
1398 record_active_regs = TRUE;
1399 else {
1400 iks->k_rbx = regs->rbx;
1401 iks->k_rsp = regs->isf.rsp;
1402 iks->k_rbp = regs->rbp;
1403 iks->k_r12 = regs->r12;
1404 iks->k_r13 = regs->r13;
1405 iks->k_r14 = regs->r14;
1406 iks->k_r15 = regs->r15;
1407 iks->k_rip = regs->isf.rip;
1408 }
1409 #endif
1410 }
1411
1412 if (record_active_regs == TRUE) {
1413 #ifdef __i386__
1414 /* Show the trap handler path */
1415 __asm__ volatile("movl %%ebx, %0" : "=m" (iks->k_ebx));
1416 __asm__ volatile("movl %%esp, %0" : "=m" (iks->k_esp));
1417 __asm__ volatile("movl %%ebp, %0" : "=m" (iks->k_ebp));
1418 __asm__ volatile("movl %%edi, %0" : "=m" (iks->k_edi));
1419 __asm__ volatile("movl %%esi, %0" : "=m" (iks->k_esi));
1420 /* "Current" instruction pointer */
1421 __asm__ volatile("movl $1f, %0\n1:" : "=m" (iks->k_eip));
1422 #else
1423 /* Show the trap handler path */
1424 __asm__ volatile("movq %%rbx, %0" : "=m" (iks->k_rbx));
1425 __asm__ volatile("movq %%rsp, %0" : "=m" (iks->k_rsp));
1426 __asm__ volatile("movq %%rbp, %0" : "=m" (iks->k_rbp));
1427 __asm__ volatile("movq %%r12, %0" : "=m" (iks->k_r12));
1428 __asm__ volatile("movq %%r13, %0" : "=m" (iks->k_r13));
1429 __asm__ volatile("movq %%r14, %0" : "=m" (iks->k_r14));
1430 __asm__ volatile("movq %%r15, %0" : "=m" (iks->k_r15));
1431 /* "Current" instruction pointer */
1432 __asm__ volatile("leaq 1f(%%rip), %%rax; mov %%rax, %0\n1:"
1433 : "=m" (iks->k_rip)
1434 :
1435 : "rax");
1436 #endif
1437 }
1438 }
1439
1440 /*
1441 * This is used by the NMI interrupt handler (from mp.c) to
1442 * uncondtionally sync the trap handler context to the IKS
1443 * irrespective of whether the NMI was fielded in kernel
1444 * or user space.
1445 */
1446 void
1447 sync_iss_to_iks_unconditionally(__unused x86_saved_state_t *saved_state) {
1448 struct x86_kernel_state *iks;
1449 vm_offset_t kstack;
1450
1451 if ((kstack = current_thread()->kernel_stack) != 0) {
1452 iks = STACK_IKS(kstack);
1453 #ifdef __i386__
1454 /* Display the trap handler path */
1455 __asm__ volatile("movl %%ebx, %0" : "=m" (iks->k_ebx));
1456 __asm__ volatile("movl %%esp, %0" : "=m" (iks->k_esp));
1457 __asm__ volatile("movl %%ebp, %0" : "=m" (iks->k_ebp));
1458 __asm__ volatile("movl %%edi, %0" : "=m" (iks->k_edi));
1459 __asm__ volatile("movl %%esi, %0" : "=m" (iks->k_esi));
1460 /* "Current" instruction pointer */
1461 __asm__ volatile("movl $1f, %0\n1:" : "=m" (iks->k_eip));
1462 #else
1463 /* Display the trap handler path */
1464 __asm__ volatile("movq %%rbx, %0" : "=m" (iks->k_rbx));
1465 __asm__ volatile("movq %%rsp, %0" : "=m" (iks->k_rsp));
1466 __asm__ volatile("movq %%rbp, %0" : "=m" (iks->k_rbp));
1467 __asm__ volatile("movq %%r12, %0" : "=m" (iks->k_r12));
1468 __asm__ volatile("movq %%r13, %0" : "=m" (iks->k_r13));
1469 __asm__ volatile("movq %%r14, %0" : "=m" (iks->k_r14));
1470 __asm__ volatile("movq %%r15, %0" : "=m" (iks->k_r15));
1471 /* "Current" instruction pointer */
1472 __asm__ volatile("leaq 1f(%%rip), %%rax; mov %%rax, %0\n1:" : "=m" (iks->k_rip)::"rax");
1473 #endif
1474 }
1475 }
mirror of XNU