]> git.saurik.com Git - apple/xnu.git/blob - osfmk/i386/trap.c
projects / apple / xnu.git / blob
? search:


5cd9b390d8dd8e7dfe3644a7571e8f1ff9ec9397
[apple/xnu.git] / osfmk / i386 / trap.c
1 /*
2 * Copyright (c) 2000-2008 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58
59 /*
60 * Hardware trap/fault handler.
61 */
62
63 #include <mach_kdb.h>
64 #include <mach_kgdb.h>
65 #include <mach_kdp.h>
66 #include <mach_ldebug.h>
67
68 #include <types.h>
69 #include <i386/eflags.h>
70 #include <i386/trap.h>
71 #include <i386/pmap.h>
72 #include <i386/fpu.h>
73 #include <i386/misc_protos.h> /* panic_io_port_read() */
74 #include <i386/lapic.h>
75
76 #include <mach/exception.h>
77 #include <mach/kern_return.h>
78 #include <mach/vm_param.h>
79 #include <mach/i386/thread_status.h>
80
81 #include <vm/vm_kern.h>
82 #include <vm/vm_fault.h>
83
84 #include <kern/kern_types.h>
85 #include <kern/processor.h>
86 #include <kern/thread.h>
87 #include <kern/task.h>
88 #include <kern/sched.h>
89 #include <kern/sched_prim.h>
90 #include <kern/exception.h>
91 #include <kern/spl.h>
92 #include <kern/misc_protos.h>
93 #include <kern/debug.h>
94
95 #include <sys/kdebug.h>
96
97 #if MACH_KGDB
98 #include <kgdb/kgdb_defs.h>
99 #endif /* MACH_KGDB */
100
101 #if MACH_KDB
102 #include <debug.h>
103 #include <ddb/db_watch.h>
104 #include <ddb/db_run.h>
105 #include <ddb/db_break.h>
106 #include <ddb/db_trap.h>
107 #endif /* MACH_KDB */
108
109 #include <string.h>
110
111 #include <i386/postcode.h>
112 #include <i386/mp_desc.h>
113 #include <i386/proc_reg.h>
114 #if CONFIG_MCA
115 #include <i386/machine_check.h>
116 #endif
117 #include <mach/i386/syscall_sw.h>
118
119 #include <libkern/OSDebug.h>
120
121 extern void throttle_lowpri_io(boolean_t);
122
123
124 /*
125 * Forward declarations
126 */
127 static void user_page_fault_continue(kern_return_t kret);
128 #ifdef __i386__
129 static void panic_trap(x86_saved_state32_t *saved_state);
130 static void set_recovery_ip(x86_saved_state32_t *saved_state, vm_offset_t ip);
131 #else
132 static void panic_trap(x86_saved_state64_t *saved_state);
133 static void set_recovery_ip(x86_saved_state64_t *saved_state, vm_offset_t ip);
134 #endif
135
136 volatile perfCallback perfTrapHook = NULL; /* Pointer to CHUD trap hook routine */
137 volatile perfCallback perfASTHook = NULL; /* Pointer to CHUD AST hook routine */
138
139 #if CONFIG_DTRACE
140 /* See <rdar://problem/4613924> */
141 perfCallback tempDTraceTrapHook = NULL; /* Pointer to DTrace fbt trap hook routine */
142
143 extern boolean_t dtrace_tally_fault(user_addr_t);
144 #endif
145
146 void
147 thread_syscall_return(
148 kern_return_t ret)
149 {
150 thread_t thr_act = current_thread();
151 boolean_t is_mach;
152 int code;
153
154
155 if (thread_is_64bit(thr_act)) {
156 x86_saved_state64_t *regs;
157
158 regs = USER_REGS64(thr_act);
159
160 code = (int) (regs->rax & SYSCALL_NUMBER_MASK);
161 is_mach = (regs->rax & SYSCALL_CLASS_MASK)
162 == (SYSCALL_CLASS_MACH << SYSCALL_CLASS_SHIFT);
163 if (kdebug_enable && is_mach) {
164 /* Mach trap */
165 KERNEL_DEBUG_CONSTANT(
166 MACHDBG_CODE(DBG_MACH_EXCP_SC,code)|DBG_FUNC_END,
167 ret, 0, 0, 0, 0);
168 }
169 regs->rax = ret;
170 #if DEBUG
171 if (is_mach)
172 DEBUG_KPRINT_SYSCALL_MACH(
173 "thread_syscall_return: 64-bit mach ret=%u\n",
174 ret);
175 else
176 DEBUG_KPRINT_SYSCALL_UNIX(
177 "thread_syscall_return: 64-bit unix ret=%u\n",
178 ret);
179 #endif
180 } else {
181 x86_saved_state32_t *regs;
182
183 regs = USER_REGS32(thr_act);
184
185 code = ((int) regs->eax);
186 is_mach = (code < 0);
187 if (kdebug_enable && is_mach) {
188 /* Mach trap */
189 KERNEL_DEBUG_CONSTANT(
190 MACHDBG_CODE(DBG_MACH_EXCP_SC,-code)|DBG_FUNC_END,
191 ret, 0, 0, 0, 0);
192 }
193 regs->eax = ret;
194 #if DEBUG
195 if (is_mach)
196 DEBUG_KPRINT_SYSCALL_MACH(
197 "thread_syscall_return: 32-bit mach ret=%u\n",
198 ret);
199 else
200 DEBUG_KPRINT_SYSCALL_UNIX(
201 "thread_syscall_return: 32-bit unix ret=%u\n",
202 ret);
203 #endif
204 }
205 throttle_lowpri_io(TRUE);
206
207 thread_exception_return();
208 /*NOTREACHED*/
209 }
210
211
212 #if MACH_KDB
213 boolean_t debug_all_traps_with_kdb = FALSE;
214 extern struct db_watchpoint *db_watchpoint_list;
215 extern boolean_t db_watchpoints_inserted;
216 extern boolean_t db_breakpoints_inserted;
217
218 void
219 thread_kdb_return(void)
220 {
221 thread_t thr_act = current_thread();
222 x86_saved_state_t *iss = USER_STATE(thr_act);
223
224
225 if (is_saved_state64(iss)) {
226 x86_saved_state64_t *regs;
227
228 regs = saved_state64(iss);
229
230 if (kdb_trap(regs->isf.trapno, (int)regs->isf.err, (void *)regs)) {
231 thread_exception_return();
232 /*NOTREACHED*/
233 }
234
235 } else {
236 x86_saved_state32_t *regs;
237
238 regs = saved_state32(iss);
239
240 if (kdb_trap(regs->trapno, regs->err, (void *)regs)) {
241 thread_exception_return();
242 /*NOTREACHED*/
243 }
244 }
245 }
246
247 #endif /* MACH_KDB */
248
249 void
250 user_page_fault_continue(
251 kern_return_t kr)
252 {
253 thread_t thread = current_thread();
254 ast_t *myast;
255 boolean_t intr;
256 user_addr_t vaddr;
257
258
259 #if MACH_KDB
260 x86_saved_state_t *regs = USER_STATE(thread);
261 int err;
262 int trapno;
263
264 assert((is_saved_state32(regs) && !thread_is_64bit(thread)) ||
265 (is_saved_state64(regs) && thread_is_64bit(thread)));
266 #endif
267
268 if (thread_is_64bit(thread)) {
269 x86_saved_state64_t *uregs;
270
271 uregs = USER_REGS64(thread);
272
273 #if MACH_KDB
274 trapno = uregs->isf.trapno;
275 err = (int)uregs->isf.err;
276 #endif
277 vaddr = (user_addr_t)uregs->cr2;
278 } else {
279 x86_saved_state32_t *uregs;
280
281 uregs = USER_REGS32(thread);
282
283 #if MACH_KDB
284 trapno = uregs->trapno;
285 err = uregs->err;
286 #endif
287 vaddr = uregs->cr2;
288 }
289
290 if ((kr == KERN_SUCCESS) || (kr == KERN_ABORTED)) {
291 #if MACH_KDB
292 if (!db_breakpoints_inserted) {
293 db_set_breakpoints();
294 }
295 if (db_watchpoint_list &&
296 db_watchpoints_inserted &&
297 (err & T_PF_WRITE) &&
298 db_find_watchpoint(thread->map,
299 (vm_offset_t)vaddr,
300 saved_state32(regs)))
301 kdb_trap(T_WATCHPOINT, 0, saved_state32(regs));
302 #endif /* MACH_KDB */
303 intr = ml_set_interrupts_enabled(FALSE);
304 myast = ast_pending();
305 while (*myast & AST_ALL) {
306 ast_taken(AST_ALL, intr);
307 ml_set_interrupts_enabled(FALSE);
308 myast = ast_pending();
309 }
310 ml_set_interrupts_enabled(intr);
311
312 thread_exception_return();
313 /*NOTREACHED*/
314 }
315
316 #if MACH_KDB
317 if (debug_all_traps_with_kdb &&
318 kdb_trap(trapno, err, saved_state32(regs))) {
319 thread_exception_return();
320 /*NOTREACHED*/
321 }
322 #endif /* MACH_KDB */
323
324
325 i386_exception(EXC_BAD_ACCESS, kr, vaddr);
326 /*NOTREACHED*/
327 }
328
329 /*
330 * Fault recovery in copyin/copyout routines.
331 */
332 struct recovery {
333 uintptr_t fault_addr;
334 uintptr_t recover_addr;
335 };
336
337 extern struct recovery recover_table[];
338 extern struct recovery recover_table_end[];
339
340 const char * trap_type[] = {TRAP_NAMES};
341 unsigned TRAP_TYPES = sizeof(trap_type)/sizeof(trap_type[0]);
342
343 #if defined(__x86_64__) && DEBUG
344 static void
345 print_state(x86_saved_state64_t *saved_state)
346 {
347 kprintf("current_cpu_datap() 0x%lx\n", (uintptr_t)current_cpu_datap());
348 kprintf("Current GS base MSR 0x%llx\n", rdmsr64(MSR_IA32_GS_BASE));
349 kprintf("Kernel GS base MSR 0x%llx\n", rdmsr64(MSR_IA32_KERNEL_GS_BASE));
350 kprintf("state at 0x%lx:\n", (uintptr_t) saved_state);
351
352 kprintf(" rdi 0x%llx\n", saved_state->rdi);
353 kprintf(" rsi 0x%llx\n", saved_state->rsi);
354 kprintf(" rdx 0x%llx\n", saved_state->rdx);
355 kprintf(" r10 0x%llx\n", saved_state->r10);
356 kprintf(" r8 0x%llx\n", saved_state->r8);
357 kprintf(" r9 0x%llx\n", saved_state->r9);
358 kprintf(" v_arg6 0x%llx\n", saved_state->v_arg6);
359 kprintf(" v_arg7 0x%llx\n", saved_state->v_arg7);
360 kprintf(" v_arg8 0x%llx\n", saved_state->v_arg8);
361
362 kprintf(" cr2 0x%llx\n", saved_state->cr2);
363 kprintf("real cr2 0x%lx\n", get_cr2());
364 kprintf(" r15 0x%llx\n", saved_state->r15);
365 kprintf(" r14 0x%llx\n", saved_state->r14);
366 kprintf(" r13 0x%llx\n", saved_state->r13);
367 kprintf(" r12 0x%llx\n", saved_state->r12);
368 kprintf(" r11 0x%llx\n", saved_state->r11);
369 kprintf(" rbp 0x%llx\n", saved_state->rbp);
370 kprintf(" rbx 0x%llx\n", saved_state->rbx);
371 kprintf(" rcx 0x%llx\n", saved_state->rcx);
372 kprintf(" rax 0x%llx\n", saved_state->rax);
373
374 kprintf(" gs 0x%x\n", saved_state->gs);
375 kprintf(" fs 0x%x\n", saved_state->fs);
376
377 kprintf(" isf.trapno 0x%x\n", saved_state->isf.trapno);
378 kprintf(" isf._pad 0x%x\n", saved_state->isf._pad);
379 kprintf(" isf.trapfn 0x%llx\n", saved_state->isf.trapfn);
380 kprintf(" isf.err 0x%llx\n", saved_state->isf.err);
381 kprintf(" isf.rip 0x%llx\n", saved_state->isf.rip);
382 kprintf(" isf.cs 0x%llx\n", saved_state->isf.cs);
383 kprintf(" isf.rflags 0x%llx\n", saved_state->isf.rflags);
384 kprintf(" isf.rsp 0x%llx\n", saved_state->isf.rsp);
385 kprintf(" isf.ss 0x%llx\n", saved_state->isf.ss);
386 }
387 /*
388 * K64 debug - fatal handler for debug code in the trap vectors.
389 */
390 extern void
391 panic_idt64(x86_saved_state_t *rsp);
392 void
393 panic_idt64(x86_saved_state_t *rsp)
394 {
395 print_state(saved_state64(rsp));
396 panic("panic_idt64");
397 }
398 #endif
399
400 extern void PE_incoming_interrupt(int interrupt);
401
402 /*
403 * Handle interrupts:
404 * - local APIC interrupts (IPIs, timers, etc) are handled by the kernel,
405 * - device interrupts go to the platform expert.
406 */
407 void
408 interrupt(x86_saved_state_t *state)
409 {
410 uint64_t rip;
411 uint64_t rsp;
412 int interrupt_num;
413 boolean_t user_mode = FALSE;
414
415
416 if (is_saved_state64(state) == TRUE) {
417 x86_saved_state64_t *state64;
418
419 state64 = saved_state64(state);
420 rip = state64->isf.rip;
421 rsp = state64->isf.rsp;
422 interrupt_num = state64->isf.trapno;
423 #ifdef __x86_64__
424 if(state64->isf.cs & 0x03)
425 #endif
426 user_mode = TRUE;
427 } else {
428 x86_saved_state32_t *state32;
429
430 state32 = saved_state32(state);
431 if (state32->cs & 0x03)
432 user_mode = TRUE;
433 rip = state32->eip;
434 rsp = state32->uesp;
435 interrupt_num = state32->trapno;
436 }
437
438 KERNEL_DEBUG_CONSTANT(
439 MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_START,
440 interrupt_num, (long) rip, user_mode, 0, 0);
441
442 /*
443 * Handle local APIC interrupts
444 * else call platform expert for devices.
445 */
446 if (!lapic_interrupt(interrupt_num, state))
447 PE_incoming_interrupt(interrupt_num);
448
449 KERNEL_DEBUG_CONSTANT(
450 MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_END,
451 0, 0, 0, 0, 0);
452
453 /*
454 * Having serviced the interrupt first, look at the interrupted stack depth.
455 */
456 if (!user_mode) {
457 uint64_t depth = current_cpu_datap()->cpu_kernel_stack
458 + sizeof(struct x86_kernel_state)
459 + sizeof(struct i386_exception_link *)
460 - rsp;
461 if (depth > kernel_stack_depth_max) {
462 kernel_stack_depth_max = (vm_offset_t)depth;
463 KERNEL_DEBUG_CONSTANT(
464 MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_DEPTH),
465 (long) depth, (long) rip, 0, 0, 0);
466 }
467 }
468 }
469
470 static inline void
471 reset_dr7(void)
472 {
473 long dr7 = 0x400; /* magic dr7 reset value; 32 bit on i386, 64 bit on x86_64 */
474 __asm__ volatile("mov %0,%%dr7" : : "r" (dr7));
475 }
476 #if MACH_KDP
477 unsigned kdp_has_active_watchpoints = 0;
478 #define NO_WATCHPOINTS (!kdp_has_active_watchpoints)
479 #else
480 #define NO_WATCHPOINTS 1
481 #endif
482 /*
483 * Trap from kernel mode. Only page-fault errors are recoverable,
484 * and then only in special circumstances. All other errors are
485 * fatal. Return value indicates if trap was handled.
486 */
487
488 void
489 kernel_trap(
490 x86_saved_state_t *state)
491 {
492 #ifdef __i386__
493 x86_saved_state32_t *saved_state;
494 #else
495 x86_saved_state64_t *saved_state;
496 #endif
497 int code;
498 user_addr_t vaddr;
499 int type;
500 vm_map_t map = 0; /* protected by T_PAGE_FAULT */
501 kern_return_t result = KERN_FAILURE;
502 thread_t thread;
503 ast_t *myast;
504 boolean_t intr;
505 vm_prot_t prot;
506 struct recovery *rp;
507 vm_offset_t kern_ip;
508 #if NCOPY_WINDOWS > 0
509 int fault_in_copy_window = -1;
510 #endif
511 int is_user = 0;
512 #if MACH_KDB
513 pt_entry_t *pte;
514 #endif /* MACH_KDB */
515
516 thread = current_thread();
517
518 #ifdef __i386__
519 if (is_saved_state64(state))
520 panic("kernel_trap(%p) with 64-bit state", state);
521 saved_state = saved_state32(state);
522 vaddr = (user_addr_t)saved_state->cr2;
523 type = saved_state->trapno;
524 code = saved_state->err & 0xffff;
525 intr = (saved_state->efl & EFL_IF) != 0; /* state of ints at trap */
526 kern_ip = (vm_offset_t)saved_state->eip;
527 #else
528 if (is_saved_state32(state))
529 panic("kernel_trap(%p) with 32-bit state", state);
530 saved_state = saved_state64(state);
531 vaddr = (user_addr_t)saved_state->cr2;
532 type = saved_state->isf.trapno;
533 code = (int)(saved_state->isf.err & 0xffff);
534 intr = (saved_state->isf.rflags & EFL_IF) != 0; /* state of ints at trap */
535 kern_ip = (vm_offset_t)saved_state->isf.rip;
536 #endif
537
538 myast = ast_pending();
539
540 perfCallback fn = perfASTHook;
541 if (fn) {
542 if (*myast & AST_CHUD_ALL)
543 fn(type, NULL, 0, 0);
544 } else
545 *myast &= ~AST_CHUD_ALL;
546
547 /*
548 * Is there a hook?
549 */
550 fn = perfTrapHook;
551 if (fn) {
552 if (fn(type, NULL, 0, 0) == KERN_SUCCESS) {
553 /*
554 * If it succeeds, we are done...
555 */
556 return;
557 }
558 }
559
560 #if CONFIG_DTRACE
561 if (tempDTraceTrapHook) {
562 if (tempDTraceTrapHook(type, state, 0, 0) == KERN_SUCCESS) {
563 /*
564 * If it succeeds, we are done...
565 */
566 return;
567 }
568 }
569 #endif /* CONFIG_DTRACE */
570
571 /*
572 * we come here with interrupts off as we don't want to recurse
573 * on preemption below. but we do want to re-enable interrupts
574 * as soon we possibly can to hold latency down
575 */
576 if (T_PREEMPT == type) {
577 ast_taken(AST_PREEMPTION, FALSE);
578
579 KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_EXCP_KTRAP_x86, type)) | DBG_FUNC_NONE,
580 0, 0, 0, kern_ip, 0);
581 return;
582 }
583
584 if (T_PAGE_FAULT == type) {
585 /*
586 * assume we're faulting in the kernel map
587 */
588 map = kernel_map;
589
590 if (thread != THREAD_NULL && thread->map != kernel_map) {
591 #if NCOPY_WINDOWS > 0
592 vm_offset_t copy_window_base;
593 vm_offset_t kvaddr;
594 int window_index;
595
596 kvaddr = (vm_offset_t)vaddr;
597 /*
598 * must determine if fault occurred in
599 * the copy window while pre-emption is
600 * disabled for this processor so that
601 * we only need to look at the window
602 * associated with this processor
603 */
604 copy_window_base = current_cpu_datap()->cpu_copywindow_base;
605
606 if (kvaddr >= copy_window_base && kvaddr < (copy_window_base + (NBPDE * NCOPY_WINDOWS)) ) {
607
608 window_index = (kvaddr - copy_window_base) / NBPDE;
609
610 if (thread->machine.copy_window[window_index].user_base != (user_addr_t)-1) {
611
612 kvaddr -= (copy_window_base + (NBPDE * window_index));
613 vaddr = thread->machine.copy_window[window_index].user_base + kvaddr;
614
615 map = thread->map;
616 fault_in_copy_window = window_index;
617 }
618 is_user = -1;
619 }
620 #else
621 if (vaddr < VM_MAX_USER_PAGE_ADDRESS) {
622 /* fault occurred in userspace */
623 map = thread->map;
624 is_user = -1;
625 /*
626 * If we're not sharing cr3 with the user
627 * and we faulted in copyio,
628 * then switch cr3 here and dismiss the fault.
629 */
630 if (no_shared_cr3 &&
631 (thread->machine.specFlags&CopyIOActive) &&
632 map->pmap->pm_cr3 != get_cr3()) {
633 set_cr3(map->pmap->pm_cr3);
634 return;
635 }
636 }
637 #endif
638 }
639 }
640 KERNEL_DEBUG_CONSTANT(
641 (MACHDBG_CODE(DBG_MACH_EXCP_KTRAP_x86, type)) | DBG_FUNC_NONE,
642 (unsigned)(vaddr >> 32), (unsigned)vaddr, is_user, kern_ip, 0);
643
644
645 (void) ml_set_interrupts_enabled(intr);
646
647 switch (type) {
648
649 case T_NO_FPU:
650 fpnoextflt();
651 return;
652
653 case T_FPU_FAULT:
654 fpextovrflt();
655 return;
656
657 case T_FLOATING_POINT_ERROR:
658 fpexterrflt();
659 return;
660
661 case T_SSE_FLOAT_ERROR:
662 fpSSEexterrflt();
663 return;
664 case T_DEBUG:
665 #ifdef __i386__
666 if ((saved_state->efl & EFL_TF) == 0 && NO_WATCHPOINTS)
667 #else
668 if ((saved_state->isf.rflags & EFL_TF) == 0 && NO_WATCHPOINTS)
669 #endif
670 {
671 /* We've somehow encountered a debug
672 * register match that does not belong
673 * to the kernel debugger.
674 * This isn't supposed to happen.
675 */
676 reset_dr7();
677 return;
678 }
679 goto debugger_entry;
680 #ifdef __x86_64__
681 case T_INT3:
682 goto debugger_entry;
683 #endif
684 case T_PAGE_FAULT:
685 /*
686 * If the current map is a submap of the kernel map,
687 * and the address is within that map, fault on that
688 * map. If the same check is done in vm_fault
689 * (vm_map_lookup), we may deadlock on the kernel map
690 * lock.
691 */
692
693 prot = VM_PROT_READ;
694
695 if (code & T_PF_WRITE)
696 prot |= VM_PROT_WRITE;
697 #if PAE
698 if (code & T_PF_EXECUTE)
699 prot |= VM_PROT_EXECUTE;
700 #endif
701
702 #if MACH_KDB
703 /*
704 * Check for watchpoint on kernel static data.
705 * vm_fault would fail in this case
706 */
707 if (map == kernel_map && db_watchpoint_list && db_watchpoints_inserted &&
708 (code & T_PF_WRITE) && vaddr < vm_map_max(map) &&
709 ((*(pte = pmap_pte(kernel_pmap, (vm_map_offset_t)vaddr))) & INTEL_PTE_WRITE) == 0) {
710 pmap_store_pte(
711 pte,
712 *pte | INTEL_PTE_VALID | INTEL_PTE_WRITE);
713 /* XXX need invltlb here? */
714
715 result = KERN_SUCCESS;
716 goto look_for_watchpoints;
717 }
718 #endif /* MACH_KDB */
719
720 #if CONFIG_DTRACE
721 if (thread->options & TH_OPT_DTRACE) { /* Executing under dtrace_probe? */
722 if (dtrace_tally_fault(vaddr)) { /* Should a fault under dtrace be ignored? */
723 /*
724 * DTrace has "anticipated" the possibility of this fault, and has
725 * established the suitable recovery state. Drop down now into the
726 * recovery handling code in "case T_GENERAL_PROTECTION:".
727 */
728 goto FALL_THROUGH;
729 }
730 }
731 #endif /* CONFIG_DTRACE */
732
733 result = vm_fault(map,
734 vm_map_trunc_page(vaddr),
735 prot,
736 FALSE,
737 THREAD_UNINT, NULL, 0);
738
739 #if MACH_KDB
740 if (result == KERN_SUCCESS) {
741 /*
742 * Look for watchpoints
743 */
744 look_for_watchpoints:
745 if (map == kernel_map && db_watchpoint_list && db_watchpoints_inserted && (code & T_PF_WRITE) &&
746 db_find_watchpoint(map, vaddr, saved_state))
747 kdb_trap(T_WATCHPOINT, 0, saved_state);
748 }
749 #endif /* MACH_KDB */
750
751 if (result == KERN_SUCCESS) {
752 #if NCOPY_WINDOWS > 0
753 if (fault_in_copy_window != -1) {
754 pt_entry_t *updp;
755 pt_entry_t *kpdp;
756
757 /*
758 * in case there was no page table assigned
759 * for the user base address and the pmap
760 * got 'expanded' due to this fault, we'll
761 * copy in the descriptor
762 *
763 * we're either setting the page table descriptor
764 * to the same value or it was 0... no need
765 * for a TLB flush in either case
766 */
767
768 ml_set_interrupts_enabled(FALSE);
769 updp = pmap_pde(map->pmap, thread->machine.copy_window[fault_in_copy_window].user_base);
770 assert(updp);
771 if (0 == updp) panic("trap: updp 0"); /* XXX DEBUG */
772 kpdp = current_cpu_datap()->cpu_copywindow_pdp;
773 kpdp += fault_in_copy_window;
774
775 #if JOE_DEBUG
776 if (*kpdp && (*kpdp & PG_FRAME) != (*updp & PG_FRAME))
777 panic("kernel_fault: user pdp doesn't match - updp = 0x%qx, kpdp = 0x%qx\n", *updp, *kpdp);
778 #endif
779 pmap_store_pte(kpdp, *updp);
780
781 (void) ml_set_interrupts_enabled(intr);
782 }
783 #endif /* NCOPY_WINDOWS > 0 */
784 return;
785 }
786 /*
787 * fall through
788 */
789 #if CONFIG_DTRACE
790 FALL_THROUGH:
791 #endif /* CONFIG_DTRACE */
792
793 case T_GENERAL_PROTECTION:
794 #if defined(__x86_64__) && DEBUG
795 print_state(saved_state);
796 #endif
797 /*
798 * If there is a failure recovery address
799 * for this fault, go there.
800 */
801 for (rp = recover_table; rp < recover_table_end; rp++) {
802 if (kern_ip == rp->fault_addr) {
803 set_recovery_ip(saved_state, rp->recover_addr);
804 return;
805 }
806 }
807
808 /*
809 * Check thread recovery address also.
810 */
811 if (thread->recover) {
812 set_recovery_ip(saved_state, thread->recover);
813 thread->recover = 0;
814 return;
815 }
816 /*
817 * Unanticipated page-fault errors in kernel
818 * should not happen.
819 *
820 * fall through...
821 */
822
823 default:
824 /*
825 * Exception 15 is reserved but some chips may generate it
826 * spuriously. Seen at startup on AMD Athlon-64.
827 */
828 if (type == 15) {
829 kprintf("kernel_trap() ignoring spurious trap 15\n");
830 return;
831 }
832 debugger_entry:
833 /* Ensure that the i386_kernel_state at the base of the
834 * current thread's stack (if any) is synchronized with the
835 * context at the moment of the trap, to facilitate
836 * access through the debugger.
837 */
838 sync_iss_to_iks(state);
839 #if MACH_KDB
840 restart_debugger:
841 #endif /* MACH_KDB */
842 #if MACH_KDP
843 if (current_debugger != KDB_CUR_DB) {
844 if (kdp_i386_trap(type, saved_state, result, (vm_offset_t)vaddr))
845 return;
846 } else {
847 #endif /* MACH_KDP */
848 #if MACH_KDB
849 if (kdb_trap(type, code, saved_state)) {
850 if (switch_debugger) {
851 current_debugger = KDP_CUR_DB;
852 switch_debugger = 0;
853 goto restart_debugger;
854 }
855 return;
856 }
857 #endif /* MACH_KDB */
858 #if MACH_KDP
859 }
860 #endif
861 }
862
863 panic_trap(saved_state);
864 /*
865 * NO RETURN
866 */
867 }
868
869
870 #ifdef __i386__
871 static void
872 set_recovery_ip(x86_saved_state32_t *saved_state, vm_offset_t ip)
873 {
874 saved_state->eip = ip;
875 }
876 #else
877 static void
878 set_recovery_ip(x86_saved_state64_t *saved_state, vm_offset_t ip)
879 {
880 saved_state->isf.rip = ip;
881 }
882 #endif
883
884
885 #ifdef __i386__
886 static void
887 panic_trap(x86_saved_state32_t *regs)
888 {
889 const char *trapname = "Unknown";
890 uint32_t cr0 = get_cr0();
891 uint32_t cr2 = get_cr2();
892 uint32_t cr3 = get_cr3();
893 uint32_t cr4 = get_cr4();
894 /*
895 * Issue an I/O port read if one has been requested - this is an
896 * event logic analyzers can use as a trigger point.
897 */
898 panic_io_port_read();
899
900 kprintf("panic trap number 0x%x, eip 0x%x\n", regs->trapno, regs->eip);
901 kprintf("cr0 0x%08x cr2 0x%08x cr3 0x%08x cr4 0x%08x\n",
902 cr0, cr2, cr3, cr4);
903
904 if (regs->trapno < TRAP_TYPES)
905 trapname = trap_type[regs->trapno];
906 #undef panic
907 panic("Kernel trap at 0x%08x, type %d=%s, registers:\n"
908 "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n"
909 "EAX: 0x%08x, EBX: 0x%08x, ECX: 0x%08x, EDX: 0x%08x\n"
910 "CR2: 0x%08x, EBP: 0x%08x, ESI: 0x%08x, EDI: 0x%08x\n"
911 "EFL: 0x%08x, EIP: 0x%08x, CS: 0x%08x, DS: 0x%08x\n"
912 "Error code: 0x%08x\n",
913 regs->eip, regs->trapno, trapname, cr0, cr2, cr3, cr4,
914 regs->eax,regs->ebx,regs->ecx,regs->edx,
915 regs->cr2,regs->ebp,regs->esi,regs->edi,
916 regs->efl,regs->eip,regs->cs, regs->ds, regs->err);
917 /*
918 * This next statement is not executed,
919 * but it's needed to stop the compiler using tail call optimization
920 * for the panic call - which confuses the subsequent backtrace.
921 */
922 cr0 = 0;
923 }
924 #else
925 static void
926 panic_trap(x86_saved_state64_t *regs)
927 {
928 const char *trapname = "Unknown";
929 uint64_t cr0 = get_cr0();
930 uint64_t cr2 = get_cr2();
931 uint64_t cr3 = get_cr3();
932 uint64_t cr4 = get_cr4();
933
934 /*
935 * Issue an I/O port read if one has been requested - this is an
936 * event logic analyzers can use as a trigger point.
937 */
938 panic_io_port_read();
939
940 kprintf("panic trap number 0x%x, rip 0x%016llx\n",
941 regs->isf.trapno, regs->isf.rip);
942 kprintf("cr0 0x%016llx cr2 0x%016llx cr3 0x%016llx cr4 0x%016llx\n",
943 cr0, cr2, cr3, cr4);
944
945 if (regs->isf.trapno < TRAP_TYPES)
946 trapname = trap_type[regs->isf.trapno];
947 #undef panic
948 panic("Kernel trap at 0x%016llx, type %d=%s, registers:\n"
949 "CR0: 0x%016llx, CR2: 0x%016llx, CR3: 0x%016llx, CR4: 0x%016llx\n"
950 "RAX: 0x%016llx, RBX: 0x%016llx, RCX: 0x%016llx, RDX: 0x%016llx\n"
951 "RSP: 0x%016llx, RBP: 0x%016llx, RSI: 0x%016llx, RDI: 0x%016llx\n"
952 "R8: 0x%016llx, R9: 0x%016llx, R10: 0x%016llx, R11: 0x%016llx\n"
953 "R12: 0x%016llx, R13: 0x%016llx, R14: 0x%016llx, R15: 0x%016llx\n"
954 "RFL: 0x%016llx, RIP: 0x%016llx, CS: 0x%016llx, SS: 0x%016llx\n"
955 "Error code: 0x%016llx\n",
956 regs->isf.rip, regs->isf.trapno, trapname,
957 cr0, cr2, cr3, cr4,
958 regs->rax, regs->rbx, regs->rcx, regs->rdx,
959 regs->isf.rsp, regs->rbp, regs->rsi, regs->rdi,
960 regs->r8, regs->r9, regs->r10, regs->r11,
961 regs->r12, regs->r13, regs->r14, regs->r15,
962 regs->isf.rflags, regs->isf.rip, regs->isf.cs, regs->isf.ss,
963 regs->isf.err);
964 /*
965 * This next statement is not executed,
966 * but it's needed to stop the compiler using tail call optimization
967 * for the panic call - which confuses the subsequent backtrace.
968 */
969 cr0 = 0;
970 }
971 #endif
972
973 extern void kprintf_break_lock(void);
974
975 #ifdef __i386__
976 static void
977 panic_32(__unused int code, __unused int pc, __unused const char *msg, boolean_t do_mca_dump, boolean_t do_bt)
978 {
979 struct i386_tss *my_ktss = current_ktss();
980
981 /* Set postcode (DEBUG only) */
982 postcode(pc);
983
984 /*
985 * Issue an I/O port read if one has been requested - this is an
986 * event logic analyzers can use as a trigger point.
987 */
988 panic_io_port_read();
989
990 /*
991 * Break kprintf lock in case of recursion,
992 * and record originally faulted instruction address.
993 */
994 kprintf_break_lock();
995
996 if (do_mca_dump) {
997 #if CONFIG_MCA
998 /*
999 * Dump the contents of the machine check MSRs (if any).
1000 */
1001 mca_dump();
1002 #endif
1003 }
1004
1005 #if MACH_KDP
1006 /*
1007 * Print backtrace leading to first fault:
1008 */
1009 if (do_bt)
1010 panic_i386_backtrace((void *) my_ktss->ebp, 10, NULL, FALSE, NULL);
1011 #endif
1012
1013 panic("%s at 0x%08x, thread:%p, code:0x%x, "
1014 "registers:\n"
1015 "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n"
1016 "EAX: 0x%08x, EBX: 0x%08x, ECX: 0x%08x, EDX: 0x%08x\n"
1017 "ESP: 0x%08x, EBP: 0x%08x, ESI: 0x%08x, EDI: 0x%08x\n"
1018 "EFL: 0x%08x, EIP: 0x%08x\n",
1019 msg,
1020 my_ktss->eip, current_thread(), code,
1021 (uint32_t)get_cr0(), (uint32_t)get_cr2(), (uint32_t)get_cr3(), (uint32_t)get_cr4(),
1022 my_ktss->eax, my_ktss->ebx, my_ktss->ecx, my_ktss->edx,
1023 my_ktss->esp, my_ktss->ebp, my_ktss->esi, my_ktss->edi,
1024 my_ktss->eflags, my_ktss->eip);
1025 }
1026
1027 /*
1028 * Called from locore on a special reserved stack after a double-fault
1029 * is taken in kernel space.
1030 * Kernel stack overflow is one route here.
1031 */
1032 void
1033 panic_double_fault32(int code)
1034 {
1035 panic_32(code, PANIC_DOUBLE_FAULT, "Double fault", FALSE, TRUE);
1036 }
1037
1038 /*
1039 * Called from locore on a special reserved stack after a machine-check
1040 */
1041 void
1042 panic_machine_check32(int code)
1043 {
1044 panic_32(code, PANIC_MACHINE_CHECK, "Machine-check", TRUE, FALSE);
1045 }
1046 #endif /* __i386__ */
1047
1048 static void
1049 panic_64(x86_saved_state_t *sp, __unused int pc, __unused const char *msg, boolean_t do_mca_dump)
1050 {
1051 /* Set postcode (DEBUG only) */
1052 postcode(pc);
1053
1054 /*
1055 * Issue an I/O port read if one has been requested - this is an
1056 * event logic analyzers can use as a trigger point.
1057 */
1058 panic_io_port_read();
1059
1060 /*
1061 * Break kprintf lock in case of recursion,
1062 * and record originally faulted instruction address.
1063 */
1064 kprintf_break_lock();
1065
1066 if (do_mca_dump) {
1067 #if CONFIG_MCA
1068 /*
1069 * Dump the contents of the machine check MSRs (if any).
1070 */
1071 mca_dump();
1072 #endif
1073 }
1074
1075 #ifdef __i386__
1076 /*
1077 * Dump the interrupt stack frame at last kernel entry.
1078 */
1079 if (is_saved_state64(sp)) {
1080 x86_saved_state64_t *ss64p = saved_state64(sp);
1081 panic("%s thread:%p, trapno:0x%x, err:0x%qx, "
1082 "registers:\n"
1083 "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n"
1084 "RAX: 0x%016qx, RBX: 0x%016qx, RCX: 0x%016qx, RDX: 0x%016qx\n"
1085 "RSP: 0x%016qx, RBP: 0x%016qx, RSI: 0x%016qx, RDI: 0x%016qx\n"
1086 "R8: 0x%016qx, R9: 0x%016qx, R10: 0x%016qx, R11: 0x%016qx\n"
1087 "R12: 0x%016qx, R13: 0x%016qx, R14: 0x%016qx, R15: 0x%016qx\n"
1088 "RFL: 0x%016qx, RIP: 0x%016qx, CR2: 0x%016qx\n",
1089 msg,
1090 current_thread(), ss64p->isf.trapno, ss64p->isf.err,
1091 (uint32_t)get_cr0(), (uint32_t)get_cr2(), (uint32_t)get_cr3(), (uint32_t)get_cr4(),
1092 ss64p->rax, ss64p->rbx, ss64p->rcx, ss64p->rdx,
1093 ss64p->isf.rsp, ss64p->rbp, ss64p->rsi, ss64p->rdi,
1094 ss64p->r8, ss64p->r9, ss64p->r10, ss64p->r11,
1095 ss64p->r12, ss64p->r13, ss64p->r14, ss64p->r15,
1096 ss64p->isf.rflags, ss64p->isf.rip, ss64p->cr2);
1097 } else {
1098 x86_saved_state32_t *ss32p = saved_state32(sp);
1099 panic("%s at 0x%08x, thread:%p, trapno:0x%x, err:0x%x,"
1100 "registers:\n"
1101 "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n"
1102 "EAX: 0x%08x, EBX: 0x%08x, ECX: 0x%08x, EDX: 0x%08x\n"
1103 "ESP: 0x%08x, EBP: 0x%08x, ESI: 0x%08x, EDI: 0x%08x\n"
1104 "EFL: 0x%08x, EIP: 0x%08x\n",
1105 msg,
1106 ss32p->eip, current_thread(), ss32p->trapno, ss32p->err,
1107 (uint32_t)get_cr0(), (uint32_t)get_cr2(), (uint32_t)get_cr3(), (uint32_t)get_cr4(),
1108 ss32p->eax, ss32p->ebx, ss32p->ecx, ss32p->edx,
1109 ss32p->uesp, ss32p->ebp, ss32p->esi, ss32p->edi,
1110 ss32p->efl, ss32p->eip);
1111 }
1112 #else
1113 x86_saved_state64_t *regs = saved_state64(sp);
1114 panic("%s thread:%p at 0x%016llx, registers:\n"
1115 "CR0: 0x%016lx, CR2: 0x%016lx, CR3: 0x%016lx, CR4: 0x%016lx\n"
1116 "RAX: 0x%016llx, RBX: 0x%016llx, RCX: 0x%016llx, RDX: 0x%016llx\n"
1117 "RSP: 0x%016llx, RBP: 0x%016llx, RSI: 0x%016llx, RDI: 0x%016llx\n"
1118 "R8: 0x%016llx, R9: 0x%016llx, R10: 0x%016llx, R11: 0x%016llx\n"
1119 "R12: 0x%016llx, R13: 0x%016llx, R14: 0x%016llx, R15: 0x%016llx\n"
1120 "RFL: 0x%016llx, RIP: 0x%016llx, CS: 0x%016llx, SS: 0x%016llx\n"
1121 "Error code: 0x%016llx\n",
1122 msg,
1123 current_thread(), regs->isf.rip,
1124 get_cr0(), get_cr2(), get_cr3(), get_cr4(),
1125 regs->rax, regs->rbx, regs->rcx, regs->rdx,
1126 regs->isf.rsp, regs->rbp, regs->rsi, regs->rdi,
1127 regs->r8, regs->r9, regs->r10, regs->r11,
1128 regs->r12, regs->r13, regs->r14, regs->r15,
1129 regs->isf.rflags, regs->isf.rip, regs->isf.cs, regs->isf.ss,
1130 regs->isf.err);
1131 #endif
1132 }
1133
1134 void
1135 panic_double_fault64(x86_saved_state_t *sp)
1136 {
1137 panic_64(sp, PANIC_DOUBLE_FAULT, "Double fault", FALSE);
1138
1139 }
1140 void
1141
1142 panic_machine_check64(x86_saved_state_t *sp)
1143 {
1144 panic_64(sp, PANIC_MACHINE_CHECK, "Machine Check", TRUE);
1145
1146 }
1147
1148 #if CONFIG_DTRACE
1149 extern kern_return_t dtrace_user_probe(x86_saved_state_t *);
1150 #endif
1151
1152 /*
1153 * Trap from user mode.
1154 */
1155 void
1156 user_trap(
1157 x86_saved_state_t *saved_state)
1158 {
1159 int exc;
1160 int err;
1161 mach_exception_code_t code;
1162 mach_exception_subcode_t subcode;
1163 int type;
1164 user_addr_t vaddr;
1165 vm_prot_t prot;
1166 thread_t thread = current_thread();
1167 ast_t *myast;
1168 kern_return_t kret;
1169 user_addr_t rip;
1170
1171 assert((is_saved_state32(saved_state) && !thread_is_64bit(thread)) ||
1172 (is_saved_state64(saved_state) && thread_is_64bit(thread)));
1173
1174 if (is_saved_state64(saved_state)) {
1175 x86_saved_state64_t *regs;
1176
1177 regs = saved_state64(saved_state);
1178
1179 type = regs->isf.trapno;
1180 err = (int)regs->isf.err & 0xffff;
1181 vaddr = (user_addr_t)regs->cr2;
1182 rip = (user_addr_t)regs->isf.rip;
1183 } else {
1184 x86_saved_state32_t *regs;
1185
1186 regs = saved_state32(saved_state);
1187
1188 type = regs->trapno;
1189 err = regs->err & 0xffff;
1190 vaddr = (user_addr_t)regs->cr2;
1191 rip = (user_addr_t)regs->eip;
1192 }
1193
1194 KERNEL_DEBUG_CONSTANT(
1195 (MACHDBG_CODE(DBG_MACH_EXCP_UTRAP_x86, type)) | DBG_FUNC_NONE,
1196 (unsigned)(vaddr>>32), (unsigned)vaddr,
1197 (unsigned)(rip>>32), (unsigned)rip, 0);
1198
1199 code = 0;
1200 subcode = 0;
1201 exc = 0;
1202
1203 #if DEBUG_TRACE
1204 kprintf("user_trap(0x%08x) type=%d vaddr=0x%016llx\n",
1205 saved_state, type, vaddr);
1206 #endif
1207 perfCallback fn = perfASTHook;
1208 if (fn) {
1209 myast = ast_pending();
1210 if (*myast & AST_CHUD_ALL) {
1211 fn(type, saved_state, 0, 0);
1212 }
1213 }
1214
1215 /* Is there a hook? */
1216 fn = perfTrapHook;
1217 if (fn) {
1218 if (fn(type, saved_state, 0, 0) == KERN_SUCCESS)
1219 return; /* If it succeeds, we are done... */
1220 }
1221
1222 /*
1223 * DTrace does not consume all user traps, only INT_3's for now.
1224 * Avoid needlessly calling tempDTraceTrapHook here, and let the
1225 * INT_3 case handle them.
1226 */
1227 DEBUG_KPRINT_SYSCALL_MASK(1,
1228 "user_trap: type=0x%x(%s) err=0x%x cr2=%p rip=%p\n",
1229 type, trap_type[type], err, (void *)(long) vaddr, (void *)(long) rip);
1230
1231 switch (type) {
1232
1233 case T_DIVIDE_ERROR:
1234 exc = EXC_ARITHMETIC;
1235 code = EXC_I386_DIV;
1236 break;
1237
1238 case T_DEBUG:
1239 {
1240 pcb_t pcb;
1241 long clear = 0; /* 32 bit for i386, 64 bit for x86_64 */
1242 /*
1243 * get dr6 and set it in the thread's pcb before
1244 * returning to userland
1245 */
1246 pcb = thread->machine.pcb;
1247 if (pcb->ids) {
1248 /*
1249 * We can get and set the status register
1250 * in 32-bit mode even on a 64-bit thread
1251 * because the high order bits are not
1252 * used on x86_64
1253 */
1254 unsigned long dr6_temp; /* 32 bit for i386, 64 bit for x86_64 */
1255 __asm__ volatile ("mov %%db6, %0" : "=r" (dr6_temp)); /* Register constraint by necessity */
1256 if (thread_is_64bit(thread)) {
1257 x86_debug_state64_t *ids = pcb->ids;
1258 ids->dr6 = dr6_temp;
1259 } else { /* 32 bit thread */
1260 x86_debug_state32_t *ids = pcb->ids;
1261 ids->dr6 = (uint32_t) dr6_temp;
1262 }
1263 __asm__ volatile ("mov %0, %%db6" : : "r" (clear));
1264 }
1265 exc = EXC_BREAKPOINT;
1266 code = EXC_I386_SGL;
1267 break;
1268 }
1269 case T_INT3:
1270 #if CONFIG_DTRACE
1271 if (dtrace_user_probe(saved_state) == KERN_SUCCESS)
1272 return; /* If it succeeds, we are done... */
1273 #endif
1274 exc = EXC_BREAKPOINT;
1275 code = EXC_I386_BPT;
1276 break;
1277
1278 case T_OVERFLOW:
1279 exc = EXC_ARITHMETIC;
1280 code = EXC_I386_INTO;
1281 break;
1282
1283 case T_OUT_OF_BOUNDS:
1284 exc = EXC_SOFTWARE;
1285 code = EXC_I386_BOUND;
1286 break;
1287
1288 case T_INVALID_OPCODE:
1289 exc = EXC_BAD_INSTRUCTION;
1290 code = EXC_I386_INVOP;
1291 break;
1292
1293 case T_NO_FPU:
1294 fpnoextflt();
1295 return;
1296
1297 case T_FPU_FAULT:
1298 fpextovrflt(); /* Propagates exception directly, doesn't return */
1299 return;
1300
1301 case T_INVALID_TSS: /* invalid TSS == iret with NT flag set */
1302 exc = EXC_BAD_INSTRUCTION;
1303 code = EXC_I386_INVTSSFLT;
1304 subcode = err;
1305 break;
1306
1307 case T_SEGMENT_NOT_PRESENT:
1308 exc = EXC_BAD_INSTRUCTION;
1309 code = EXC_I386_SEGNPFLT;
1310 subcode = err;
1311 break;
1312
1313 case T_STACK_FAULT:
1314 exc = EXC_BAD_INSTRUCTION;
1315 code = EXC_I386_STKFLT;
1316 subcode = err;
1317 break;
1318
1319 case T_GENERAL_PROTECTION:
1320 /*
1321 * There's a wide range of circumstances which generate this
1322 * class of exception. From user-space, many involve bad
1323 * addresses (such as a non-canonical 64-bit address).
1324 * So we map this to EXC_BAD_ACCESS (and thereby SIGSEGV).
1325 * The trouble is cr2 doesn't contain the faulting address;
1326 * we'd need to decode the faulting instruction to really
1327 * determine this. We'll leave that to debuggers.
1328 * However, attempted execution of privileged instructions
1329 * (e.g. cli) also generate GP faults and so we map these to
1330 * to EXC_BAD_ACCESS (and thence SIGSEGV) also - rather than
1331 * EXC_BAD_INSTRUCTION which is more accurate. We just can't
1332 * win!
1333 */
1334 exc = EXC_BAD_ACCESS;
1335 code = EXC_I386_GPFLT;
1336 subcode = err;
1337 break;
1338
1339 case T_PAGE_FAULT:
1340 prot = VM_PROT_READ;
1341
1342 if (err & T_PF_WRITE)
1343 prot |= VM_PROT_WRITE;
1344 #if PAE
1345 if (err & T_PF_EXECUTE)
1346 prot |= VM_PROT_EXECUTE;
1347 #endif
1348 kret = vm_fault(thread->map, vm_map_trunc_page(vaddr),
1349 prot, FALSE,
1350 THREAD_ABORTSAFE, NULL, 0);
1351
1352 user_page_fault_continue(kret);
1353
1354 /* NOTREACHED */
1355 break;
1356
1357 case T_SSE_FLOAT_ERROR:
1358 fpSSEexterrflt(); /* Propagates exception directly, doesn't return */
1359 return;
1360
1361
1362 case T_FLOATING_POINT_ERROR:
1363 fpexterrflt(); /* Propagates exception directly, doesn't return */
1364 return;
1365
1366 case T_DTRACE_RET:
1367 #if CONFIG_DTRACE
1368 if (dtrace_user_probe(saved_state) == KERN_SUCCESS)
1369 return; /* If it succeeds, we are done... */
1370 #endif
1371 /*
1372 * If we get an INT 0x7f when we do not expect to,
1373 * treat it as an illegal instruction
1374 */
1375 exc = EXC_BAD_INSTRUCTION;
1376 code = EXC_I386_INVOP;
1377 break;
1378
1379 default:
1380 #if MACH_KGDB
1381 Debugger("Unanticipated user trap");
1382 return;
1383 #endif /* MACH_KGDB */
1384 #if MACH_KDB
1385 if (kdb_trap(type, err, saved_state32(saved_state)))
1386 return;
1387 #endif /* MACH_KDB */
1388 panic("Unexpected user trap, type %d", type);
1389 return;
1390 }
1391 /* Note: Codepaths that directly return from user_trap() have pending
1392 * ASTs processed in locore
1393 */
1394 i386_exception(exc, code, subcode);
1395 /* NOTREACHED */
1396 }
1397
1398
1399 /*
1400 * Handle AST traps for i386.
1401 * Check for delayed floating-point exception from
1402 * AT-bus machines.
1403 */
1404
1405 extern void log_thread_action (thread_t, char *);
1406
1407 void
1408 i386_astintr(int preemption)
1409 {
1410 ast_t mask = AST_ALL;
1411 spl_t s;
1412
1413 if (preemption)
1414 mask = AST_PREEMPTION;
1415
1416 s = splsched();
1417
1418 ast_taken(mask, s);
1419
1420 splx(s);
1421 }
1422
1423 /*
1424 * Handle exceptions for i386.
1425 *
1426 * If we are an AT bus machine, we must turn off the AST for a
1427 * delayed floating-point exception.
1428 *
1429 * If we are providing floating-point emulation, we may have
1430 * to retrieve the real register values from the floating point
1431 * emulator.
1432 */
1433 void
1434 i386_exception(
1435 int exc,
1436 mach_exception_code_t code,
1437 mach_exception_subcode_t subcode)
1438 {
1439 mach_exception_data_type_t codes[EXCEPTION_CODE_MAX];
1440
1441 DEBUG_KPRINT_SYSCALL_MACH("i386_exception: exc=%d code=0x%llx subcode=0x%llx\n",
1442 exc, code, subcode);
1443 codes[0] = code; /* new exception interface */
1444 codes[1] = subcode;
1445 exception_triage(exc, codes, 2);
1446 /*NOTREACHED*/
1447 }
1448
1449
1450
1451 void
1452 kernel_preempt_check(void)
1453 {
1454 ast_t *myast;
1455 boolean_t intr;
1456
1457 /*
1458 * disable interrupts to both prevent pre-emption
1459 * and to keep the ast state from changing via
1460 * an interrupt handler making something runnable
1461 */
1462 intr = ml_set_interrupts_enabled(FALSE);
1463
1464 myast = ast_pending();
1465
1466 if ((*myast & AST_URGENT) && intr == TRUE && get_interrupt_level() == 0) {
1467 /*
1468 * can handle interrupts and preemptions
1469 * at this point
1470 */
1471 ml_set_interrupts_enabled(intr);
1472
1473 /*
1474 * now cause the PRE-EMPTION trap
1475 */
1476 __asm__ volatile (" int $0xff");
1477 } else {
1478 /*
1479 * if interrupts were already disabled or
1480 * we're in an interrupt context, we can't
1481 * preempt... of course if AST_URGENT
1482 * isn't set we also don't want to
1483 */
1484 ml_set_interrupts_enabled(intr);
1485 }
1486 }
1487
1488 #if MACH_KDB
1489
1490 extern void db_i386_state(x86_saved_state32_t *regs);
1491
1492 #include <ddb/db_output.h>
1493
1494 void
1495 db_i386_state(
1496 x86_saved_state32_t *regs)
1497 {
1498 db_printf("eip %8x\n", regs->eip);
1499 db_printf("trap %8x\n", regs->trapno);
1500 db_printf("err %8x\n", regs->err);
1501 db_printf("efl %8x\n", regs->efl);
1502 db_printf("ebp %8x\n", regs->ebp);
1503 db_printf("esp %8x\n", regs->cr2);
1504 db_printf("uesp %8x\n", regs->uesp);
1505 db_printf("cs %8x\n", regs->cs & 0xff);
1506 db_printf("ds %8x\n", regs->ds & 0xff);
1507 db_printf("es %8x\n", regs->es & 0xff);
1508 db_printf("fs %8x\n", regs->fs & 0xff);
1509 db_printf("gs %8x\n", regs->gs & 0xff);
1510 db_printf("ss %8x\n", regs->ss & 0xff);
1511 db_printf("eax %8x\n", regs->eax);
1512 db_printf("ebx %8x\n", regs->ebx);
1513 db_printf("ecx %8x\n", regs->ecx);
1514 db_printf("edx %8x\n", regs->edx);
1515 db_printf("esi %8x\n", regs->esi);
1516 db_printf("edi %8x\n", regs->edi);
1517 }
1518
1519 #endif /* MACH_KDB */
1520
1521 /* Synchronize a thread's i386_kernel_state (if any) with the given
1522 * i386_saved_state_t obtained from the trap/IPI handler; called in
1523 * kernel_trap() prior to entering the debugger, and when receiving
1524 * an "MP_KDP" IPI.
1525 */
1526
1527 void
1528 sync_iss_to_iks(x86_saved_state_t *saved_state)
1529 {
1530 struct x86_kernel_state *iks;
1531 vm_offset_t kstack;
1532 boolean_t record_active_regs = FALSE;
1533
1534 if ((kstack = current_thread()->kernel_stack) != 0) {
1535 #ifdef __i386__
1536 x86_saved_state32_t *regs = saved_state32(saved_state);
1537 #else
1538 x86_saved_state64_t *regs = saved_state64(saved_state);
1539 #endif
1540
1541 iks = STACK_IKS(kstack);
1542
1543
1544 /* Did we take the trap/interrupt in kernel mode? */
1545 #ifdef __i386__
1546 if (regs == USER_REGS32(current_thread()))
1547 record_active_regs = TRUE;
1548 else {
1549 iks->k_ebx = regs->ebx;
1550 iks->k_esp = (int)regs;
1551 iks->k_ebp = regs->ebp;
1552 iks->k_edi = regs->edi;
1553 iks->k_esi = regs->esi;
1554 iks->k_eip = regs->eip;
1555 }
1556 #else
1557 if (regs == USER_REGS64(current_thread()))
1558 record_active_regs = TRUE;
1559 else {
1560 iks->k_rbx = regs->rbx;
1561 iks->k_rsp = regs->isf.rsp;
1562 iks->k_rbp = regs->rbp;
1563 iks->k_r12 = regs->r12;
1564 iks->k_r13 = regs->r13;
1565 iks->k_r14 = regs->r14;
1566 iks->k_r15 = regs->r15;
1567 iks->k_rip = regs->isf.rip;
1568 }
1569 #endif
1570 }
1571
1572 if (record_active_regs == TRUE) {
1573 #ifdef __i386__
1574 /* Show the trap handler path */
1575 __asm__ volatile("movl %%ebx, %0" : "=m" (iks->k_ebx));
1576 __asm__ volatile("movl %%esp, %0" : "=m" (iks->k_esp));
1577 __asm__ volatile("movl %%ebp, %0" : "=m" (iks->k_ebp));
1578 __asm__ volatile("movl %%edi, %0" : "=m" (iks->k_edi));
1579 __asm__ volatile("movl %%esi, %0" : "=m" (iks->k_esi));
1580 /* "Current" instruction pointer */
1581 __asm__ volatile("movl $1f, %0\n1:" : "=m" (iks->k_eip));
1582 #else
1583 /* Show the trap handler path */
1584 __asm__ volatile("movq %%rbx, %0" : "=m" (iks->k_rbx));
1585 __asm__ volatile("movq %%rsp, %0" : "=m" (iks->k_rsp));
1586 __asm__ volatile("movq %%rbp, %0" : "=m" (iks->k_rbp));
1587 __asm__ volatile("movq %%r12, %0" : "=m" (iks->k_r12));
1588 __asm__ volatile("movq %%r13, %0" : "=m" (iks->k_r13));
1589 __asm__ volatile("movq %%r14, %0" : "=m" (iks->k_r14));
1590 __asm__ volatile("movq %%r15, %0" : "=m" (iks->k_r15));
1591 /* "Current" instruction pointer */
1592 __asm__ volatile("leaq 1f(%%rip), %%rax; mov %%rax, %0\n1:"
1593 : "=m" (iks->k_rip)
1594 :
1595 : "rax");
1596 #endif
1597 }
1598 }
1599
1600 /*
1601 * This is used by the NMI interrupt handler (from mp.c) to
1602 * uncondtionally sync the trap handler context to the IKS
1603 * irrespective of whether the NMI was fielded in kernel
1604 * or user space.
1605 */
1606 void
1607 sync_iss_to_iks_unconditionally(__unused x86_saved_state_t *saved_state) {
1608 struct x86_kernel_state *iks;
1609 vm_offset_t kstack;
1610
1611 if ((kstack = current_thread()->kernel_stack) != 0) {
1612 iks = STACK_IKS(kstack);
1613 #ifdef __i386__
1614 /* Display the trap handler path */
1615 __asm__ volatile("movl %%ebx, %0" : "=m" (iks->k_ebx));
1616 __asm__ volatile("movl %%esp, %0" : "=m" (iks->k_esp));
1617 __asm__ volatile("movl %%ebp, %0" : "=m" (iks->k_ebp));
1618 __asm__ volatile("movl %%edi, %0" : "=m" (iks->k_edi));
1619 __asm__ volatile("movl %%esi, %0" : "=m" (iks->k_esi));
1620 /* "Current" instruction pointer */
1621 __asm__ volatile("movl $1f, %0\n1:" : "=m" (iks->k_eip));
1622 #else
1623 /* Display the trap handler path */
1624 __asm__ volatile("movq %%rbx, %0" : "=m" (iks->k_rbx));
1625 __asm__ volatile("movq %%rsp, %0" : "=m" (iks->k_rsp));
1626 __asm__ volatile("movq %%rbp, %0" : "=m" (iks->k_rbp));
1627 __asm__ volatile("movq %%r12, %0" : "=m" (iks->k_r12));
1628 __asm__ volatile("movq %%r13, %0" : "=m" (iks->k_r13));
1629 __asm__ volatile("movq %%r14, %0" : "=m" (iks->k_r14));
1630 __asm__ volatile("movq %%r15, %0" : "=m" (iks->k_r15));
1631 /* "Current" instruction pointer */
1632 __asm__ volatile("leaq 1f(%%rip), %%rax; mov %%rax, %0\n1:" : "=m" (iks->k_rip)::"rax");
1633 #endif
1634 }
1635 }
mirror of XNU