]> git.saurik.com Git - apple/xnu.git/blob - osfmk/i386/trap.c
projects / apple / xnu.git / blob
? search:


8f27f5e58ff23c057780ee354698316bc5a90d5c
[apple/xnu.git] / osfmk / i386 / trap.c
1 /*
2 * Copyright (c) 2000-2008 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58
59 /*
60 * Hardware trap/fault handler.
61 */
62
63 #include <mach_kdb.h>
64 #include <mach_kgdb.h>
65 #include <mach_kdp.h>
66 #include <mach_ldebug.h>
67
68 #include <types.h>
69 #include <i386/eflags.h>
70 #include <i386/trap.h>
71 #include <i386/pmap.h>
72 #include <i386/fpu.h>
73 #include <i386/misc_protos.h> /* panic_io_port_read() */
74 #include <i386/lapic.h>
75
76 #include <mach/exception.h>
77 #include <mach/kern_return.h>
78 #include <mach/vm_param.h>
79 #include <mach/i386/thread_status.h>
80
81 #include <vm/vm_kern.h>
82 #include <vm/vm_fault.h>
83
84 #include <kern/kern_types.h>
85 #include <kern/processor.h>
86 #include <kern/thread.h>
87 #include <kern/task.h>
88 #include <kern/sched.h>
89 #include <kern/sched_prim.h>
90 #include <kern/exception.h>
91 #include <kern/spl.h>
92 #include <kern/misc_protos.h>
93 #include <kern/debug.h>
94
95 #include <sys/kdebug.h>
96
97 #if MACH_KGDB
98 #include <kgdb/kgdb_defs.h>
99 #endif /* MACH_KGDB */
100
101 #if MACH_KDB
102 #include <debug.h>
103 #include <ddb/db_watch.h>
104 #include <ddb/db_run.h>
105 #include <ddb/db_break.h>
106 #include <ddb/db_trap.h>
107 #endif /* MACH_KDB */
108
109 #include <string.h>
110
111 #include <i386/postcode.h>
112 #include <i386/mp_desc.h>
113 #include <i386/proc_reg.h>
114 #if CONFIG_MCA
115 #include <i386/machine_check.h>
116 #endif
117 #include <mach/i386/syscall_sw.h>
118
119 #include <libkern/OSDebug.h>
120
121 extern void throttle_lowpri_io(boolean_t);
122
123
124 /*
125 * Forward declarations
126 */
127 static void user_page_fault_continue(kern_return_t kret);
128 #ifdef __i386__
129 static void panic_trap(x86_saved_state32_t *saved_state);
130 static void set_recovery_ip(x86_saved_state32_t *saved_state, vm_offset_t ip);
131 #else
132 static void panic_trap(x86_saved_state64_t *saved_state);
133 static void set_recovery_ip(x86_saved_state64_t *saved_state, vm_offset_t ip);
134 #endif
135
136 volatile perfCallback perfTrapHook = NULL; /* Pointer to CHUD trap hook routine */
137 volatile perfCallback perfASTHook = NULL; /* Pointer to CHUD AST hook routine */
138
139 #if CONFIG_DTRACE
140 /* See <rdar://problem/4613924> */
141 perfCallback tempDTraceTrapHook = NULL; /* Pointer to DTrace fbt trap hook routine */
142
143 extern boolean_t dtrace_tally_fault(user_addr_t);
144 #endif
145
146 void
147 thread_syscall_return(
148 kern_return_t ret)
149 {
150 thread_t thr_act = current_thread();
151 boolean_t is_mach;
152 int code;
153
154
155 if (thread_is_64bit(thr_act)) {
156 x86_saved_state64_t *regs;
157
158 regs = USER_REGS64(thr_act);
159
160 code = (int) (regs->rax & SYSCALL_NUMBER_MASK);
161 is_mach = (regs->rax & SYSCALL_CLASS_MASK)
162 == (SYSCALL_CLASS_MACH << SYSCALL_CLASS_SHIFT);
163 if (kdebug_enable && is_mach) {
164 /* Mach trap */
165 KERNEL_DEBUG_CONSTANT(
166 MACHDBG_CODE(DBG_MACH_EXCP_SC,code)|DBG_FUNC_END,
167 ret, 0, 0, 0, 0);
168 }
169 regs->rax = ret;
170 #if DEBUG
171 if (is_mach)
172 DEBUG_KPRINT_SYSCALL_MACH(
173 "thread_syscall_return: 64-bit mach ret=%u\n",
174 ret);
175 else
176 DEBUG_KPRINT_SYSCALL_UNIX(
177 "thread_syscall_return: 64-bit unix ret=%u\n",
178 ret);
179 #endif
180 } else {
181 x86_saved_state32_t *regs;
182
183 regs = USER_REGS32(thr_act);
184
185 code = ((int) regs->eax);
186 is_mach = (code < 0);
187 if (kdebug_enable && is_mach) {
188 /* Mach trap */
189 KERNEL_DEBUG_CONSTANT(
190 MACHDBG_CODE(DBG_MACH_EXCP_SC,-code)|DBG_FUNC_END,
191 ret, 0, 0, 0, 0);
192 }
193 regs->eax = ret;
194 #if DEBUG
195 if (is_mach)
196 DEBUG_KPRINT_SYSCALL_MACH(
197 "thread_syscall_return: 32-bit mach ret=%u\n",
198 ret);
199 else
200 DEBUG_KPRINT_SYSCALL_UNIX(
201 "thread_syscall_return: 32-bit unix ret=%u\n",
202 ret);
203 #endif
204 }
205 throttle_lowpri_io(TRUE);
206
207 thread_exception_return();
208 /*NOTREACHED*/
209 }
210
211
212 #if MACH_KDB
213 boolean_t debug_all_traps_with_kdb = FALSE;
214 extern struct db_watchpoint *db_watchpoint_list;
215 extern boolean_t db_watchpoints_inserted;
216 extern boolean_t db_breakpoints_inserted;
217
218 void
219 thread_kdb_return(void)
220 {
221 thread_t thr_act = current_thread();
222 x86_saved_state_t *iss = USER_STATE(thr_act);
223
224
225 if (is_saved_state64(iss)) {
226 x86_saved_state64_t *regs;
227
228 regs = saved_state64(iss);
229
230 if (kdb_trap(regs->isf.trapno, (int)regs->isf.err, (void *)regs)) {
231 thread_exception_return();
232 /*NOTREACHED*/
233 }
234
235 } else {
236 x86_saved_state32_t *regs;
237
238 regs = saved_state32(iss);
239
240 if (kdb_trap(regs->trapno, regs->err, (void *)regs)) {
241 thread_exception_return();
242 /*NOTREACHED*/
243 }
244 }
245 }
246
247 #endif /* MACH_KDB */
248
249 void
250 user_page_fault_continue(
251 kern_return_t kr)
252 {
253 thread_t thread = current_thread();
254 ast_t *myast;
255 boolean_t intr;
256 user_addr_t vaddr;
257
258
259 #if MACH_KDB
260 x86_saved_state_t *regs = USER_STATE(thread);
261 int err;
262 int trapno;
263
264 assert((is_saved_state32(regs) && !thread_is_64bit(thread)) ||
265 (is_saved_state64(regs) && thread_is_64bit(thread)));
266 #endif
267
268 if (thread_is_64bit(thread)) {
269 x86_saved_state64_t *uregs;
270
271 uregs = USER_REGS64(thread);
272
273 #if MACH_KDB
274 trapno = uregs->isf.trapno;
275 err = (int)uregs->isf.err;
276 #endif
277 vaddr = (user_addr_t)uregs->cr2;
278 } else {
279 x86_saved_state32_t *uregs;
280
281 uregs = USER_REGS32(thread);
282
283 #if MACH_KDB
284 trapno = uregs->trapno;
285 err = uregs->err;
286 #endif
287 vaddr = uregs->cr2;
288 }
289
290 if ((kr == KERN_SUCCESS) || (kr == KERN_ABORTED)) {
291 #if MACH_KDB
292 if (!db_breakpoints_inserted) {
293 db_set_breakpoints();
294 }
295 if (db_watchpoint_list &&
296 db_watchpoints_inserted &&
297 (err & T_PF_WRITE) &&
298 db_find_watchpoint(thread->map,
299 (vm_offset_t)vaddr,
300 saved_state32(regs)))
301 kdb_trap(T_WATCHPOINT, 0, saved_state32(regs));
302 #endif /* MACH_KDB */
303 intr = ml_set_interrupts_enabled(FALSE);
304 myast = ast_pending();
305 while (*myast & AST_ALL) {
306 ast_taken(AST_ALL, intr);
307 ml_set_interrupts_enabled(FALSE);
308 myast = ast_pending();
309 }
310 ml_set_interrupts_enabled(intr);
311
312 thread_exception_return();
313 /*NOTREACHED*/
314 }
315
316 #if MACH_KDB
317 if (debug_all_traps_with_kdb &&
318 kdb_trap(trapno, err, saved_state32(regs))) {
319 thread_exception_return();
320 /*NOTREACHED*/
321 }
322 #endif /* MACH_KDB */
323
324
325 i386_exception(EXC_BAD_ACCESS, kr, vaddr);
326 /*NOTREACHED*/
327 }
328
329 /*
330 * Fault recovery in copyin/copyout routines.
331 */
332 struct recovery {
333 uintptr_t fault_addr;
334 uintptr_t recover_addr;
335 };
336
337 extern struct recovery recover_table[];
338 extern struct recovery recover_table_end[];
339
340 const char * trap_type[] = {TRAP_NAMES};
341 unsigned TRAP_TYPES = sizeof(trap_type)/sizeof(trap_type[0]);
342
343 #if defined(__x86_64__) && DEBUG
344 static void
345 print_state(x86_saved_state64_t *saved_state)
346 {
347 kprintf("current_cpu_datap() 0x%lx\n", (uintptr_t)current_cpu_datap());
348 kprintf("Current GS base MSR 0x%llx\n", rdmsr64(MSR_IA32_GS_BASE));
349 kprintf("Kernel GS base MSR 0x%llx\n", rdmsr64(MSR_IA32_KERNEL_GS_BASE));
350 kprintf("state at 0x%lx:\n", (uintptr_t) saved_state);
351
352 kprintf(" rdi 0x%llx\n", saved_state->rdi);
353 kprintf(" rsi 0x%llx\n", saved_state->rsi);
354 kprintf(" rdx 0x%llx\n", saved_state->rdx);
355 kprintf(" r10 0x%llx\n", saved_state->r10);
356 kprintf(" r8 0x%llx\n", saved_state->r8);
357 kprintf(" r9 0x%llx\n", saved_state->r9);
358 kprintf(" v_arg6 0x%llx\n", saved_state->v_arg6);
359 kprintf(" v_arg7 0x%llx\n", saved_state->v_arg7);
360 kprintf(" v_arg8 0x%llx\n", saved_state->v_arg8);
361
362 kprintf(" cr2 0x%llx\n", saved_state->cr2);
363 kprintf("real cr2 0x%lx\n", get_cr2());
364 kprintf(" r15 0x%llx\n", saved_state->r15);
365 kprintf(" r14 0x%llx\n", saved_state->r14);
366 kprintf(" r13 0x%llx\n", saved_state->r13);
367 kprintf(" r12 0x%llx\n", saved_state->r12);
368 kprintf(" r11 0x%llx\n", saved_state->r11);
369 kprintf(" rbp 0x%llx\n", saved_state->rbp);
370 kprintf(" rbx 0x%llx\n", saved_state->rbx);
371 kprintf(" rcx 0x%llx\n", saved_state->rcx);
372 kprintf(" rax 0x%llx\n", saved_state->rax);
373
374 kprintf(" gs 0x%x\n", saved_state->gs);
375 kprintf(" fs 0x%x\n", saved_state->fs);
376
377 kprintf(" isf.trapno 0x%x\n", saved_state->isf.trapno);
378 kprintf(" isf._pad 0x%x\n", saved_state->isf._pad);
379 kprintf(" isf.trapfn 0x%llx\n", saved_state->isf.trapfn);
380 kprintf(" isf.err 0x%llx\n", saved_state->isf.err);
381 kprintf(" isf.rip 0x%llx\n", saved_state->isf.rip);
382 kprintf(" isf.cs 0x%llx\n", saved_state->isf.cs);
383 kprintf(" isf.rflags 0x%llx\n", saved_state->isf.rflags);
384 kprintf(" isf.rsp 0x%llx\n", saved_state->isf.rsp);
385 kprintf(" isf.ss 0x%llx\n", saved_state->isf.ss);
386 }
387 /*
388 * K64 debug - fatal handler for debug code in the trap vectors.
389 */
390 extern void
391 panic_idt64(x86_saved_state_t *rsp);
392 void
393 panic_idt64(x86_saved_state_t *rsp)
394 {
395 print_state(saved_state64(rsp));
396 panic("panic_idt64");
397 }
398 #endif
399
400 extern void PE_incoming_interrupt(int interrupt);
401
402 /*
403 * Handle interrupts:
404 * - local APIC interrupts (IPIs, timers, etc) are handled by the kernel,
405 * - device interrupts go to the platform expert.
406 */
407 void
408 interrupt(x86_saved_state_t *state)
409 {
410 uint64_t rip;
411 uint64_t rsp;
412 int interrupt_num;
413 boolean_t user_mode = FALSE;
414
415
416 if (is_saved_state64(state) == TRUE) {
417 x86_saved_state64_t *state64;
418
419 state64 = saved_state64(state);
420 rip = state64->isf.rip;
421 rsp = state64->isf.rsp;
422 interrupt_num = state64->isf.trapno;
423 #ifdef __x86_64__
424 if(state64->isf.cs & 0x03)
425 #endif
426 user_mode = TRUE;
427 } else {
428 x86_saved_state32_t *state32;
429
430 state32 = saved_state32(state);
431 if (state32->cs & 0x03)
432 user_mode = TRUE;
433 rip = state32->eip;
434 rsp = state32->uesp;
435 interrupt_num = state32->trapno;
436 }
437
438 KERNEL_DEBUG_CONSTANT(
439 MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_START,
440 interrupt_num, (long) rip, user_mode, 0, 0);
441
442 /*
443 * Handle local APIC interrupts
444 * else call platform expert for devices.
445 */
446 if (!lapic_interrupt(interrupt_num, state))
447 PE_incoming_interrupt(interrupt_num);
448
449 KERNEL_DEBUG_CONSTANT(
450 MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_END,
451 0, 0, 0, 0, 0);
452
453 /*
454 * Having serviced the interrupt first, look at the interrupted stack depth.
455 */
456 if (!user_mode) {
457 uint64_t depth = current_cpu_datap()->cpu_kernel_stack
458 + sizeof(struct x86_kernel_state)
459 + sizeof(struct i386_exception_link *)
460 - rsp;
461 if (depth > kernel_stack_depth_max) {
462 kernel_stack_depth_max = (vm_offset_t)depth;
463 KERNEL_DEBUG_CONSTANT(
464 MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_DEPTH),
465 (long) depth, (long) rip, 0, 0, 0);
466 }
467 }
468 }
469
470 static inline void
471 reset_dr7(void)
472 {
473 long dr7 = 0x400; /* magic dr7 reset value; 32 bit on i386, 64 bit on x86_64 */
474 __asm__ volatile("mov %0,%%dr7" : : "r" (dr7));
475 }
476 #if MACH_KDP
477 unsigned kdp_has_active_watchpoints = 0;
478 #define NO_WATCHPOINTS (!kdp_has_active_watchpoints)
479 #else
480 #define NO_WATCHPOINTS 1
481 #endif
482 /*
483 * Trap from kernel mode. Only page-fault errors are recoverable,
484 * and then only in special circumstances. All other errors are
485 * fatal. Return value indicates if trap was handled.
486 */
487
488 void
489 kernel_trap(
490 x86_saved_state_t *state)
491 {
492 #ifdef __i386__
493 x86_saved_state32_t *saved_state;
494 #else
495 x86_saved_state64_t *saved_state;
496 #endif
497 int code;
498 user_addr_t vaddr;
499 int type;
500 vm_map_t map = 0; /* protected by T_PAGE_FAULT */
501 kern_return_t result = KERN_FAILURE;
502 thread_t thread;
503 ast_t *myast;
504 boolean_t intr;
505 vm_prot_t prot;
506 struct recovery *rp;
507 vm_offset_t kern_ip;
508 #if NCOPY_WINDOWS > 0
509 int fault_in_copy_window = -1;
510 #endif
511 int is_user = 0;
512 #if MACH_KDB
513 pt_entry_t *pte;
514 #endif /* MACH_KDB */
515
516 thread = current_thread();
517
518 #ifdef __i386__
519 if (is_saved_state64(state))
520 panic("kernel_trap(%p) with 64-bit state", state);
521 saved_state = saved_state32(state);
522 vaddr = (user_addr_t)saved_state->cr2;
523 type = saved_state->trapno;
524 code = saved_state->err & 0xffff;
525 intr = (saved_state->efl & EFL_IF) != 0; /* state of ints at trap */
526 kern_ip = (vm_offset_t)saved_state->eip;
527 #else
528 if (is_saved_state32(state))
529 panic("kernel_trap(%p) with 32-bit state", state);
530 saved_state = saved_state64(state);
531 vaddr = (user_addr_t)saved_state->cr2;
532 type = saved_state->isf.trapno;
533 code = (int)(saved_state->isf.err & 0xffff);
534 intr = (saved_state->isf.rflags & EFL_IF) != 0; /* state of ints at trap */
535 kern_ip = (vm_offset_t)saved_state->isf.rip;
536 #endif
537
538 myast = ast_pending();
539
540 perfCallback fn = perfASTHook;
541 if (fn) {
542 if (*myast & AST_CHUD_ALL)
543 fn(type, NULL, 0, 0);
544 } else
545 *myast &= ~AST_CHUD_ALL;
546
547 /*
548 * Is there a hook?
549 */
550 fn = perfTrapHook;
551 if (fn) {
552 if (fn(type, NULL, 0, 0) == KERN_SUCCESS) {
553 /*
554 * If it succeeds, we are done...
555 */
556 return;
557 }
558 }
559
560 #if CONFIG_DTRACE
561 if (tempDTraceTrapHook) {
562 if (tempDTraceTrapHook(type, state, 0, 0) == KERN_SUCCESS) {
563 /*
564 * If it succeeds, we are done...
565 */
566 return;
567 }
568 }
569 #endif /* CONFIG_DTRACE */
570
571 /*
572 * we come here with interrupts off as we don't want to recurse
573 * on preemption below. but we do want to re-enable interrupts
574 * as soon we possibly can to hold latency down
575 */
576 if (T_PREEMPT == type) {
577 ast_taken(AST_PREEMPTION, FALSE);
578
579 KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_EXCP_KTRAP_x86, type)) | DBG_FUNC_NONE,
580 0, 0, 0, kern_ip, 0);
581 return;
582 }
583
584 if (T_PAGE_FAULT == type) {
585 /*
586 * assume we're faulting in the kernel map
587 */
588 map = kernel_map;
589
590 if (thread != THREAD_NULL && thread->map != kernel_map) {
591 #if NCOPY_WINDOWS > 0
592 vm_offset_t copy_window_base;
593 vm_offset_t kvaddr;
594 int window_index;
595
596 kvaddr = (vm_offset_t)vaddr;
597 /*
598 * must determine if fault occurred in
599 * the copy window while pre-emption is
600 * disabled for this processor so that
601 * we only need to look at the window
602 * associated with this processor
603 */
604 copy_window_base = current_cpu_datap()->cpu_copywindow_base;
605
606 if (kvaddr >= copy_window_base && kvaddr < (copy_window_base + (NBPDE * NCOPY_WINDOWS)) ) {
607
608 window_index = (kvaddr - copy_window_base) / NBPDE;
609
610 if (thread->machine.copy_window[window_index].user_base != (user_addr_t)-1) {
611
612 kvaddr -= (copy_window_base + (NBPDE * window_index));
613 vaddr = thread->machine.copy_window[window_index].user_base + kvaddr;
614
615 map = thread->map;
616 fault_in_copy_window = window_index;
617 }
618 is_user = -1;
619 }
620 #else
621 if (vaddr < VM_MAX_USER_PAGE_ADDRESS) {
622 /* fault occurred in userspace */
623 map = thread->map;
624 is_user = -1;
625 /*
626 * If we're not sharing cr3 with the user
627 * and we faulted in copyio,
628 * then switch cr3 here and dismiss the fault.
629 */
630 if (no_shared_cr3 &&
631 (thread->machine.specFlags&CopyIOActive) &&
632 map->pmap->pm_cr3 != get_cr3()) {
633 set_cr3(map->pmap->pm_cr3);
634 return;
635 }
636 }
637 #endif
638 }
639 }
640 KERNEL_DEBUG_CONSTANT(
641 (MACHDBG_CODE(DBG_MACH_EXCP_KTRAP_x86, type)) | DBG_FUNC_NONE,
642 (unsigned)(vaddr >> 32), (unsigned)vaddr, is_user, kern_ip, 0);
643
644
645 (void) ml_set_interrupts_enabled(intr);
646
647 switch (type) {
648
649 case T_NO_FPU:
650 fpnoextflt();
651 return;
652
653 case T_FPU_FAULT:
654 fpextovrflt();
655 return;
656
657 case T_FLOATING_POINT_ERROR:
658 fpexterrflt();
659 return;
660
661 case T_SSE_FLOAT_ERROR:
662 fpSSEexterrflt();
663 return;
664 case T_DEBUG:
665 #ifdef __i386__
666 if ((saved_state->efl & EFL_TF) == 0 && NO_WATCHPOINTS)
667 #else
668 if ((saved_state->isf.rflags & EFL_TF) == 0 && NO_WATCHPOINTS)
669 #endif
670 {
671 /* We've somehow encountered a debug
672 * register match that does not belong
673 * to the kernel debugger.
674 * This isn't supposed to happen.
675 */
676 reset_dr7();
677 return;
678 }
679 goto debugger_entry;
680 #ifdef __x86_64__
681 case T_INT3:
682 goto debugger_entry;
683 #endif
684 case T_PAGE_FAULT:
685 /*
686 * If the current map is a submap of the kernel map,
687 * and the address is within that map, fault on that
688 * map. If the same check is done in vm_fault
689 * (vm_map_lookup), we may deadlock on the kernel map
690 * lock.
691 */
692
693 prot = VM_PROT_READ;
694
695 if (code & T_PF_WRITE)
696 prot |= VM_PROT_WRITE;
697 #if PAE
698 if (code & T_PF_EXECUTE)
699 prot |= VM_PROT_EXECUTE;
700 #endif
701
702 #if MACH_KDB
703 /*
704 * Check for watchpoint on kernel static data.
705 * vm_fault would fail in this case
706 */
707 if (map == kernel_map && db_watchpoint_list && db_watchpoints_inserted &&
708 (code & T_PF_WRITE) && vaddr < vm_map_max(map) &&
709 ((*(pte = pmap_pte(kernel_pmap, (vm_map_offset_t)vaddr))) & INTEL_PTE_WRITE) == 0) {
710 pmap_store_pte(
711 pte,
712 *pte | INTEL_PTE_VALID | INTEL_PTE_WRITE);
713 /* XXX need invltlb here? */
714
715 result = KERN_SUCCESS;
716 goto look_for_watchpoints;
717 }
718 #endif /* MACH_KDB */
719
720 #if CONFIG_DTRACE
721 if (thread->options & TH_OPT_DTRACE) { /* Executing under dtrace_probe? */
722 if (dtrace_tally_fault(vaddr)) { /* Should a fault under dtrace be ignored? */
723 /*
724 * DTrace has "anticipated" the possibility of this fault, and has
725 * established the suitable recovery state. Drop down now into the
726 * recovery handling code in "case T_GENERAL_PROTECTION:".
727 */
728 goto FALL_THROUGH;
729 }
730 }
731 #endif /* CONFIG_DTRACE */
732
733 result = vm_fault(map,
734 vm_map_trunc_page(vaddr),
735 prot,
736 FALSE,
737 THREAD_UNINT, NULL, 0);
738
739 #if MACH_KDB
740 if (result == KERN_SUCCESS) {
741 /*
742 * Look for watchpoints
743 */
744 look_for_watchpoints:
745 if (map == kernel_map && db_watchpoint_list && db_watchpoints_inserted && (code & T_PF_WRITE) &&
746 db_find_watchpoint(map, vaddr, saved_state))
747 kdb_trap(T_WATCHPOINT, 0, saved_state);
748 }
749 #endif /* MACH_KDB */
750
751 if (result == KERN_SUCCESS) {
752 #if NCOPY_WINDOWS > 0
753 if (fault_in_copy_window != -1) {
754 pt_entry_t *updp;
755 pt_entry_t *kpdp;
756
757 /*
758 * in case there was no page table assigned
759 * for the user base address and the pmap
760 * got 'expanded' due to this fault, we'll
761 * copy in the descriptor
762 *
763 * we're either setting the page table descriptor
764 * to the same value or it was 0... no need
765 * for a TLB flush in either case
766 */
767
768 ml_set_interrupts_enabled(FALSE);
769 updp = pmap_pde(map->pmap, thread->machine.copy_window[fault_in_copy_window].user_base);
770 assert(updp);
771 if (0 == updp) panic("trap: updp 0"); /* XXX DEBUG */
772 kpdp = current_cpu_datap()->cpu_copywindow_pdp;
773 kpdp += fault_in_copy_window;
774
775 #if JOE_DEBUG
776 if (*kpdp && (*kpdp & PG_FRAME) != (*updp & PG_FRAME))
777 panic("kernel_fault: user pdp doesn't match - updp = 0x%qx, kpdp = 0x%qx\n", *updp, *kpdp);
778 #endif
779 pmap_store_pte(kpdp, *updp);
780
781 (void) ml_set_interrupts_enabled(intr);
782 }
783 #endif /* NCOPY_WINDOWS > 0 */
784 return;
785 }
786 /*
787 * fall through
788 */
789 #if CONFIG_DTRACE
790 FALL_THROUGH:
791 #endif /* CONFIG_DTRACE */
792
793 case T_GENERAL_PROTECTION:
794 #if defined(__x86_64__) && DEBUG
795 print_state(saved_state);
796 #endif
797 /*
798 * If there is a failure recovery address
799 * for this fault, go there.
800 */
801 for (rp = recover_table; rp < recover_table_end; rp++) {
802 if (kern_ip == rp->fault_addr) {
803 set_recovery_ip(saved_state, rp->recover_addr);
804 return;
805 }
806 }
807
808 /*
809 * Check thread recovery address also.
810 */
811 if (thread->recover) {
812 set_recovery_ip(saved_state, thread->recover);
813 thread->recover = 0;
814 return;
815 }
816 /*
817 * Unanticipated page-fault errors in kernel
818 * should not happen.
819 *
820 * fall through...
821 */
822
823 default:
824 /*
825 * Exception 15 is reserved but some chips may generate it
826 * spuriously. Seen at startup on AMD Athlon-64.
827 */
828 if (type == 15) {
829 kprintf("kernel_trap() ignoring spurious trap 15\n");
830 return;
831 }
832 debugger_entry:
833 /* Ensure that the i386_kernel_state at the base of the
834 * current thread's stack (if any) is synchronized with the
835 * context at the moment of the trap, to facilitate
836 * access through the debugger.
837 */
838 sync_iss_to_iks(state);
839 #if MACH_KDB
840 restart_debugger:
841 #endif /* MACH_KDB */
842 #if MACH_KDP
843 if (current_debugger != KDB_CUR_DB) {
844 if (kdp_i386_trap(type, saved_state, result, (vm_offset_t)vaddr))
845 return;
846 } else {
847 #endif /* MACH_KDP */
848 #if MACH_KDB
849 if (kdb_trap(type, code, saved_state)) {
850 if (switch_debugger) {
851 current_debugger = KDP_CUR_DB;
852 switch_debugger = 0;
853 goto restart_debugger;
854 }
855 return;
856 }
857 #endif /* MACH_KDB */
858 #if MACH_KDP
859 }
860 #endif
861 }
862
863 panic_trap(saved_state);
864 /*
865 * NO RETURN
866 */
867 }
868
869
870 #ifdef __i386__
871 static void
872 set_recovery_ip(x86_saved_state32_t *saved_state, vm_offset_t ip)
873 {
874 saved_state->eip = ip;
875 }
876 #else
877 static void
878 set_recovery_ip(x86_saved_state64_t *saved_state, vm_offset_t ip)
879 {
880 saved_state->isf.rip = ip;
881 }
882 #endif
883
884
885 #ifdef __i386__
886 static void
887 panic_trap(x86_saved_state32_t *regs)
888 {
889 const char *trapname = "Unknown";
890 uint32_t cr0 = get_cr0();
891 uint32_t cr2 = get_cr2();
892 uint32_t cr3 = get_cr3();
893 uint32_t cr4 = get_cr4();
894 /*
895 * Issue an I/O port read if one has been requested - this is an
896 * event logic analyzers can use as a trigger point.
897 */
898 panic_io_port_read();
899
900 kprintf("panic trap number 0x%x, eip 0x%x\n", regs->trapno, regs->eip);
901 kprintf("cr0 0x%08x cr2 0x%08x cr3 0x%08x cr4 0x%08x\n",
902 cr0, cr2, cr3, cr4);
903
904 if (regs->trapno < TRAP_TYPES)
905 trapname = trap_type[regs->trapno];
906 #undef panic
907 panic("Kernel trap at 0x%08x, type %d=%s, registers:\n"
908 "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n"
909 "EAX: 0x%08x, EBX: 0x%08x, ECX: 0x%08x, EDX: 0x%08x\n"
910 "CR2: 0x%08x, EBP: 0x%08x, ESI: 0x%08x, EDI: 0x%08x\n"
911 "EFL: 0x%08x, EIP: 0x%08x, CS: 0x%08x, DS: 0x%08x\n"
912 "Error code: 0x%08x\n",
913 regs->eip, regs->trapno, trapname, cr0, cr2, cr3, cr4,
914 regs->eax,regs->ebx,regs->ecx,regs->edx,
915 regs->cr2,regs->ebp,regs->esi,regs->edi,
916 regs->efl,regs->eip,regs->cs, regs->ds, regs->err);
917 /*
918 * This next statement is not executed,
919 * but it's needed to stop the compiler using tail call optimization
920 * for the panic call - which confuses the subsequent backtrace.
921 */
922 cr0 = 0;
923 }
924 #else
925 static void
926 panic_trap(x86_saved_state64_t *regs)
927 {
928 const char *trapname = "Unknown";
929 uint64_t cr0 = get_cr0();
930 uint64_t cr2 = get_cr2();
931 uint64_t cr3 = get_cr3();
932 uint64_t cr4 = get_cr4();
933
934 /*
935 * Issue an I/O port read if one has been requested - this is an
936 * event logic analyzers can use as a trigger point.
937 */
938 panic_io_port_read();
939
940 kprintf("panic trap number 0x%x, rip 0x%016llx\n",
941 regs->isf.trapno, regs->isf.rip);
942 kprintf("cr0 0x%016llx cr2 0x%016llx cr3 0x%016llx cr4 0x%016llx\n",
943 cr0, cr2, cr3, cr4);
944
945 if (regs->isf.trapno < TRAP_TYPES)
946 trapname = trap_type[regs->isf.trapno];
947 #undef panic
948 panic("Kernel trap at 0x%016llx, type %d=%s, registers:\n"
949 "CR0: 0x%016llx, CR2: 0x%016llx, CR3: 0x%016llx, CR4: 0x%016llx\n"
950 "RAX: 0x%016llx, RBX: 0x%016llx, RCX: 0x%016llx, RDX: 0x%016llx\n"
951 "RSP: 0x%016llx, RBP: 0x%016llx, RSI: 0x%016llx, RDI: 0x%016llx\n"
952 "R8: 0x%016llx, R9: 0x%016llx, R10: 0x%016llx, R11: 0x%016llx\n"
953 "R12: 0x%016llx, R13: 0x%016llx, R14: 0x%016llx, R15: 0x%016llx\n"
954 "RFL: 0x%016llx, RIP: 0x%016llx, CS: 0x%016llx, SS: 0x%016llx\n"
955 "Error code: 0x%016llx\n",
956 regs->isf.rip, regs->isf.trapno, trapname,
957 cr0, cr2, cr3, cr4,
958 regs->rax, regs->rbx, regs->rcx, regs->rdx,
959 regs->isf.rsp, regs->rbp, regs->rsi, regs->rdi,
960 regs->r8, regs->r9, regs->r10, regs->r11,
961 regs->r12, regs->r13, regs->r14, regs->r15,
962 regs->isf.rflags, regs->isf.rip, regs->isf.cs, regs->isf.ss,
963 regs->isf.err);
964 /*
965 * This next statement is not executed,
966 * but it's needed to stop the compiler using tail call optimization
967 * for the panic call - which confuses the subsequent backtrace.
968 */
969 cr0 = 0;
970 }
971 #endif
972
973 extern void kprintf_break_lock(void);
974
975 #ifdef __i386__
976 static void
977 panic_32(__unused int code, __unused int pc, __unused const char *msg, boolean_t do_mca_dump, boolean_t do_bt)
978 {
979 struct i386_tss *my_ktss = current_ktss();
980
981 /* Set postcode (DEBUG only) */
982 postcode(pc);
983
984 /*
985 * Issue an I/O port read if one has been requested - this is an
986 * event logic analyzers can use as a trigger point.
987 */
988 panic_io_port_read();
989
990 /*
991 * Break kprintf lock in case of recursion,
992 * and record originally faulted instruction address.
993 */
994 kprintf_break_lock();
995
996 if (do_mca_dump) {
997 #if CONFIG_MCA
998 /*
999 * Dump the contents of the machine check MSRs (if any).
1000 */
1001 mca_dump();
1002 #endif
1003 }
1004
1005 #if MACH_KDP
1006 /*
1007 * Print backtrace leading to first fault:
1008 */
1009 if (do_bt)
1010 panic_i386_backtrace((void *) my_ktss->ebp, 10, NULL, FALSE, NULL);
1011 #endif
1012
1013 panic("%s at 0x%08x, thread:%p, code:0x%x, "
1014 "registers:\n"
1015 "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n"
1016 "EAX: 0x%08x, EBX: 0x%08x, ECX: 0x%08x, EDX: 0x%08x\n"
1017 "ESP: 0x%08x, EBP: 0x%08x, ESI: 0x%08x, EDI: 0x%08x\n"
1018 "EFL: 0x%08x, EIP: 0x%08x\n",
1019 msg,
1020 my_ktss->eip, current_thread(), code,
1021 (uint32_t)get_cr0(), (uint32_t)get_cr2(), (uint32_t)get_cr3(), (uint32_t)get_cr4(),
1022 my_ktss->eax, my_ktss->ebx, my_ktss->ecx, my_ktss->edx,
1023 my_ktss->esp, my_ktss->ebp, my_ktss->esi, my_ktss->edi,
1024 my_ktss->eflags, my_ktss->eip);
1025 }
1026
1027 /*
1028 * Called from locore on a special reserved stack after a double-fault
1029 * is taken in kernel space.
1030 * Kernel stack overflow is one route here.
1031 */
1032 void
1033 panic_double_fault32(int code)
1034 {
1035 panic_32(code, PANIC_DOUBLE_FAULT, "Double fault", FALSE, TRUE);
1036 }
1037
1038 /*
1039 * Called from locore on a special reserved stack after a machine-check
1040 */
1041 void
1042 panic_machine_check32(int code)
1043 {
1044 panic_32(code, PANIC_MACHINE_CHECK, "Machine-check", TRUE, FALSE);
1045 }
1046 #endif /* __i386__ */
1047
1048 static void
1049 panic_64(x86_saved_state_t *sp, __unused int pc, __unused const char *msg, boolean_t do_mca_dump)
1050 {
1051 /* Set postcode (DEBUG only) */
1052 postcode(pc);
1053
1054 /*
1055 * Issue an I/O port read if one has been requested - this is an
1056 * event logic analyzers can use as a trigger point.
1057 */
1058 panic_io_port_read();
1059
1060 /*
1061 * Break kprintf lock in case of recursion,
1062 * and record originally faulted instruction address.
1063 */
1064 kprintf_break_lock();
1065
1066 if (do_mca_dump) {
1067 #if CONFIG_MCA
1068 /*
1069 * Dump the contents of the machine check MSRs (if any).
1070 */
1071 mca_dump();
1072 #endif
1073 }
1074
1075 #ifdef __i386__
1076 /*
1077 * Dump the interrupt stack frame at last kernel entry.
1078 */
1079 if (is_saved_state64(sp)) {
1080 x86_saved_state64_t *ss64p = saved_state64(sp);
1081 panic("%s thread:%p, trapno:0x%x, err:0x%qx, "
1082 "registers:\n"
1083 "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n"
1084 "RAX: 0x%016qx, RBX: 0x%016qx, RCX: 0x%016qx, RDX: 0x%016qx\n"
1085 "RSP: 0x%016qx, RBP: 0x%016qx, RSI: 0x%016qx, RDI: 0x%016qx\n"
1086 "R8: 0x%016qx, R9: 0x%016qx, R10: 0x%016qx, R11: 0x%016qx\n"
1087 "R12: 0x%016qx, R13: 0x%016qx, R14: 0x%016qx, R15: 0x%016qx\n"
1088 "RFL: 0x%016qx, RIP: 0x%016qx, CR2: 0x%016qx\n",
1089 msg,
1090 current_thread(), ss64p->isf.trapno, ss64p->isf.err,
1091 (uint32_t)get_cr0(), (uint32_t)get_cr2(), (uint32_t)get_cr3(), (uint32_t)get_cr4(),
1092 ss64p->rax, ss64p->rbx, ss64p->rcx, ss64p->rdx,
1093 ss64p->isf.rsp, ss64p->rbp, ss64p->rsi, ss64p->rdi,
1094 ss64p->r8, ss64p->r9, ss64p->r10, ss64p->r11,
1095 ss64p->r12, ss64p->r13, ss64p->r14, ss64p->r15,
1096 ss64p->isf.rflags, ss64p->isf.rip, ss64p->cr2);
1097 } else {
1098 x86_saved_state32_t *ss32p = saved_state32(sp);
1099 panic("%s at 0x%08x, thread:%p, trapno:0x%x, err:0x%x,"
1100 "registers:\n"
1101 "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n"
1102 "EAX: 0x%08x, EBX: 0x%08x, ECX: 0x%08x, EDX: 0x%08x\n"
1103 "ESP: 0x%08x, EBP: 0x%08x, ESI: 0x%08x, EDI: 0x%08x\n"
1104 "EFL: 0x%08x, EIP: 0x%08x\n",
1105 msg,
1106 ss32p->eip, current_thread(), ss32p->trapno, ss32p->err,
1107 (uint32_t)get_cr0(), (uint32_t)get_cr2(), (uint32_t)get_cr3(), (uint32_t)get_cr4(),
1108 ss32p->eax, ss32p->ebx, ss32p->ecx, ss32p->edx,
1109 ss32p->uesp, ss32p->ebp, ss32p->esi, ss32p->edi,
1110 ss32p->efl, ss32p->eip);
1111 }
1112 #else
1113 x86_saved_state64_t *regs = saved_state64(sp);
1114 panic("%s thread:%p at 0x%016llx, registers:\n"
1115 "CR0: 0x%016lx, CR2: 0x%016lx, CR3: 0x%016lx, CR4: 0x%016lx\n"
1116 "RAX: 0x%016llx, RBX: 0x%016llx, RCX: 0x%016llx, RDX: 0x%016llx\n"
1117 "RSP: 0x%016llx, RBP: 0x%016llx, RSI: 0x%016llx, RDI: 0x%016llx\n"
1118 "R8: 0x%016llx, R9: 0x%016llx, R10: 0x%016llx, R11: 0x%016llx\n"
1119 "R12: 0x%016llx, R13: 0x%016llx, R14: 0x%016llx, R15: 0x%016llx\n"
1120 "RFL: 0x%016llx, RIP: 0x%016llx, CS: 0x%016llx, SS: 0x%016llx\n"
1121 "Error code: 0x%016llx\n",
1122 msg,
1123 current_thread(), regs->isf.rip,
1124 get_cr0(), get_cr2(), get_cr3(), get_cr4(),
1125 regs->rax, regs->rbx, regs->rcx, regs->rdx,
1126 regs->isf.rsp, regs->rbp, regs->rsi, regs->rdi,
1127 regs->r8, regs->r9, regs->r10, regs->r11,
1128 regs->r12, regs->r13, regs->r14, regs->r15,
1129 regs->isf.rflags, regs->isf.rip, regs->isf.cs, regs->isf.ss,
1130 regs->isf.err);
1131 #endif
1132 }
1133
1134 void
1135 panic_double_fault64(x86_saved_state_t *sp)
1136 {
1137 panic_64(sp, PANIC_DOUBLE_FAULT, "Double fault", FALSE);
1138
1139 }
1140 void
1141
1142 panic_machine_check64(x86_saved_state_t *sp)
1143 {
1144 panic_64(sp, PANIC_MACHINE_CHECK, "Machine Check", TRUE);
1145
1146 }
1147
1148 #if CONFIG_DTRACE
1149 extern kern_return_t dtrace_user_probe(x86_saved_state_t *);
1150 #endif
1151
1152 /*
1153 * Trap from user mode.
1154 */
1155 void
1156 user_trap(
1157 x86_saved_state_t *saved_state)
1158 {
1159 int exc;
1160 int err;
1161 mach_exception_code_t code;
1162 mach_exception_subcode_t subcode;
1163 int type;
1164 user_addr_t vaddr;
1165 vm_prot_t prot;
1166 thread_t thread = current_thread();
1167 ast_t *myast;
1168 kern_return_t kret;
1169 user_addr_t rip;
1170
1171 assert((is_saved_state32(saved_state) && !thread_is_64bit(thread)) ||
1172 (is_saved_state64(saved_state) && thread_is_64bit(thread)));
1173
1174 if (is_saved_state64(saved_state)) {
1175 x86_saved_state64_t *regs;
1176
1177 regs = saved_state64(saved_state);
1178
1179 type = regs->isf.trapno;
1180 err = (int)regs->isf.err & 0xffff;
1181 vaddr = (user_addr_t)regs->cr2;
1182 rip = (user_addr_t)regs->isf.rip;
1183 } else {
1184 x86_saved_state32_t *regs;
1185
1186 regs = saved_state32(saved_state);
1187
1188 type = regs->trapno;
1189 err = regs->err & 0xffff;
1190 vaddr = (user_addr_t)regs->cr2;
1191 rip = (user_addr_t)regs->eip;
1192 }
1193
1194 KERNEL_DEBUG_CONSTANT(
1195 (MACHDBG_CODE(DBG_MACH_EXCP_UTRAP_x86, type)) | DBG_FUNC_NONE,
1196 (unsigned)(vaddr>>32), (unsigned)vaddr,
1197 (unsigned)(rip>>32), (unsigned)rip, 0);
1198
1199 code = 0;
1200 subcode = 0;
1201 exc = 0;
1202
1203 #if DEBUG_TRACE
1204 kprintf("user_trap(0x%08x) type=%d vaddr=0x%016llx\n",
1205 saved_state, type, vaddr);
1206 #endif
1207 myast = ast_pending();
1208 perfCallback fn = perfASTHook;
1209 if (fn) {
1210 if (*myast & AST_CHUD_ALL) {
1211 fn(type, saved_state, 0, 0);
1212 }
1213 } else {
1214 *myast &= ~AST_CHUD_ALL;
1215 }
1216
1217 /* Is there a hook? */
1218 fn = perfTrapHook;
1219 if (fn) {
1220 if (fn(type, saved_state, 0, 0) == KERN_SUCCESS)
1221 return; /* If it succeeds, we are done... */
1222 }
1223
1224 /*
1225 * DTrace does not consume all user traps, only INT_3's for now.
1226 * Avoid needlessly calling tempDTraceTrapHook here, and let the
1227 * INT_3 case handle them.
1228 */
1229 DEBUG_KPRINT_SYSCALL_MASK(1,
1230 "user_trap: type=0x%x(%s) err=0x%x cr2=%p rip=%p\n",
1231 type, trap_type[type], err, (void *)(long) vaddr, (void *)(long) rip);
1232
1233 switch (type) {
1234
1235 case T_DIVIDE_ERROR:
1236 exc = EXC_ARITHMETIC;
1237 code = EXC_I386_DIV;
1238 break;
1239
1240 case T_DEBUG:
1241 {
1242 pcb_t pcb;
1243 long clear = 0; /* 32 bit for i386, 64 bit for x86_64 */
1244 /*
1245 * get dr6 and set it in the thread's pcb before
1246 * returning to userland
1247 */
1248 pcb = thread->machine.pcb;
1249 if (pcb->ids) {
1250 /*
1251 * We can get and set the status register
1252 * in 32-bit mode even on a 64-bit thread
1253 * because the high order bits are not
1254 * used on x86_64
1255 */
1256 unsigned long dr6_temp; /* 32 bit for i386, 64 bit for x86_64 */
1257 __asm__ volatile ("mov %%db6, %0" : "=r" (dr6_temp)); /* Register constraint by necessity */
1258 if (thread_is_64bit(thread)) {
1259 x86_debug_state64_t *ids = pcb->ids;
1260 ids->dr6 = dr6_temp;
1261 } else { /* 32 bit thread */
1262 x86_debug_state32_t *ids = pcb->ids;
1263 ids->dr6 = (uint32_t) dr6_temp;
1264 }
1265 __asm__ volatile ("mov %0, %%db6" : : "r" (clear));
1266 }
1267 exc = EXC_BREAKPOINT;
1268 code = EXC_I386_SGL;
1269 break;
1270 }
1271 case T_INT3:
1272 #if CONFIG_DTRACE
1273 if (dtrace_user_probe(saved_state) == KERN_SUCCESS)
1274 return; /* If it succeeds, we are done... */
1275 #endif
1276 exc = EXC_BREAKPOINT;
1277 code = EXC_I386_BPT;
1278 break;
1279
1280 case T_OVERFLOW:
1281 exc = EXC_ARITHMETIC;
1282 code = EXC_I386_INTO;
1283 break;
1284
1285 case T_OUT_OF_BOUNDS:
1286 exc = EXC_SOFTWARE;
1287 code = EXC_I386_BOUND;
1288 break;
1289
1290 case T_INVALID_OPCODE:
1291 exc = EXC_BAD_INSTRUCTION;
1292 code = EXC_I386_INVOP;
1293 break;
1294
1295 case T_NO_FPU:
1296 fpnoextflt();
1297 return;
1298
1299 case T_FPU_FAULT:
1300 fpextovrflt(); /* Propagates exception directly, doesn't return */
1301 return;
1302
1303 case T_INVALID_TSS: /* invalid TSS == iret with NT flag set */
1304 exc = EXC_BAD_INSTRUCTION;
1305 code = EXC_I386_INVTSSFLT;
1306 subcode = err;
1307 break;
1308
1309 case T_SEGMENT_NOT_PRESENT:
1310 exc = EXC_BAD_INSTRUCTION;
1311 code = EXC_I386_SEGNPFLT;
1312 subcode = err;
1313 break;
1314
1315 case T_STACK_FAULT:
1316 exc = EXC_BAD_INSTRUCTION;
1317 code = EXC_I386_STKFLT;
1318 subcode = err;
1319 break;
1320
1321 case T_GENERAL_PROTECTION:
1322 /*
1323 * There's a wide range of circumstances which generate this
1324 * class of exception. From user-space, many involve bad
1325 * addresses (such as a non-canonical 64-bit address).
1326 * So we map this to EXC_BAD_ACCESS (and thereby SIGSEGV).
1327 * The trouble is cr2 doesn't contain the faulting address;
1328 * we'd need to decode the faulting instruction to really
1329 * determine this. We'll leave that to debuggers.
1330 * However, attempted execution of privileged instructions
1331 * (e.g. cli) also generate GP faults and so we map these to
1332 * to EXC_BAD_ACCESS (and thence SIGSEGV) also - rather than
1333 * EXC_BAD_INSTRUCTION which is more accurate. We just can't
1334 * win!
1335 */
1336 exc = EXC_BAD_ACCESS;
1337 code = EXC_I386_GPFLT;
1338 subcode = err;
1339 break;
1340
1341 case T_PAGE_FAULT:
1342 prot = VM_PROT_READ;
1343
1344 if (err & T_PF_WRITE)
1345 prot |= VM_PROT_WRITE;
1346 #if PAE
1347 if (err & T_PF_EXECUTE)
1348 prot |= VM_PROT_EXECUTE;
1349 #endif
1350 kret = vm_fault(thread->map, vm_map_trunc_page(vaddr),
1351 prot, FALSE,
1352 THREAD_ABORTSAFE, NULL, 0);
1353
1354 user_page_fault_continue(kret);
1355
1356 /* NOTREACHED */
1357 break;
1358
1359 case T_SSE_FLOAT_ERROR:
1360 fpSSEexterrflt(); /* Propagates exception directly, doesn't return */
1361 return;
1362
1363
1364 case T_FLOATING_POINT_ERROR:
1365 fpexterrflt(); /* Propagates exception directly, doesn't return */
1366 return;
1367
1368 case T_DTRACE_RET:
1369 #if CONFIG_DTRACE
1370 if (dtrace_user_probe(saved_state) == KERN_SUCCESS)
1371 return; /* If it succeeds, we are done... */
1372 #endif
1373 /*
1374 * If we get an INT 0x7f when we do not expect to,
1375 * treat it as an illegal instruction
1376 */
1377 exc = EXC_BAD_INSTRUCTION;
1378 code = EXC_I386_INVOP;
1379 break;
1380
1381 default:
1382 #if MACH_KGDB
1383 Debugger("Unanticipated user trap");
1384 return;
1385 #endif /* MACH_KGDB */
1386 #if MACH_KDB
1387 if (kdb_trap(type, err, saved_state32(saved_state)))
1388 return;
1389 #endif /* MACH_KDB */
1390 panic("Unexpected user trap, type %d", type);
1391 return;
1392 }
1393 /* Note: Codepaths that directly return from user_trap() have pending
1394 * ASTs processed in locore
1395 */
1396 i386_exception(exc, code, subcode);
1397 /* NOTREACHED */
1398 }
1399
1400
1401 /*
1402 * Handle AST traps for i386.
1403 * Check for delayed floating-point exception from
1404 * AT-bus machines.
1405 */
1406
1407 extern void log_thread_action (thread_t, char *);
1408
1409 void
1410 i386_astintr(int preemption)
1411 {
1412 ast_t mask = AST_ALL;
1413 spl_t s;
1414
1415 if (preemption)
1416 mask = AST_PREEMPTION;
1417
1418 s = splsched();
1419
1420 ast_taken(mask, s);
1421
1422 splx(s);
1423 }
1424
1425 /*
1426 * Handle exceptions for i386.
1427 *
1428 * If we are an AT bus machine, we must turn off the AST for a
1429 * delayed floating-point exception.
1430 *
1431 * If we are providing floating-point emulation, we may have
1432 * to retrieve the real register values from the floating point
1433 * emulator.
1434 */
1435 void
1436 i386_exception(
1437 int exc,
1438 mach_exception_code_t code,
1439 mach_exception_subcode_t subcode)
1440 {
1441 mach_exception_data_type_t codes[EXCEPTION_CODE_MAX];
1442
1443 DEBUG_KPRINT_SYSCALL_MACH("i386_exception: exc=%d code=0x%llx subcode=0x%llx\n",
1444 exc, code, subcode);
1445 codes[0] = code; /* new exception interface */
1446 codes[1] = subcode;
1447 exception_triage(exc, codes, 2);
1448 /*NOTREACHED*/
1449 }
1450
1451
1452
1453 void
1454 kernel_preempt_check(void)
1455 {
1456 ast_t *myast;
1457 boolean_t intr;
1458
1459 /*
1460 * disable interrupts to both prevent pre-emption
1461 * and to keep the ast state from changing via
1462 * an interrupt handler making something runnable
1463 */
1464 intr = ml_set_interrupts_enabled(FALSE);
1465
1466 myast = ast_pending();
1467
1468 if ((*myast & AST_URGENT) && intr == TRUE && get_interrupt_level() == 0) {
1469 /*
1470 * can handle interrupts and preemptions
1471 * at this point
1472 */
1473 ml_set_interrupts_enabled(intr);
1474
1475 /*
1476 * now cause the PRE-EMPTION trap
1477 */
1478 __asm__ volatile (" int $0xff");
1479 } else {
1480 /*
1481 * if interrupts were already disabled or
1482 * we're in an interrupt context, we can't
1483 * preempt... of course if AST_URGENT
1484 * isn't set we also don't want to
1485 */
1486 ml_set_interrupts_enabled(intr);
1487 }
1488 }
1489
1490 #if MACH_KDB
1491
1492 extern void db_i386_state(x86_saved_state32_t *regs);
1493
1494 #include <ddb/db_output.h>
1495
1496 void
1497 db_i386_state(
1498 x86_saved_state32_t *regs)
1499 {
1500 db_printf("eip %8x\n", regs->eip);
1501 db_printf("trap %8x\n", regs->trapno);
1502 db_printf("err %8x\n", regs->err);
1503 db_printf("efl %8x\n", regs->efl);
1504 db_printf("ebp %8x\n", regs->ebp);
1505 db_printf("esp %8x\n", regs->cr2);
1506 db_printf("uesp %8x\n", regs->uesp);
1507 db_printf("cs %8x\n", regs->cs & 0xff);
1508 db_printf("ds %8x\n", regs->ds & 0xff);
1509 db_printf("es %8x\n", regs->es & 0xff);
1510 db_printf("fs %8x\n", regs->fs & 0xff);
1511 db_printf("gs %8x\n", regs->gs & 0xff);
1512 db_printf("ss %8x\n", regs->ss & 0xff);
1513 db_printf("eax %8x\n", regs->eax);
1514 db_printf("ebx %8x\n", regs->ebx);
1515 db_printf("ecx %8x\n", regs->ecx);
1516 db_printf("edx %8x\n", regs->edx);
1517 db_printf("esi %8x\n", regs->esi);
1518 db_printf("edi %8x\n", regs->edi);
1519 }
1520
1521 #endif /* MACH_KDB */
1522
1523 /* Synchronize a thread's i386_kernel_state (if any) with the given
1524 * i386_saved_state_t obtained from the trap/IPI handler; called in
1525 * kernel_trap() prior to entering the debugger, and when receiving
1526 * an "MP_KDP" IPI.
1527 */
1528
1529 void
1530 sync_iss_to_iks(x86_saved_state_t *saved_state)
1531 {
1532 struct x86_kernel_state *iks;
1533 vm_offset_t kstack;
1534 boolean_t record_active_regs = FALSE;
1535
1536 if ((kstack = current_thread()->kernel_stack) != 0) {
1537 #ifdef __i386__
1538 x86_saved_state32_t *regs = saved_state32(saved_state);
1539 #else
1540 x86_saved_state64_t *regs = saved_state64(saved_state);
1541 #endif
1542
1543 iks = STACK_IKS(kstack);
1544
1545
1546 /* Did we take the trap/interrupt in kernel mode? */
1547 #ifdef __i386__
1548 if (regs == USER_REGS32(current_thread()))
1549 record_active_regs = TRUE;
1550 else {
1551 iks->k_ebx = regs->ebx;
1552 iks->k_esp = (int)regs;
1553 iks->k_ebp = regs->ebp;
1554 iks->k_edi = regs->edi;
1555 iks->k_esi = regs->esi;
1556 iks->k_eip = regs->eip;
1557 }
1558 #else
1559 if (regs == USER_REGS64(current_thread()))
1560 record_active_regs = TRUE;
1561 else {
1562 iks->k_rbx = regs->rbx;
1563 iks->k_rsp = regs->isf.rsp;
1564 iks->k_rbp = regs->rbp;
1565 iks->k_r12 = regs->r12;
1566 iks->k_r13 = regs->r13;
1567 iks->k_r14 = regs->r14;
1568 iks->k_r15 = regs->r15;
1569 iks->k_rip = regs->isf.rip;
1570 }
1571 #endif
1572 }
1573
1574 if (record_active_regs == TRUE) {
1575 #ifdef __i386__
1576 /* Show the trap handler path */
1577 __asm__ volatile("movl %%ebx, %0" : "=m" (iks->k_ebx));
1578 __asm__ volatile("movl %%esp, %0" : "=m" (iks->k_esp));
1579 __asm__ volatile("movl %%ebp, %0" : "=m" (iks->k_ebp));
1580 __asm__ volatile("movl %%edi, %0" : "=m" (iks->k_edi));
1581 __asm__ volatile("movl %%esi, %0" : "=m" (iks->k_esi));
1582 /* "Current" instruction pointer */
1583 __asm__ volatile("movl $1f, %0\n1:" : "=m" (iks->k_eip));
1584 #else
1585 /* Show the trap handler path */
1586 __asm__ volatile("movq %%rbx, %0" : "=m" (iks->k_rbx));
1587 __asm__ volatile("movq %%rsp, %0" : "=m" (iks->k_rsp));
1588 __asm__ volatile("movq %%rbp, %0" : "=m" (iks->k_rbp));
1589 __asm__ volatile("movq %%r12, %0" : "=m" (iks->k_r12));
1590 __asm__ volatile("movq %%r13, %0" : "=m" (iks->k_r13));
1591 __asm__ volatile("movq %%r14, %0" : "=m" (iks->k_r14));
1592 __asm__ volatile("movq %%r15, %0" : "=m" (iks->k_r15));
1593 /* "Current" instruction pointer */
1594 __asm__ volatile("leaq 1f(%%rip), %%rax; mov %%rax, %0\n1:"
1595 : "=m" (iks->k_rip)
1596 :
1597 : "rax");
1598 #endif
1599 }
1600 }
1601
1602 /*
1603 * This is used by the NMI interrupt handler (from mp.c) to
1604 * uncondtionally sync the trap handler context to the IKS
1605 * irrespective of whether the NMI was fielded in kernel
1606 * or user space.
1607 */
1608 void
1609 sync_iss_to_iks_unconditionally(__unused x86_saved_state_t *saved_state) {
1610 struct x86_kernel_state *iks;
1611 vm_offset_t kstack;
1612
1613 if ((kstack = current_thread()->kernel_stack) != 0) {
1614 iks = STACK_IKS(kstack);
1615 #ifdef __i386__
1616 /* Display the trap handler path */
1617 __asm__ volatile("movl %%ebx, %0" : "=m" (iks->k_ebx));
1618 __asm__ volatile("movl %%esp, %0" : "=m" (iks->k_esp));
1619 __asm__ volatile("movl %%ebp, %0" : "=m" (iks->k_ebp));
1620 __asm__ volatile("movl %%edi, %0" : "=m" (iks->k_edi));
1621 __asm__ volatile("movl %%esi, %0" : "=m" (iks->k_esi));
1622 /* "Current" instruction pointer */
1623 __asm__ volatile("movl $1f, %0\n1:" : "=m" (iks->k_eip));
1624 #else
1625 /* Display the trap handler path */
1626 __asm__ volatile("movq %%rbx, %0" : "=m" (iks->k_rbx));
1627 __asm__ volatile("movq %%rsp, %0" : "=m" (iks->k_rsp));
1628 __asm__ volatile("movq %%rbp, %0" : "=m" (iks->k_rbp));
1629 __asm__ volatile("movq %%r12, %0" : "=m" (iks->k_r12));
1630 __asm__ volatile("movq %%r13, %0" : "=m" (iks->k_r13));
1631 __asm__ volatile("movq %%r14, %0" : "=m" (iks->k_r14));
1632 __asm__ volatile("movq %%r15, %0" : "=m" (iks->k_r15));
1633 /* "Current" instruction pointer */
1634 __asm__ volatile("leaq 1f(%%rip), %%rax; mov %%rax, %0\n1:" : "=m" (iks->k_rip)::"rax");
1635 #endif
1636 }
1637 }
mirror of XNU