]> git.saurik.com Git - apple/xnu.git/blob - osfmk/i386/trap.c
projects / apple / xnu.git / blob
? search:


2a77aedf3903c551fdcaff9f03c186177587369c
[apple/xnu.git] / osfmk / i386 / trap.c
1 /*
2 * Copyright (c) 2000-2010 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58
59 /*
60 * Hardware trap/fault handler.
61 */
62
63 #include <mach_kdb.h>
64 #include <mach_kgdb.h>
65 #include <mach_kdp.h>
66 #include <mach_ldebug.h>
67
68 #include <types.h>
69 #include <i386/eflags.h>
70 #include <i386/trap.h>
71 #include <i386/pmap.h>
72 #include <i386/fpu.h>
73 #include <i386/misc_protos.h> /* panic_io_port_read() */
74 #include <i386/lapic.h>
75
76 #include <mach/exception.h>
77 #include <mach/kern_return.h>
78 #include <mach/vm_param.h>
79 #include <mach/i386/thread_status.h>
80
81 #include <vm/vm_kern.h>
82 #include <vm/vm_fault.h>
83
84 #include <kern/kern_types.h>
85 #include <kern/processor.h>
86 #include <kern/thread.h>
87 #include <kern/task.h>
88 #include <kern/sched.h>
89 #include <kern/sched_prim.h>
90 #include <kern/exception.h>
91 #include <kern/spl.h>
92 #include <kern/misc_protos.h>
93 #include <kern/debug.h>
94
95 #include <sys/kdebug.h>
96
97 #if MACH_KGDB
98 #include <kgdb/kgdb_defs.h>
99 #endif /* MACH_KGDB */
100
101 #if MACH_KDB
102 #include <debug.h>
103 #include <ddb/db_watch.h>
104 #include <ddb/db_run.h>
105 #include <ddb/db_break.h>
106 #include <ddb/db_trap.h>
107 #endif /* MACH_KDB */
108
109 #include <string.h>
110
111 #include <i386/postcode.h>
112 #include <i386/mp_desc.h>
113 #include <i386/proc_reg.h>
114 #if CONFIG_MCA
115 #include <i386/machine_check.h>
116 #endif
117 #include <mach/i386/syscall_sw.h>
118
119 #include <libkern/OSDebug.h>
120
121 #include <machine/pal_routines.h>
122
123 extern void throttle_lowpri_io(int);
124 extern void kprint_state(x86_saved_state64_t *saved_state);
125
126 /*
127 * Forward declarations
128 */
129 static void user_page_fault_continue(kern_return_t kret);
130 #ifdef __i386__
131 static void panic_trap(x86_saved_state32_t *saved_state);
132 static void set_recovery_ip(x86_saved_state32_t *saved_state, vm_offset_t ip);
133 extern void panic_64(x86_saved_state_t *, int, const char *, boolean_t);
134 #else
135 static void panic_trap(x86_saved_state64_t *saved_state);
136 static void set_recovery_ip(x86_saved_state64_t *saved_state, vm_offset_t ip);
137 #endif
138
139 volatile perfCallback perfTrapHook = NULL; /* Pointer to CHUD trap hook routine */
140
141 #if CONFIG_DTRACE
142 /* See <rdar://problem/4613924> */
143 perfCallback tempDTraceTrapHook = NULL; /* Pointer to DTrace fbt trap hook routine */
144
145 extern boolean_t dtrace_tally_fault(user_addr_t);
146 #endif
147
148 extern boolean_t pmap_smep_enabled;
149
150 void
151 thread_syscall_return(
152 kern_return_t ret)
153 {
154 thread_t thr_act = current_thread();
155 boolean_t is_mach;
156 int code;
157
158 pal_register_cache_state(thr_act, DIRTY);
159
160 if (thread_is_64bit(thr_act)) {
161 x86_saved_state64_t *regs;
162
163 regs = USER_REGS64(thr_act);
164
165 code = (int) (regs->rax & SYSCALL_NUMBER_MASK);
166 is_mach = (regs->rax & SYSCALL_CLASS_MASK)
167 == (SYSCALL_CLASS_MACH << SYSCALL_CLASS_SHIFT);
168 if (kdebug_enable && is_mach) {
169 /* Mach trap */
170 KERNEL_DEBUG_CONSTANT(
171 MACHDBG_CODE(DBG_MACH_EXCP_SC,code)|DBG_FUNC_END,
172 ret, 0, 0, 0, 0);
173 }
174 regs->rax = ret;
175 #if DEBUG
176 if (is_mach)
177 DEBUG_KPRINT_SYSCALL_MACH(
178 "thread_syscall_return: 64-bit mach ret=%u\n",
179 ret);
180 else
181 DEBUG_KPRINT_SYSCALL_UNIX(
182 "thread_syscall_return: 64-bit unix ret=%u\n",
183 ret);
184 #endif
185 } else {
186 x86_saved_state32_t *regs;
187
188 regs = USER_REGS32(thr_act);
189
190 code = ((int) regs->eax);
191 is_mach = (code < 0);
192 if (kdebug_enable && is_mach) {
193 /* Mach trap */
194 KERNEL_DEBUG_CONSTANT(
195 MACHDBG_CODE(DBG_MACH_EXCP_SC,-code)|DBG_FUNC_END,
196 ret, 0, 0, 0, 0);
197 }
198 regs->eax = ret;
199 #if DEBUG
200 if (is_mach)
201 DEBUG_KPRINT_SYSCALL_MACH(
202 "thread_syscall_return: 32-bit mach ret=%u\n",
203 ret);
204 else
205 DEBUG_KPRINT_SYSCALL_UNIX(
206 "thread_syscall_return: 32-bit unix ret=%u\n",
207 ret);
208 #endif
209 }
210 throttle_lowpri_io(TRUE);
211
212 thread_exception_return();
213 /*NOTREACHED*/
214 }
215
216
217 #if MACH_KDB
218 boolean_t debug_all_traps_with_kdb = FALSE;
219 extern struct db_watchpoint *db_watchpoint_list;
220 extern boolean_t db_watchpoints_inserted;
221 extern boolean_t db_breakpoints_inserted;
222
223 void
224 thread_kdb_return(void)
225 {
226 thread_t thr_act = current_thread();
227 x86_saved_state_t *iss = USER_STATE(thr_act);
228
229 pal_register_cache_state(thr_act, DIRTY);
230
231 if (is_saved_state64(iss)) {
232 x86_saved_state64_t *regs;
233
234 regs = saved_state64(iss);
235
236 if (kdb_trap(regs->isf.trapno, (int)regs->isf.err, (void *)regs)) {
237 thread_exception_return();
238 /*NOTREACHED*/
239 }
240
241 } else {
242 x86_saved_state32_t *regs;
243
244 regs = saved_state32(iss);
245
246 if (kdb_trap(regs->trapno, regs->err, (void *)regs)) {
247 thread_exception_return();
248 /*NOTREACHED*/
249 }
250 }
251 }
252
253 #endif /* MACH_KDB */
254
255 static inline void
256 user_page_fault_continue(
257 kern_return_t kr)
258 {
259 thread_t thread = current_thread();
260 user_addr_t vaddr;
261
262 #if MACH_KDB
263 x86_saved_state_t *regs = USER_STATE(thread);
264 int err;
265 int trapno;
266
267 assert((is_saved_state32(regs) && !thread_is_64bit(thread)) ||
268 (is_saved_state64(regs) && thread_is_64bit(thread)));
269 #endif
270
271 if (thread_is_64bit(thread)) {
272 x86_saved_state64_t *uregs;
273
274 uregs = USER_REGS64(thread);
275
276 #if MACH_KDB
277 trapno = uregs->isf.trapno;
278 err = (int)uregs->isf.err;
279 #endif
280 vaddr = (user_addr_t)uregs->cr2;
281 } else {
282 x86_saved_state32_t *uregs;
283
284 uregs = USER_REGS32(thread);
285
286 #if MACH_KDB
287 trapno = uregs->trapno;
288 err = uregs->err;
289 #endif
290 vaddr = uregs->cr2;
291 }
292
293 if (__probable((kr == KERN_SUCCESS) || (kr == KERN_ABORTED))) {
294 #if MACH_KDB
295 if (!db_breakpoints_inserted) {
296 db_set_breakpoints();
297 }
298 if (db_watchpoint_list &&
299 db_watchpoints_inserted &&
300 (err & T_PF_WRITE) &&
301 db_find_watchpoint(thread->map,
302 (vm_offset_t)vaddr,
303 saved_state32(regs)))
304 kdb_trap(T_WATCHPOINT, 0, saved_state32(regs));
305 #endif /* MACH_KDB */
306 thread_exception_return();
307 /*NOTREACHED*/
308 }
309
310 #if MACH_KDB
311 if (debug_all_traps_with_kdb &&
312 kdb_trap(trapno, err, saved_state32(regs))) {
313 thread_exception_return();
314 /*NOTREACHED*/
315 }
316 #endif /* MACH_KDB */
317
318 /* PAL debug hook */
319 pal_dbg_page_fault( thread, vaddr, kr );
320
321 i386_exception(EXC_BAD_ACCESS, kr, vaddr);
322 /*NOTREACHED*/
323 }
324
325 /*
326 * Fault recovery in copyin/copyout routines.
327 */
328 struct recovery {
329 uintptr_t fault_addr;
330 uintptr_t recover_addr;
331 };
332
333 extern struct recovery recover_table[];
334 extern struct recovery recover_table_end[];
335
336 const char * trap_type[] = {TRAP_NAMES};
337 unsigned TRAP_TYPES = sizeof(trap_type)/sizeof(trap_type[0]);
338
339 extern void PE_incoming_interrupt(int interrupt);
340
341 #if defined(__x86_64__) && DEBUG
342 void
343 kprint_state(x86_saved_state64_t *saved_state)
344 {
345 kprintf("current_cpu_datap() 0x%lx\n", (uintptr_t)current_cpu_datap());
346 kprintf("Current GS base MSR 0x%llx\n", rdmsr64(MSR_IA32_GS_BASE));
347 kprintf("Kernel GS base MSR 0x%llx\n", rdmsr64(MSR_IA32_KERNEL_GS_BASE));
348 kprintf("state at 0x%lx:\n", (uintptr_t) saved_state);
349
350 kprintf(" rdi 0x%llx\n", saved_state->rdi);
351 kprintf(" rsi 0x%llx\n", saved_state->rsi);
352 kprintf(" rdx 0x%llx\n", saved_state->rdx);
353 kprintf(" r10 0x%llx\n", saved_state->r10);
354 kprintf(" r8 0x%llx\n", saved_state->r8);
355 kprintf(" r9 0x%llx\n", saved_state->r9);
356 kprintf(" v_arg6 0x%llx\n", saved_state->v_arg6);
357 kprintf(" v_arg7 0x%llx\n", saved_state->v_arg7);
358 kprintf(" v_arg8 0x%llx\n", saved_state->v_arg8);
359
360 kprintf(" cr2 0x%llx\n", saved_state->cr2);
361 kprintf("real cr2 0x%lx\n", get_cr2());
362 kprintf(" r15 0x%llx\n", saved_state->r15);
363 kprintf(" r14 0x%llx\n", saved_state->r14);
364 kprintf(" r13 0x%llx\n", saved_state->r13);
365 kprintf(" r12 0x%llx\n", saved_state->r12);
366 kprintf(" r11 0x%llx\n", saved_state->r11);
367 kprintf(" rbp 0x%llx\n", saved_state->rbp);
368 kprintf(" rbx 0x%llx\n", saved_state->rbx);
369 kprintf(" rcx 0x%llx\n", saved_state->rcx);
370 kprintf(" rax 0x%llx\n", saved_state->rax);
371
372 kprintf(" gs 0x%x\n", saved_state->gs);
373 kprintf(" fs 0x%x\n", saved_state->fs);
374
375 kprintf(" isf.trapno 0x%x\n", saved_state->isf.trapno);
376 kprintf(" isf._pad 0x%x\n", saved_state->isf._pad);
377 kprintf(" isf.trapfn 0x%llx\n", saved_state->isf.trapfn);
378 kprintf(" isf.err 0x%llx\n", saved_state->isf.err);
379 kprintf(" isf.rip 0x%llx\n", saved_state->isf.rip);
380 kprintf(" isf.cs 0x%llx\n", saved_state->isf.cs);
381 kprintf(" isf.rflags 0x%llx\n", saved_state->isf.rflags);
382 kprintf(" isf.rsp 0x%llx\n", saved_state->isf.rsp);
383 kprintf(" isf.ss 0x%llx\n", saved_state->isf.ss);
384 }
385 #endif
386
387
388 /*
389 * Non-zero indicates latency assert is enabled and capped at valued
390 * absolute time units.
391 */
392
393 uint64_t interrupt_latency_cap = 0;
394 boolean_t ilat_assert = FALSE;
395
396 void
397 interrupt_latency_tracker_setup(void) {
398 uint32_t ilat_cap_us;
399 if (PE_parse_boot_argn("interrupt_latency_cap_us", &ilat_cap_us, sizeof(ilat_cap_us))) {
400 interrupt_latency_cap = ilat_cap_us * NSEC_PER_USEC;
401 nanoseconds_to_absolutetime(interrupt_latency_cap, &interrupt_latency_cap);
402 } else {
403 interrupt_latency_cap = LockTimeOut;
404 }
405 PE_parse_boot_argn("-interrupt_latency_assert_enable", &ilat_assert, sizeof(ilat_assert));
406 }
407
408 void interrupt_reset_latency_stats(void) {
409 uint32_t i;
410 for (i = 0; i < real_ncpus; i++) {
411 cpu_data_ptr[i]->cpu_max_observed_int_latency =
412 cpu_data_ptr[i]->cpu_max_observed_int_latency_vector = 0;
413 }
414 }
415
416 void interrupt_populate_latency_stats(char *buf, unsigned bufsize) {
417 uint32_t i, tcpu = ~0;
418 uint64_t cur_max = 0;
419
420 for (i = 0; i < real_ncpus; i++) {
421 if (cur_max < cpu_data_ptr[i]->cpu_max_observed_int_latency) {
422 cur_max = cpu_data_ptr[i]->cpu_max_observed_int_latency;
423 tcpu = i;
424 }
425 }
426
427 if (tcpu < real_ncpus)
428 snprintf(buf, bufsize, "0x%x 0x%x 0x%llx", tcpu, cpu_data_ptr[tcpu]->cpu_max_observed_int_latency_vector, cpu_data_ptr[tcpu]->cpu_max_observed_int_latency);
429 }
430
431 /*
432 * Handle interrupts:
433 * - local APIC interrupts (IPIs, timers, etc) are handled by the kernel,
434 * - device interrupts go to the platform expert.
435 */
436 void
437 interrupt(x86_saved_state_t *state)
438 {
439 uint64_t rip;
440 uint64_t rsp;
441 int interrupt_num;
442 boolean_t user_mode = FALSE;
443 int ipl;
444 int cnum = cpu_number();
445
446 if (is_saved_state64(state) == TRUE) {
447 x86_saved_state64_t *state64;
448
449 state64 = saved_state64(state);
450 rip = state64->isf.rip;
451 rsp = state64->isf.rsp;
452 interrupt_num = state64->isf.trapno;
453 #ifdef __x86_64__
454 if(state64->isf.cs & 0x03)
455 #endif
456 user_mode = TRUE;
457 } else {
458 x86_saved_state32_t *state32;
459
460 state32 = saved_state32(state);
461 if (state32->cs & 0x03)
462 user_mode = TRUE;
463 rip = state32->eip;
464 rsp = state32->uesp;
465 interrupt_num = state32->trapno;
466 }
467
468 KERNEL_DEBUG_CONSTANT(
469 MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_START,
470 interrupt_num, rip, user_mode, 0, 0);
471
472 SCHED_STATS_INTERRUPT(current_processor());
473
474 ipl = get_preemption_level();
475
476 /*
477 * Handle local APIC interrupts
478 * else call platform expert for devices.
479 */
480 if (!lapic_interrupt(interrupt_num, state))
481 PE_incoming_interrupt(interrupt_num);
482
483 if (__improbable(get_preemption_level() != ipl)) {
484 panic("Preemption level altered by interrupt vector 0x%x: initial 0x%x, final: 0x%x\n", interrupt_num, ipl, get_preemption_level());
485 }
486
487 KERNEL_DEBUG_CONSTANT(
488 MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_END,
489 interrupt_num, 0, 0, 0, 0);
490
491 if (cpu_data_ptr[cnum]->cpu_nested_istack) {
492 cpu_data_ptr[cnum]->cpu_nested_istack_events++;
493 }
494 else {
495 uint64_t int_latency = mach_absolute_time() - cpu_data_ptr[cnum]->cpu_int_event_time;
496 if (ilat_assert && (int_latency > interrupt_latency_cap) && !machine_timeout_suspended()) {
497 panic("Interrupt vector 0x%x exceeded interrupt latency threshold, 0x%llx absolute time delta, prior signals: 0x%x, current signals: 0x%x", interrupt_num, int_latency, cpu_data_ptr[cnum]->cpu_prior_signals, cpu_data_ptr[cnum]->cpu_signals);
498 }
499 if (int_latency > cpu_data_ptr[cnum]->cpu_max_observed_int_latency) {
500 cpu_data_ptr[cnum]->cpu_max_observed_int_latency = int_latency;
501 cpu_data_ptr[cnum]->cpu_max_observed_int_latency_vector = interrupt_num;
502 }
503 }
504
505 /*
506 * Having serviced the interrupt first, look at the interrupted stack depth.
507 */
508 if (!user_mode) {
509 uint64_t depth = cpu_data_ptr[cnum]->cpu_kernel_stack
510 + sizeof(struct x86_kernel_state)
511 + sizeof(struct i386_exception_link *)
512 - rsp;
513 if (depth > kernel_stack_depth_max) {
514 kernel_stack_depth_max = (vm_offset_t)depth;
515 KERNEL_DEBUG_CONSTANT(
516 MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_DEPTH),
517 (long) depth, (long) rip, 0, 0, 0);
518 }
519 }
520 }
521
522 static inline void
523 reset_dr7(void)
524 {
525 long dr7 = 0x400; /* magic dr7 reset value; 32 bit on i386, 64 bit on x86_64 */
526 __asm__ volatile("mov %0,%%dr7" : : "r" (dr7));
527 }
528 #if MACH_KDP
529 unsigned kdp_has_active_watchpoints = 0;
530 #define NO_WATCHPOINTS (!kdp_has_active_watchpoints)
531 #else
532 #define NO_WATCHPOINTS 1
533 #endif
534 /*
535 * Trap from kernel mode. Only page-fault errors are recoverable,
536 * and then only in special circumstances. All other errors are
537 * fatal. Return value indicates if trap was handled.
538 */
539
540 void
541 kernel_trap(
542 x86_saved_state_t *state,
543 uintptr_t *lo_spp)
544 {
545 #ifdef __i386__
546 x86_saved_state32_t *saved_state;
547 #else
548 x86_saved_state64_t *saved_state;
549 #endif
550 int code;
551 user_addr_t vaddr;
552 int type;
553 vm_map_t map = 0; /* protected by T_PAGE_FAULT */
554 kern_return_t result = KERN_FAILURE;
555 thread_t thread;
556 ast_t *myast;
557 boolean_t intr;
558 vm_prot_t prot;
559 struct recovery *rp;
560 vm_offset_t kern_ip;
561 #if NCOPY_WINDOWS > 0
562 int fault_in_copy_window = -1;
563 #endif
564 int is_user = 0;
565 #if MACH_KDB
566 pt_entry_t *pte;
567 #endif /* MACH_KDB */
568
569 thread = current_thread();
570
571 #ifdef __i386__
572 if (__improbable(is_saved_state64(state))) {
573 panic_64(state, 0, "Kernel trap with 64-bit state", FALSE);
574 }
575
576 saved_state = saved_state32(state);
577
578 /* Record cpu where state was captured (trampolines don't set this) */
579 saved_state->cpu = cpu_number();
580
581 vaddr = (user_addr_t)saved_state->cr2;
582 type = saved_state->trapno;
583 code = saved_state->err & 0xffff;
584 intr = (saved_state->efl & EFL_IF) != 0; /* state of ints at trap */
585 kern_ip = (vm_offset_t)saved_state->eip;
586 #else
587 if (__improbable(is_saved_state32(state)))
588 panic("kernel_trap(%p) with 32-bit state", state);
589 saved_state = saved_state64(state);
590
591 /* Record cpu where state was captured */
592 saved_state->isf.cpu = cpu_number();
593
594 vaddr = (user_addr_t)saved_state->cr2;
595 type = saved_state->isf.trapno;
596 code = (int)(saved_state->isf.err & 0xffff);
597 intr = (saved_state->isf.rflags & EFL_IF) != 0; /* state of ints at trap */
598 kern_ip = (vm_offset_t)saved_state->isf.rip;
599 #endif
600
601 myast = ast_pending();
602
603 perfASTCallback astfn = perfASTHook;
604 if (__improbable(astfn != NULL)) {
605 if (*myast & AST_CHUD_ALL)
606 astfn(AST_CHUD_ALL, myast);
607 } else
608 *myast &= ~AST_CHUD_ALL;
609
610 /*
611 * Is there a hook?
612 */
613 perfCallback fn = perfTrapHook;
614 if (__improbable(fn != NULL)) {
615 if (fn(type, NULL, 0, 0) == KERN_SUCCESS) {
616 /*
617 * If it succeeds, we are done...
618 */
619 return;
620 }
621 }
622
623 #if CONFIG_DTRACE
624 if (__improbable(tempDTraceTrapHook != NULL)) {
625 if (tempDTraceTrapHook(type, state, lo_spp, 0) == KERN_SUCCESS) {
626 /*
627 * If it succeeds, we are done...
628 */
629 return;
630 }
631 }
632 #endif /* CONFIG_DTRACE */
633
634 /*
635 * we come here with interrupts off as we don't want to recurse
636 * on preemption below. but we do want to re-enable interrupts
637 * as soon we possibly can to hold latency down
638 */
639 if (__improbable(T_PREEMPT == type)) {
640 ast_taken(AST_PREEMPTION, FALSE);
641
642 KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_EXCP_KTRAP_x86, type)) | DBG_FUNC_NONE,
643 0, 0, 0, kern_ip, 0);
644 return;
645 }
646
647 if (T_PAGE_FAULT == type) {
648 /*
649 * assume we're faulting in the kernel map
650 */
651 map = kernel_map;
652
653 if (__probable(thread != THREAD_NULL && thread->map != kernel_map)) {
654 #if NCOPY_WINDOWS > 0
655 vm_offset_t copy_window_base;
656 vm_offset_t kvaddr;
657 int window_index;
658
659 kvaddr = (vm_offset_t)vaddr;
660 /*
661 * must determine if fault occurred in
662 * the copy window while pre-emption is
663 * disabled for this processor so that
664 * we only need to look at the window
665 * associated with this processor
666 */
667 copy_window_base = current_cpu_datap()->cpu_copywindow_base;
668
669 if (kvaddr >= copy_window_base && kvaddr < (copy_window_base + (NBPDE * NCOPY_WINDOWS)) ) {
670
671 window_index = (int)((kvaddr - copy_window_base) / NBPDE);
672
673 if (thread->machine.copy_window[window_index].user_base != (user_addr_t)-1) {
674
675 kvaddr -= (copy_window_base + (NBPDE * window_index));
676 vaddr = thread->machine.copy_window[window_index].user_base + kvaddr;
677
678 map = thread->map;
679 fault_in_copy_window = window_index;
680 }
681 is_user = -1;
682 }
683 #else
684 if (__probable(vaddr < VM_MAX_USER_PAGE_ADDRESS)) {
685 /* fault occurred in userspace */
686 map = thread->map;
687 is_user = -1;
688
689 /* Intercept a potential Supervisor Mode Execute
690 * Protection fault. These criteria identify
691 * both NX faults and SMEP faults, but both
692 * are fatal. We avoid checking PTEs (racy).
693 * (The VM could just redrive a SMEP fault, hence
694 * the intercept).
695 */
696 if (__improbable((code == (T_PF_PROT | T_PF_EXECUTE)) && (pmap_smep_enabled) && (saved_state->isf.rip == vaddr))) {
697 goto debugger_entry;
698 }
699
700 /*
701 * If we're not sharing cr3 with the user
702 * and we faulted in copyio,
703 * then switch cr3 here and dismiss the fault.
704 */
705 if (no_shared_cr3 &&
706 (thread->machine.specFlags&CopyIOActive) &&
707 map->pmap->pm_cr3 != get_cr3_base()) {
708 pmap_assert(current_cpu_datap()->cpu_pmap_pcid_enabled == FALSE);
709 set_cr3_raw(map->pmap->pm_cr3);
710 return;
711 }
712 }
713 #endif
714 }
715 }
716
717 KERNEL_DEBUG_CONSTANT(
718 (MACHDBG_CODE(DBG_MACH_EXCP_KTRAP_x86, type)) | DBG_FUNC_NONE,
719 (unsigned)(vaddr >> 32), (unsigned)vaddr, is_user, kern_ip, 0);
720
721
722 (void) ml_set_interrupts_enabled(intr);
723
724 switch (type) {
725
726 case T_NO_FPU:
727 fpnoextflt();
728 return;
729
730 case T_FPU_FAULT:
731 fpextovrflt();
732 return;
733
734 case T_FLOATING_POINT_ERROR:
735 fpexterrflt();
736 return;
737
738 case T_SSE_FLOAT_ERROR:
739 fpSSEexterrflt();
740 return;
741 case T_DEBUG:
742 #ifdef __i386__
743 if ((saved_state->efl & EFL_TF) == 0 && NO_WATCHPOINTS)
744 #else
745 if ((saved_state->isf.rflags & EFL_TF) == 0 && NO_WATCHPOINTS)
746 #endif
747 {
748 /* We've somehow encountered a debug
749 * register match that does not belong
750 * to the kernel debugger.
751 * This isn't supposed to happen.
752 */
753 reset_dr7();
754 return;
755 }
756 goto debugger_entry;
757 #ifdef __x86_64__
758 case T_INT3:
759 goto debugger_entry;
760 #endif
761 case T_PAGE_FAULT:
762
763 #if MACH_KDB
764 /*
765 * Check for watchpoint on kernel static data.
766 * vm_fault would fail in this case
767 */
768 if (map == kernel_map && db_watchpoint_list && db_watchpoints_inserted &&
769 (code & T_PF_WRITE) && vaddr < vm_map_max(map) &&
770 ((*(pte = pmap_pte(kernel_pmap, (vm_map_offset_t)vaddr))) & INTEL_PTE_WRITE) == 0) {
771 pmap_store_pte(
772 pte,
773 *pte | INTEL_PTE_VALID | INTEL_PTE_WRITE);
774 /* XXX need invltlb here? */
775
776 result = KERN_SUCCESS;
777 goto look_for_watchpoints;
778 }
779 #endif /* MACH_KDB */
780
781 #if CONFIG_DTRACE
782 if (thread != THREAD_NULL && thread->options & TH_OPT_DTRACE) { /* Executing under dtrace_probe? */
783 if (dtrace_tally_fault(vaddr)) { /* Should a fault under dtrace be ignored? */
784 /*
785 * DTrace has "anticipated" the possibility of this fault, and has
786 * established the suitable recovery state. Drop down now into the
787 * recovery handling code in "case T_GENERAL_PROTECTION:".
788 */
789 goto FALL_THROUGH;
790 }
791 }
792 #endif /* CONFIG_DTRACE */
793
794
795 prot = VM_PROT_READ;
796
797 if (code & T_PF_WRITE)
798 prot |= VM_PROT_WRITE;
799 #if PAE
800 if (code & T_PF_EXECUTE)
801 prot |= VM_PROT_EXECUTE;
802 #endif
803
804 result = vm_fault(map,
805 vm_map_trunc_page(vaddr),
806 prot,
807 FALSE,
808 THREAD_UNINT, NULL, 0);
809
810 #if MACH_KDB
811 if (result == KERN_SUCCESS) {
812 /*
813 * Look for watchpoints
814 */
815 look_for_watchpoints:
816 if (map == kernel_map && db_watchpoint_list && db_watchpoints_inserted && (code & T_PF_WRITE) &&
817 db_find_watchpoint(map, vaddr, saved_state))
818 kdb_trap(T_WATCHPOINT, 0, saved_state);
819 }
820 #endif /* MACH_KDB */
821
822 if (result == KERN_SUCCESS) {
823 #if NCOPY_WINDOWS > 0
824 if (fault_in_copy_window != -1) {
825 ml_set_interrupts_enabled(FALSE);
826 copy_window_fault(thread, map,
827 fault_in_copy_window);
828 (void) ml_set_interrupts_enabled(intr);
829 }
830 #endif /* NCOPY_WINDOWS > 0 */
831 return;
832 }
833 /*
834 * fall through
835 */
836 #if CONFIG_DTRACE
837 FALL_THROUGH:
838 #endif /* CONFIG_DTRACE */
839
840 case T_GENERAL_PROTECTION:
841 /*
842 * If there is a failure recovery address
843 * for this fault, go there.
844 */
845 for (rp = recover_table; rp < recover_table_end; rp++) {
846 if (kern_ip == rp->fault_addr) {
847 set_recovery_ip(saved_state, rp->recover_addr);
848 return;
849 }
850 }
851
852 /*
853 * Check thread recovery address also.
854 */
855 if (thread != THREAD_NULL && thread->recover) {
856 set_recovery_ip(saved_state, thread->recover);
857 thread->recover = 0;
858 return;
859 }
860 /*
861 * Unanticipated page-fault errors in kernel
862 * should not happen.
863 *
864 * fall through...
865 */
866 default:
867 /*
868 * Exception 15 is reserved but some chips may generate it
869 * spuriously. Seen at startup on AMD Athlon-64.
870 */
871 if (type == 15) {
872 kprintf("kernel_trap() ignoring spurious trap 15\n");
873 return;
874 }
875 debugger_entry:
876 /* Ensure that the i386_kernel_state at the base of the
877 * current thread's stack (if any) is synchronized with the
878 * context at the moment of the trap, to facilitate
879 * access through the debugger.
880 */
881 sync_iss_to_iks(state);
882 #if MACH_KDB
883 restart_debugger:
884 #endif /* MACH_KDB */
885 #if MACH_KDP
886 if (current_debugger != KDB_CUR_DB) {
887 if (kdp_i386_trap(type, saved_state, result, (vm_offset_t)vaddr))
888 return;
889 } else {
890 #endif /* MACH_KDP */
891 #if MACH_KDB
892 if (kdb_trap(type, code, saved_state)) {
893 if (switch_debugger) {
894 current_debugger = KDP_CUR_DB;
895 switch_debugger = 0;
896 goto restart_debugger;
897 }
898 return;
899 }
900 #endif /* MACH_KDB */
901 #if MACH_KDP
902 }
903 #endif
904 }
905 __asm__ volatile("cli":::"cc");
906 panic_trap(saved_state);
907 /*
908 * NO RETURN
909 */
910 }
911
912
913 #ifdef __i386__
914 static void
915 set_recovery_ip(x86_saved_state32_t *saved_state, vm_offset_t ip)
916 {
917 saved_state->eip = ip;
918 }
919 #else
920 static void
921 set_recovery_ip(x86_saved_state64_t *saved_state, vm_offset_t ip)
922 {
923 saved_state->isf.rip = ip;
924 }
925 #endif
926
927
928 #ifdef __i386__
929 static void
930 panic_trap(x86_saved_state32_t *regs)
931 {
932 const char *trapname = "Unknown";
933 pal_cr_t cr0, cr2, cr3, cr4;
934
935 pal_get_control_registers( &cr0, &cr2, &cr3, &cr4 );
936
937 /*
938 * Issue an I/O port read if one has been requested - this is an
939 * event logic analyzers can use as a trigger point.
940 */
941 panic_io_port_read();
942
943 kprintf("panic trap number 0x%x, eip 0x%x\n", regs->trapno, regs->eip);
944 kprintf("cr0 0x%08x cr2 0x%08x cr3 0x%08x cr4 0x%08x\n",
945 cr0, cr2, cr3, cr4);
946
947 if (regs->trapno < TRAP_TYPES)
948 trapname = trap_type[regs->trapno];
949 #undef panic
950 panic("Kernel trap at 0x%08x, type %d=%s, registers:\n"
951 "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n"
952 "EAX: 0x%08x, EBX: 0x%08x, ECX: 0x%08x, EDX: 0x%08x\n"
953 "CR2: 0x%08x, EBP: 0x%08x, ESI: 0x%08x, EDI: 0x%08x\n"
954 "EFL: 0x%08x, EIP: 0x%08x, CS: 0x%08x, DS: 0x%08x\n"
955 "Error code: 0x%08x\n",
956 regs->eip, regs->trapno, trapname, cr0, cr2, cr3, cr4,
957 regs->eax,regs->ebx,regs->ecx,regs->edx,
958 regs->cr2,regs->ebp,regs->esi,regs->edi,
959 regs->efl,regs->eip,regs->cs & 0xFFFF, regs->ds & 0xFFFF, regs->err);
960 /*
961 * This next statement is not executed,
962 * but it's needed to stop the compiler using tail call optimization
963 * for the panic call - which confuses the subsequent backtrace.
964 */
965 cr0 = 0;
966 }
967 #else
968
969
970 static void
971 panic_trap(x86_saved_state64_t *regs)
972 {
973 const char *trapname = "Unknown";
974 pal_cr_t cr0, cr2, cr3, cr4;
975 boolean_t potential_smep_fault = FALSE;
976
977 pal_get_control_registers( &cr0, &cr2, &cr3, &cr4 );
978 assert(ml_get_interrupts_enabled() == FALSE);
979 current_cpu_datap()->cpu_fatal_trap_state = regs;
980 /*
981 * Issue an I/O port read if one has been requested - this is an
982 * event logic analyzers can use as a trigger point.
983 */
984 panic_io_port_read();
985
986 kprintf("panic trap number 0x%x, rip 0x%016llx\n",
987 regs->isf.trapno, regs->isf.rip);
988 kprintf("cr0 0x%016llx cr2 0x%016llx cr3 0x%016llx cr4 0x%016llx\n",
989 cr0, cr2, cr3, cr4);
990
991 if (regs->isf.trapno < TRAP_TYPES)
992 trapname = trap_type[regs->isf.trapno];
993
994 if ((regs->isf.trapno == T_PAGE_FAULT) && (regs->isf.err == (T_PF_PROT | T_PF_EXECUTE)) && (pmap_smep_enabled) && (regs->isf.rip == regs->cr2) && (regs->isf.rip < VM_MAX_USER_PAGE_ADDRESS)) {
995 potential_smep_fault = TRUE;
996 }
997
998 #undef panic
999 panic("Kernel trap at 0x%016llx, type %d=%s, registers:\n"
1000 "CR0: 0x%016llx, CR2: 0x%016llx, CR3: 0x%016llx, CR4: 0x%016llx\n"
1001 "RAX: 0x%016llx, RBX: 0x%016llx, RCX: 0x%016llx, RDX: 0x%016llx\n"
1002 "RSP: 0x%016llx, RBP: 0x%016llx, RSI: 0x%016llx, RDI: 0x%016llx\n"
1003 "R8: 0x%016llx, R9: 0x%016llx, R10: 0x%016llx, R11: 0x%016llx\n"
1004 "R12: 0x%016llx, R13: 0x%016llx, R14: 0x%016llx, R15: 0x%016llx\n"
1005 "RFL: 0x%016llx, RIP: 0x%016llx, CS: 0x%016llx, SS: 0x%016llx\n"
1006 "CR2: 0x%016llx, Error code: 0x%016llx, Faulting CPU: 0x%x%s\n",
1007 regs->isf.rip, regs->isf.trapno, trapname,
1008 cr0, cr2, cr3, cr4,
1009 regs->rax, regs->rbx, regs->rcx, regs->rdx,
1010 regs->isf.rsp, regs->rbp, regs->rsi, regs->rdi,
1011 regs->r8, regs->r9, regs->r10, regs->r11,
1012 regs->r12, regs->r13, regs->r14, regs->r15,
1013 regs->isf.rflags, regs->isf.rip, regs->isf.cs & 0xFFFF,
1014 regs->isf.ss & 0xFFFF,regs->cr2, regs->isf.err, regs->isf.cpu,
1015 potential_smep_fault ? " SMEP/NX fault" : "");
1016 /*
1017 * This next statement is not executed,
1018 * but it's needed to stop the compiler using tail call optimization
1019 * for the panic call - which confuses the subsequent backtrace.
1020 */
1021 cr0 = 0;
1022 }
1023 #endif
1024
1025 #if CONFIG_DTRACE
1026 extern kern_return_t dtrace_user_probe(x86_saved_state_t *);
1027 #endif
1028
1029 /*
1030 * Trap from user mode.
1031 */
1032 void
1033 user_trap(
1034 x86_saved_state_t *saved_state)
1035 {
1036 int exc;
1037 int err;
1038 mach_exception_code_t code;
1039 mach_exception_subcode_t subcode;
1040 int type;
1041 user_addr_t vaddr;
1042 vm_prot_t prot;
1043 thread_t thread = current_thread();
1044 ast_t *myast;
1045 kern_return_t kret;
1046 user_addr_t rip;
1047 unsigned long dr6 = 0; /* 32 bit for i386, 64 bit for x86_64 */
1048
1049 assert((is_saved_state32(saved_state) && !thread_is_64bit(thread)) ||
1050 (is_saved_state64(saved_state) && thread_is_64bit(thread)));
1051
1052 if (is_saved_state64(saved_state)) {
1053 x86_saved_state64_t *regs;
1054
1055 regs = saved_state64(saved_state);
1056
1057 /* Record cpu where state was captured */
1058 regs->isf.cpu = cpu_number();
1059
1060 type = regs->isf.trapno;
1061 err = (int)regs->isf.err & 0xffff;
1062 vaddr = (user_addr_t)regs->cr2;
1063 rip = (user_addr_t)regs->isf.rip;
1064 } else {
1065 x86_saved_state32_t *regs;
1066
1067 regs = saved_state32(saved_state);
1068
1069 /* Record cpu where state was captured */
1070 regs->cpu = cpu_number();
1071
1072 type = regs->trapno;
1073 err = regs->err & 0xffff;
1074 vaddr = (user_addr_t)regs->cr2;
1075 rip = (user_addr_t)regs->eip;
1076 }
1077
1078 if ((type == T_DEBUG) && thread->machine.ids) {
1079 unsigned long clear = 0;
1080 /* Stash and clear this processor's DR6 value, in the event
1081 * this was a debug register match
1082 */
1083 __asm__ volatile ("mov %%db6, %0" : "=r" (dr6));
1084 __asm__ volatile ("mov %0, %%db6" : : "r" (clear));
1085 }
1086
1087 pal_sti();
1088
1089 KERNEL_DEBUG_CONSTANT(
1090 (MACHDBG_CODE(DBG_MACH_EXCP_UTRAP_x86, type)) | DBG_FUNC_NONE,
1091 (unsigned)(vaddr>>32), (unsigned)vaddr,
1092 (unsigned)(rip>>32), (unsigned)rip, 0);
1093
1094 code = 0;
1095 subcode = 0;
1096 exc = 0;
1097
1098 #if DEBUG_TRACE
1099 kprintf("user_trap(0x%08x) type=%d vaddr=0x%016llx\n",
1100 saved_state, type, vaddr);
1101 #endif
1102
1103 perfASTCallback astfn = perfASTHook;
1104 if (__improbable(astfn != NULL)) {
1105 myast = ast_pending();
1106 if (*myast & AST_CHUD_ALL) {
1107 astfn(AST_CHUD_ALL, myast);
1108 }
1109 }
1110
1111 /* Is there a hook? */
1112 perfCallback fn = perfTrapHook;
1113 if (__improbable(fn != NULL)) {
1114 if (fn(type, saved_state, 0, 0) == KERN_SUCCESS)
1115 return; /* If it succeeds, we are done... */
1116 }
1117
1118 /*
1119 * DTrace does not consume all user traps, only INT_3's for now.
1120 * Avoid needlessly calling tempDTraceTrapHook here, and let the
1121 * INT_3 case handle them.
1122 */
1123 DEBUG_KPRINT_SYSCALL_MASK(1,
1124 "user_trap: type=0x%x(%s) err=0x%x cr2=%p rip=%p\n",
1125 type, trap_type[type], err, (void *)(long) vaddr, (void *)(long) rip);
1126
1127 switch (type) {
1128
1129 case T_DIVIDE_ERROR:
1130 exc = EXC_ARITHMETIC;
1131 code = EXC_I386_DIV;
1132 break;
1133
1134 case T_DEBUG:
1135 {
1136 pcb_t pcb;
1137 /*
1138 * Update the PCB with this processor's DR6 value
1139 * in the event this was a debug register match.
1140 */
1141 pcb = THREAD_TO_PCB(thread);
1142 if (pcb->ids) {
1143 /*
1144 * We can get and set the status register
1145 * in 32-bit mode even on a 64-bit thread
1146 * because the high order bits are not
1147 * used on x86_64
1148 */
1149 if (thread_is_64bit(thread)) {
1150 x86_debug_state64_t *ids = pcb->ids;
1151 ids->dr6 = dr6;
1152 } else { /* 32 bit thread */
1153 x86_debug_state32_t *ids = pcb->ids;
1154 ids->dr6 = (uint32_t) dr6;
1155 }
1156 }
1157 exc = EXC_BREAKPOINT;
1158 code = EXC_I386_SGL;
1159 break;
1160 }
1161 case T_INT3:
1162 #if CONFIG_DTRACE
1163 if (dtrace_user_probe(saved_state) == KERN_SUCCESS)
1164 return; /* If it succeeds, we are done... */
1165 #endif
1166 exc = EXC_BREAKPOINT;
1167 code = EXC_I386_BPT;
1168 break;
1169
1170 case T_OVERFLOW:
1171 exc = EXC_ARITHMETIC;
1172 code = EXC_I386_INTO;
1173 break;
1174
1175 case T_OUT_OF_BOUNDS:
1176 exc = EXC_SOFTWARE;
1177 code = EXC_I386_BOUND;
1178 break;
1179
1180 case T_INVALID_OPCODE:
1181 exc = EXC_BAD_INSTRUCTION;
1182 code = EXC_I386_INVOP;
1183 break;
1184
1185 case T_NO_FPU:
1186 fpnoextflt();
1187 return;
1188
1189 case T_FPU_FAULT:
1190 fpextovrflt(); /* Propagates exception directly, doesn't return */
1191 return;
1192
1193 case T_INVALID_TSS: /* invalid TSS == iret with NT flag set */
1194 exc = EXC_BAD_INSTRUCTION;
1195 code = EXC_I386_INVTSSFLT;
1196 subcode = err;
1197 break;
1198
1199 case T_SEGMENT_NOT_PRESENT:
1200 exc = EXC_BAD_INSTRUCTION;
1201 code = EXC_I386_SEGNPFLT;
1202 subcode = err;
1203 break;
1204
1205 case T_STACK_FAULT:
1206 exc = EXC_BAD_INSTRUCTION;
1207 code = EXC_I386_STKFLT;
1208 subcode = err;
1209 break;
1210
1211 case T_GENERAL_PROTECTION:
1212 /*
1213 * There's a wide range of circumstances which generate this
1214 * class of exception. From user-space, many involve bad
1215 * addresses (such as a non-canonical 64-bit address).
1216 * So we map this to EXC_BAD_ACCESS (and thereby SIGSEGV).
1217 * The trouble is cr2 doesn't contain the faulting address;
1218 * we'd need to decode the faulting instruction to really
1219 * determine this. We'll leave that to debuggers.
1220 * However, attempted execution of privileged instructions
1221 * (e.g. cli) also generate GP faults and so we map these to
1222 * to EXC_BAD_ACCESS (and thence SIGSEGV) also - rather than
1223 * EXC_BAD_INSTRUCTION which is more accurate. We just can't
1224 * win!
1225 */
1226 exc = EXC_BAD_ACCESS;
1227 code = EXC_I386_GPFLT;
1228 subcode = err;
1229 break;
1230
1231 case T_PAGE_FAULT:
1232 prot = VM_PROT_READ;
1233
1234 if (err & T_PF_WRITE)
1235 prot |= VM_PROT_WRITE;
1236 #if PAE
1237 if (__improbable(err & T_PF_EXECUTE))
1238 prot |= VM_PROT_EXECUTE;
1239 #endif
1240 kret = vm_fault(thread->map, vm_map_trunc_page(vaddr),
1241 prot, FALSE,
1242 THREAD_ABORTSAFE, NULL, 0);
1243
1244 user_page_fault_continue(kret);
1245
1246 /* NOTREACHED */
1247 break;
1248
1249 case T_SSE_FLOAT_ERROR:
1250 fpSSEexterrflt(); /* Propagates exception directly, doesn't return */
1251 return;
1252
1253
1254 case T_FLOATING_POINT_ERROR:
1255 fpexterrflt(); /* Propagates exception directly, doesn't return */
1256 return;
1257
1258 case T_DTRACE_RET:
1259 #if CONFIG_DTRACE
1260 if (dtrace_user_probe(saved_state) == KERN_SUCCESS)
1261 return; /* If it succeeds, we are done... */
1262 #endif
1263 /*
1264 * If we get an INT 0x7f when we do not expect to,
1265 * treat it as an illegal instruction
1266 */
1267 exc = EXC_BAD_INSTRUCTION;
1268 code = EXC_I386_INVOP;
1269 break;
1270
1271 default:
1272 #if MACH_KGDB
1273 Debugger("Unanticipated user trap");
1274 return;
1275 #endif /* MACH_KGDB */
1276 #if MACH_KDB
1277 if (kdb_trap(type, err, saved_state32(saved_state)))
1278 return;
1279 #endif /* MACH_KDB */
1280 panic("Unexpected user trap, type %d", type);
1281 return;
1282 }
1283 /* Note: Codepaths that directly return from user_trap() have pending
1284 * ASTs processed in locore
1285 */
1286 i386_exception(exc, code, subcode);
1287 /* NOTREACHED */
1288 }
1289
1290
1291 /*
1292 * Handle AST traps for i386.
1293 */
1294
1295 extern void log_thread_action (thread_t, char *);
1296
1297 void
1298 i386_astintr(int preemption)
1299 {
1300 ast_t mask = AST_ALL;
1301 spl_t s;
1302
1303 if (preemption)
1304 mask = AST_PREEMPTION;
1305
1306 s = splsched();
1307
1308 ast_taken(mask, s);
1309
1310 splx(s);
1311 }
1312
1313 /*
1314 * Handle exceptions for i386.
1315 *
1316 * If we are an AT bus machine, we must turn off the AST for a
1317 * delayed floating-point exception.
1318 *
1319 * If we are providing floating-point emulation, we may have
1320 * to retrieve the real register values from the floating point
1321 * emulator.
1322 */
1323 void
1324 i386_exception(
1325 int exc,
1326 mach_exception_code_t code,
1327 mach_exception_subcode_t subcode)
1328 {
1329 mach_exception_data_type_t codes[EXCEPTION_CODE_MAX];
1330
1331 DEBUG_KPRINT_SYSCALL_MACH("i386_exception: exc=%d code=0x%llx subcode=0x%llx\n",
1332 exc, code, subcode);
1333 codes[0] = code; /* new exception interface */
1334 codes[1] = subcode;
1335 exception_triage(exc, codes, 2);
1336 /*NOTREACHED*/
1337 }
1338
1339
1340 #if MACH_KDB
1341
1342 extern void db_i386_state(x86_saved_state32_t *regs);
1343
1344 #include <ddb/db_output.h>
1345
1346 void
1347 db_i386_state(
1348 x86_saved_state32_t *regs)
1349 {
1350 db_printf("eip %8x\n", regs->eip);
1351 db_printf("trap %8x\n", regs->trapno);
1352 db_printf("err %8x\n", regs->err);
1353 db_printf("efl %8x\n", regs->efl);
1354 db_printf("ebp %8x\n", regs->ebp);
1355 db_printf("esp %8x\n", regs->cr2);
1356 db_printf("uesp %8x\n", regs->uesp);
1357 db_printf("cs %8x\n", regs->cs & 0xff);
1358 db_printf("ds %8x\n", regs->ds & 0xff);
1359 db_printf("es %8x\n", regs->es & 0xff);
1360 db_printf("fs %8x\n", regs->fs & 0xff);
1361 db_printf("gs %8x\n", regs->gs & 0xff);
1362 db_printf("ss %8x\n", regs->ss & 0xff);
1363 db_printf("eax %8x\n", regs->eax);
1364 db_printf("ebx %8x\n", regs->ebx);
1365 db_printf("ecx %8x\n", regs->ecx);
1366 db_printf("edx %8x\n", regs->edx);
1367 db_printf("esi %8x\n", regs->esi);
1368 db_printf("edi %8x\n", regs->edi);
1369 }
1370
1371 #endif /* MACH_KDB */
1372
1373 /* Synchronize a thread's i386_kernel_state (if any) with the given
1374 * i386_saved_state_t obtained from the trap/IPI handler; called in
1375 * kernel_trap() prior to entering the debugger, and when receiving
1376 * an "MP_KDP" IPI.
1377 */
1378
1379 void
1380 sync_iss_to_iks(x86_saved_state_t *saved_state)
1381 {
1382 struct x86_kernel_state *iks;
1383 vm_offset_t kstack;
1384 boolean_t record_active_regs = FALSE;
1385
1386 /* The PAL may have a special way to sync registers */
1387 if( saved_state->flavor == THREAD_STATE_NONE )
1388 pal_get_kern_regs( saved_state );
1389
1390 if ((kstack = current_thread()->kernel_stack) != 0) {
1391 #ifdef __i386__
1392 x86_saved_state32_t *regs = saved_state32(saved_state);
1393 #else
1394 x86_saved_state64_t *regs = saved_state64(saved_state);
1395 #endif
1396
1397 iks = STACK_IKS(kstack);
1398
1399 /* Did we take the trap/interrupt in kernel mode? */
1400 #ifdef __i386__
1401 if (regs == USER_REGS32(current_thread()))
1402 record_active_regs = TRUE;
1403 else {
1404 iks->k_ebx = regs->ebx;
1405 iks->k_esp = (int)regs;
1406 iks->k_ebp = regs->ebp;
1407 iks->k_edi = regs->edi;
1408 iks->k_esi = regs->esi;
1409 iks->k_eip = regs->eip;
1410 }
1411 #else
1412 if (regs == USER_REGS64(current_thread()))
1413 record_active_regs = TRUE;
1414 else {
1415 iks->k_rbx = regs->rbx;
1416 iks->k_rsp = regs->isf.rsp;
1417 iks->k_rbp = regs->rbp;
1418 iks->k_r12 = regs->r12;
1419 iks->k_r13 = regs->r13;
1420 iks->k_r14 = regs->r14;
1421 iks->k_r15 = regs->r15;
1422 iks->k_rip = regs->isf.rip;
1423 }
1424 #endif
1425 }
1426
1427 if (record_active_regs == TRUE) {
1428 #ifdef __i386__
1429 /* Show the trap handler path */
1430 __asm__ volatile("movl %%ebx, %0" : "=m" (iks->k_ebx));
1431 __asm__ volatile("movl %%esp, %0" : "=m" (iks->k_esp));
1432 __asm__ volatile("movl %%ebp, %0" : "=m" (iks->k_ebp));
1433 __asm__ volatile("movl %%edi, %0" : "=m" (iks->k_edi));
1434 __asm__ volatile("movl %%esi, %0" : "=m" (iks->k_esi));
1435 /* "Current" instruction pointer */
1436 __asm__ volatile("movl $1f, %0\n1:" : "=m" (iks->k_eip));
1437 #else
1438 /* Show the trap handler path */
1439 __asm__ volatile("movq %%rbx, %0" : "=m" (iks->k_rbx));
1440 __asm__ volatile("movq %%rsp, %0" : "=m" (iks->k_rsp));
1441 __asm__ volatile("movq %%rbp, %0" : "=m" (iks->k_rbp));
1442 __asm__ volatile("movq %%r12, %0" : "=m" (iks->k_r12));
1443 __asm__ volatile("movq %%r13, %0" : "=m" (iks->k_r13));
1444 __asm__ volatile("movq %%r14, %0" : "=m" (iks->k_r14));
1445 __asm__ volatile("movq %%r15, %0" : "=m" (iks->k_r15));
1446 /* "Current" instruction pointer */
1447 __asm__ volatile("leaq 1f(%%rip), %%rax; mov %%rax, %0\n1:"
1448 : "=m" (iks->k_rip)
1449 :
1450 : "rax");
1451 #endif
1452 }
1453 }
1454
1455 /*
1456 * This is used by the NMI interrupt handler (from mp.c) to
1457 * uncondtionally sync the trap handler context to the IKS
1458 * irrespective of whether the NMI was fielded in kernel
1459 * or user space.
1460 */
1461 void
1462 sync_iss_to_iks_unconditionally(__unused x86_saved_state_t *saved_state) {
1463 struct x86_kernel_state *iks;
1464 vm_offset_t kstack;
1465
1466 if ((kstack = current_thread()->kernel_stack) != 0) {
1467 iks = STACK_IKS(kstack);
1468 #ifdef __i386__
1469 /* Display the trap handler path */
1470 __asm__ volatile("movl %%ebx, %0" : "=m" (iks->k_ebx));
1471 __asm__ volatile("movl %%esp, %0" : "=m" (iks->k_esp));
1472 __asm__ volatile("movl %%ebp, %0" : "=m" (iks->k_ebp));
1473 __asm__ volatile("movl %%edi, %0" : "=m" (iks->k_edi));
1474 __asm__ volatile("movl %%esi, %0" : "=m" (iks->k_esi));
1475 /* "Current" instruction pointer */
1476 __asm__ volatile("movl $1f, %0\n1:" : "=m" (iks->k_eip));
1477 #else
1478 /* Display the trap handler path */
1479 __asm__ volatile("movq %%rbx, %0" : "=m" (iks->k_rbx));
1480 __asm__ volatile("movq %%rsp, %0" : "=m" (iks->k_rsp));
1481 __asm__ volatile("movq %%rbp, %0" : "=m" (iks->k_rbp));
1482 __asm__ volatile("movq %%r12, %0" : "=m" (iks->k_r12));
1483 __asm__ volatile("movq %%r13, %0" : "=m" (iks->k_r13));
1484 __asm__ volatile("movq %%r14, %0" : "=m" (iks->k_r14));
1485 __asm__ volatile("movq %%r15, %0" : "=m" (iks->k_r15));
1486 /* "Current" instruction pointer */
1487 __asm__ volatile("leaq 1f(%%rip), %%rax; mov %%rax, %0\n1:" : "=m" (iks->k_rip)::"rax");
1488 #endif
1489 }
1490 }
mirror of XNU