X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/593a1d5fd87cdf5b46dd5fcb84467b432cea0f91..6d2010ae8f7a6078e10b361c6962983bab233e0f:/osfmk/i386/trap.c diff --git a/osfmk/i386/trap.c b/osfmk/i386/trap.c index b263be9ff..55be4fc75 100644 --- a/osfmk/i386/trap.c +++ b/osfmk/i386/trap.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2010 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -71,6 +71,7 @@ #include #include #include /* panic_io_port_read() */ +#include #include #include @@ -89,6 +90,7 @@ #include #include #include +#include #include @@ -109,21 +111,32 @@ #include #include #include +#if CONFIG_MCA #include +#endif #include +#include + +#include -extern void throttle_lowpri_io(boolean_t); +extern void throttle_lowpri_io(int); +extern void kprint_state(x86_saved_state64_t *saved_state); /* * Forward declarations */ static void user_page_fault_continue(kern_return_t kret); +#ifdef __i386__ static void panic_trap(x86_saved_state32_t *saved_state); static void set_recovery_ip(x86_saved_state32_t *saved_state, vm_offset_t ip); +extern void panic_64(x86_saved_state_t *, int, const char *, boolean_t); +#else +static void panic_trap(x86_saved_state64_t *saved_state); +static void set_recovery_ip(x86_saved_state64_t *saved_state, vm_offset_t ip); +#endif -perfCallback perfTrapHook = NULL; /* Pointer to CHUD trap hook routine */ -perfCallback perfASTHook = NULL; /* Pointer to CHUD AST hook routine */ +volatile perfCallback perfTrapHook = NULL; /* Pointer to CHUD trap hook routine */ #if CONFIG_DTRACE /* See */ @@ -137,34 +150,60 @@ thread_syscall_return( kern_return_t ret) { thread_t thr_act = current_thread(); + boolean_t is_mach; + int code; + + pal_register_cache_state(thr_act, DIRTY); if (thread_is_64bit(thr_act)) { x86_saved_state64_t *regs; regs = USER_REGS64(thr_act); - if (kdebug_enable && ((regs->rax & SYSCALL_CLASS_MASK) == (SYSCALL_CLASS_MACH << SYSCALL_CLASS_SHIFT))) { + code = (int) (regs->rax & SYSCALL_NUMBER_MASK); + is_mach = (regs->rax & SYSCALL_CLASS_MASK) + == (SYSCALL_CLASS_MACH << SYSCALL_CLASS_SHIFT); + if (kdebug_enable && is_mach) { /* Mach trap */ KERNEL_DEBUG_CONSTANT( - MACHDBG_CODE(DBG_MACH_EXCP_SC, ((int) (regs->rax & SYSCALL_NUMBER_MASK))) - | DBG_FUNC_END, - ret, 0, 0, 0, 0); + MACHDBG_CODE(DBG_MACH_EXCP_SC,code)|DBG_FUNC_END, + ret, 0, 0, 0, 0); } regs->rax = ret; - +#if DEBUG + if (is_mach) + DEBUG_KPRINT_SYSCALL_MACH( + "thread_syscall_return: 64-bit mach ret=%u\n", + ret); + else + DEBUG_KPRINT_SYSCALL_UNIX( + "thread_syscall_return: 64-bit unix ret=%u\n", + ret); +#endif } else { x86_saved_state32_t *regs; regs = USER_REGS32(thr_act); - if (kdebug_enable && ((int) regs->eax < 0)) { + code = ((int) regs->eax); + is_mach = (code < 0); + if (kdebug_enable && is_mach) { /* Mach trap */ KERNEL_DEBUG_CONSTANT( - MACHDBG_CODE(DBG_MACH_EXCP_SC, -((int) regs->eax)) - | DBG_FUNC_END, - ret, 0, 0, 0, 0); + MACHDBG_CODE(DBG_MACH_EXCP_SC,-code)|DBG_FUNC_END, + ret, 0, 0, 0, 0); } regs->eax = ret; +#if DEBUG + if (is_mach) + DEBUG_KPRINT_SYSCALL_MACH( + "thread_syscall_return: 32-bit mach ret=%u\n", + ret); + else + DEBUG_KPRINT_SYSCALL_UNIX( + "thread_syscall_return: 32-bit unix ret=%u\n", + ret); +#endif } throttle_lowpri_io(TRUE); @@ -185,6 +224,8 @@ thread_kdb_return(void) thread_t thr_act = current_thread(); x86_saved_state_t *iss = USER_STATE(thr_act); + pal_register_cache_state(thr_act, DIRTY); + if (is_saved_state64(iss)) { x86_saved_state64_t *regs; @@ -209,14 +250,13 @@ thread_kdb_return(void) #endif /* MACH_KDB */ -void +static inline void user_page_fault_continue( kern_return_t kr) { thread_t thread = current_thread(); - ast_t *myast; - boolean_t intr; user_addr_t vaddr; + #if MACH_KDB x86_saved_state_t *regs = USER_STATE(thread); int err; @@ -233,7 +273,7 @@ user_page_fault_continue( #if MACH_KDB trapno = uregs->isf.trapno; - err = uregs->isf.err; + err = (int)uregs->isf.err; #endif vaddr = (user_addr_t)uregs->cr2; } else { @@ -248,7 +288,7 @@ user_page_fault_continue( vaddr = uregs->cr2; } - if ((kr == KERN_SUCCESS) || (kr == KERN_ABORTED)) { + if (__probable((kr == KERN_SUCCESS) || (kr == KERN_ABORTED))) { #if MACH_KDB if (!db_breakpoints_inserted) { db_set_breakpoints(); @@ -261,15 +301,6 @@ user_page_fault_continue( saved_state32(regs))) kdb_trap(T_WATCHPOINT, 0, saved_state32(regs)); #endif /* MACH_KDB */ - intr = ml_set_interrupts_enabled(FALSE); - myast = ast_pending(); - while (*myast & AST_ALL) { - ast_taken(AST_ALL, intr); - ml_set_interrupts_enabled(FALSE); - myast = ast_pending(); - } - ml_set_interrupts_enabled(intr); - thread_exception_return(); /*NOTREACHED*/ } @@ -282,6 +313,9 @@ user_page_fault_continue( } #endif /* MACH_KDB */ + /* PAL debug hook */ + pal_dbg_page_fault( thread, vaddr, kr ); + i386_exception(EXC_BAD_ACCESS, kr, vaddr); /*NOTREACHED*/ } @@ -290,8 +324,8 @@ user_page_fault_continue( * Fault recovery in copyin/copyout routines. */ struct recovery { - uint32_t fault_addr; - uint32_t recover_addr; + uintptr_t fault_addr; + uintptr_t recover_addr; }; extern struct recovery recover_table[]; @@ -300,26 +334,217 @@ extern struct recovery recover_table_end[]; const char * trap_type[] = {TRAP_NAMES}; unsigned TRAP_TYPES = sizeof(trap_type)/sizeof(trap_type[0]); +extern void PE_incoming_interrupt(int interrupt); + +#if defined(__x86_64__) && DEBUG +void +kprint_state(x86_saved_state64_t *saved_state) +{ + kprintf("current_cpu_datap() 0x%lx\n", (uintptr_t)current_cpu_datap()); + kprintf("Current GS base MSR 0x%llx\n", rdmsr64(MSR_IA32_GS_BASE)); + kprintf("Kernel GS base MSR 0x%llx\n", rdmsr64(MSR_IA32_KERNEL_GS_BASE)); + kprintf("state at 0x%lx:\n", (uintptr_t) saved_state); + + kprintf(" rdi 0x%llx\n", saved_state->rdi); + kprintf(" rsi 0x%llx\n", saved_state->rsi); + kprintf(" rdx 0x%llx\n", saved_state->rdx); + kprintf(" r10 0x%llx\n", saved_state->r10); + kprintf(" r8 0x%llx\n", saved_state->r8); + kprintf(" r9 0x%llx\n", saved_state->r9); + kprintf(" v_arg6 0x%llx\n", saved_state->v_arg6); + kprintf(" v_arg7 0x%llx\n", saved_state->v_arg7); + kprintf(" v_arg8 0x%llx\n", saved_state->v_arg8); + + kprintf(" cr2 0x%llx\n", saved_state->cr2); + kprintf("real cr2 0x%lx\n", get_cr2()); + kprintf(" r15 0x%llx\n", saved_state->r15); + kprintf(" r14 0x%llx\n", saved_state->r14); + kprintf(" r13 0x%llx\n", saved_state->r13); + kprintf(" r12 0x%llx\n", saved_state->r12); + kprintf(" r11 0x%llx\n", saved_state->r11); + kprintf(" rbp 0x%llx\n", saved_state->rbp); + kprintf(" rbx 0x%llx\n", saved_state->rbx); + kprintf(" rcx 0x%llx\n", saved_state->rcx); + kprintf(" rax 0x%llx\n", saved_state->rax); + + kprintf(" gs 0x%x\n", saved_state->gs); + kprintf(" fs 0x%x\n", saved_state->fs); + + kprintf(" isf.trapno 0x%x\n", saved_state->isf.trapno); + kprintf(" isf._pad 0x%x\n", saved_state->isf._pad); + kprintf(" isf.trapfn 0x%llx\n", saved_state->isf.trapfn); + kprintf(" isf.err 0x%llx\n", saved_state->isf.err); + kprintf(" isf.rip 0x%llx\n", saved_state->isf.rip); + kprintf(" isf.cs 0x%llx\n", saved_state->isf.cs); + kprintf(" isf.rflags 0x%llx\n", saved_state->isf.rflags); + kprintf(" isf.rsp 0x%llx\n", saved_state->isf.rsp); + kprintf(" isf.ss 0x%llx\n", saved_state->isf.ss); +} +#endif + + +/* + * Non-zero indicates latency assert is enabled and capped at valued + * absolute time units. + */ + +uint64_t interrupt_latency_cap = 0; +boolean_t ilat_assert = FALSE; + +void +interrupt_latency_tracker_setup(void) { + uint32_t ilat_cap_us; + if (PE_parse_boot_argn("interrupt_latency_cap_us", &ilat_cap_us, sizeof(ilat_cap_us))) { + interrupt_latency_cap = ilat_cap_us * NSEC_PER_USEC; + nanoseconds_to_absolutetime(interrupt_latency_cap, &interrupt_latency_cap); + } else { + interrupt_latency_cap = LockTimeOut; + } + PE_parse_boot_argn("-interrupt_latency_assert_enable", &ilat_assert, sizeof(ilat_assert)); +} + +void interrupt_reset_latency_stats(void) { + uint32_t i; + for (i = 0; i < real_ncpus; i++) { + cpu_data_ptr[i]->cpu_max_observed_int_latency = + cpu_data_ptr[i]->cpu_max_observed_int_latency_vector = 0; + } +} + +void interrupt_populate_latency_stats(char *buf, unsigned bufsize) { + uint32_t i, tcpu = ~0; + uint64_t cur_max = 0; + + for (i = 0; i < real_ncpus; i++) { + if (cur_max < cpu_data_ptr[i]->cpu_max_observed_int_latency) { + cur_max = cpu_data_ptr[i]->cpu_max_observed_int_latency; + tcpu = i; + } + } + + if (tcpu < real_ncpus) + snprintf(buf, bufsize, "0x%x 0x%x 0x%llx", tcpu, cpu_data_ptr[tcpu]->cpu_max_observed_int_latency_vector, cpu_data_ptr[tcpu]->cpu_max_observed_int_latency); +} + +/* + * Handle interrupts: + * - local APIC interrupts (IPIs, timers, etc) are handled by the kernel, + * - device interrupts go to the platform expert. + */ +void +interrupt(x86_saved_state_t *state) +{ + uint64_t rip; + uint64_t rsp; + int interrupt_num; + boolean_t user_mode = FALSE; + int ipl; + int cnum = cpu_number(); + + if (is_saved_state64(state) == TRUE) { + x86_saved_state64_t *state64; + + state64 = saved_state64(state); + rip = state64->isf.rip; + rsp = state64->isf.rsp; + interrupt_num = state64->isf.trapno; +#ifdef __x86_64__ + if(state64->isf.cs & 0x03) +#endif + user_mode = TRUE; + } else { + x86_saved_state32_t *state32; + + state32 = saved_state32(state); + if (state32->cs & 0x03) + user_mode = TRUE; + rip = state32->eip; + rsp = state32->uesp; + interrupt_num = state32->trapno; + } + + KERNEL_DEBUG_CONSTANT( + MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_START, + interrupt_num, rip, user_mode, 0, 0); + + SCHED_STATS_INTERRUPT(current_processor()); + + ipl = get_preemption_level(); + + /* + * Handle local APIC interrupts + * else call platform expert for devices. + */ + if (!lapic_interrupt(interrupt_num, state)) + PE_incoming_interrupt(interrupt_num); + + if (__improbable(get_preemption_level() != ipl)) { + panic("Preemption level altered by interrupt vector 0x%x: initial 0x%x, final: 0x%x\n", interrupt_num, ipl, get_preemption_level()); + } + + KERNEL_DEBUG_CONSTANT( + MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_END, + interrupt_num, 0, 0, 0, 0); + + if (cpu_data_ptr[cnum]->cpu_nested_istack) { + cpu_data_ptr[cnum]->cpu_nested_istack_events++; + } + else { + uint64_t int_latency = mach_absolute_time() - cpu_data_ptr[cnum]->cpu_int_event_time; + if (ilat_assert && (int_latency > interrupt_latency_cap) && !machine_timeout_suspended()) { + panic("Interrupt vector 0x%x exceeded interrupt latency threshold, 0x%llx absolute time delta, prior signals: 0x%x, current signals: 0x%x", interrupt_num, int_latency, cpu_data_ptr[cnum]->cpu_prior_signals, cpu_data_ptr[cnum]->cpu_signals); + } + if (int_latency > cpu_data_ptr[cnum]->cpu_max_observed_int_latency) { + cpu_data_ptr[cnum]->cpu_max_observed_int_latency = int_latency; + cpu_data_ptr[cnum]->cpu_max_observed_int_latency_vector = interrupt_num; + } + } + + /* + * Having serviced the interrupt first, look at the interrupted stack depth. + */ + if (!user_mode) { + uint64_t depth = cpu_data_ptr[cnum]->cpu_kernel_stack + + sizeof(struct x86_kernel_state) + + sizeof(struct i386_exception_link *) + - rsp; + if (depth > kernel_stack_depth_max) { + kernel_stack_depth_max = (vm_offset_t)depth; + KERNEL_DEBUG_CONSTANT( + MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_DEPTH), + (long) depth, (long) rip, 0, 0, 0); + } + } +} static inline void reset_dr7(void) { - uint32_t dr7 = 0x400; /* magic dr7 reset value */ - __asm__ volatile("movl %0,%%dr7" : : "r" (dr7)); + long dr7 = 0x400; /* magic dr7 reset value; 32 bit on i386, 64 bit on x86_64 */ + __asm__ volatile("mov %0,%%dr7" : : "r" (dr7)); } #if MACH_KDP unsigned kdp_has_active_watchpoints = 0; +#define NO_WATCHPOINTS (!kdp_has_active_watchpoints) +#else +#define NO_WATCHPOINTS 1 #endif /* * Trap from kernel mode. Only page-fault errors are recoverable, * and then only in special circumstances. All other errors are * fatal. Return value indicates if trap was handled. */ + void kernel_trap( - x86_saved_state_t *state) + x86_saved_state_t *state, + uintptr_t *lo_spp) { +#ifdef __i386__ x86_saved_state32_t *saved_state; +#else + x86_saved_state64_t *saved_state; +#endif int code; user_addr_t vaddr; int type; @@ -331,38 +556,61 @@ kernel_trap( vm_prot_t prot; struct recovery *rp; vm_offset_t kern_ip; +#if NCOPY_WINDOWS > 0 int fault_in_copy_window = -1; +#endif int is_user = 0; -#if MACH_KDB +#if MACH_KDB pt_entry_t *pte; #endif /* MACH_KDB */ - + thread = current_thread(); - if (is_saved_state64(state)) - panic("kernel_trap(%p) with 64-bit state", state); +#ifdef __i386__ + if (__improbable(is_saved_state64(state))) { + panic_64(state, 0, "Kernel trap with 64-bit state", FALSE); + } + saved_state = saved_state32(state); + /* Record cpu where state was captured (trampolines don't set this) */ + saved_state->cpu = cpu_number(); + vaddr = (user_addr_t)saved_state->cr2; type = saved_state->trapno; code = saved_state->err & 0xffff; intr = (saved_state->efl & EFL_IF) != 0; /* state of ints at trap */ - kern_ip = (vm_offset_t)saved_state->eip; +#else + if (__improbable(is_saved_state32(state))) + panic("kernel_trap(%p) with 32-bit state", state); + saved_state = saved_state64(state); + + /* Record cpu where state was captured */ + saved_state->isf.cpu = cpu_number(); + + vaddr = (user_addr_t)saved_state->cr2; + type = saved_state->isf.trapno; + code = (int)(saved_state->isf.err & 0xffff); + intr = (saved_state->isf.rflags & EFL_IF) != 0; /* state of ints at trap */ + kern_ip = (vm_offset_t)saved_state->isf.rip; +#endif myast = ast_pending(); - if (perfASTHook) { + perfASTCallback astfn = perfASTHook; + if (__improbable(astfn != NULL)) { if (*myast & AST_CHUD_ALL) - perfASTHook(type, NULL, 0, 0); + astfn(AST_CHUD_ALL, myast); } else *myast &= ~AST_CHUD_ALL; /* * Is there a hook? */ - if (perfTrapHook) { - if (perfTrapHook(type, NULL, 0, 0) == KERN_SUCCESS) { + perfCallback fn = perfTrapHook; + if (__improbable(fn != NULL)) { + if (fn(type, NULL, 0, 0) == KERN_SUCCESS) { /* * If it succeeds, we are done... */ @@ -371,8 +619,8 @@ kernel_trap( } #if CONFIG_DTRACE - if (tempDTraceTrapHook) { - if (tempDTraceTrapHook(type, state, 0, 0) == KERN_SUCCESS) { + if (__improbable(tempDTraceTrapHook != NULL)) { + if (tempDTraceTrapHook(type, state, lo_spp, 0) == KERN_SUCCESS) { /* * If it succeeds, we are done... */ @@ -386,7 +634,7 @@ kernel_trap( * on preemption below. but we do want to re-enable interrupts * as soon we possibly can to hold latency down */ - if (T_PREEMPT == type) { + if (__improbable(T_PREEMPT == type)) { ast_taken(AST_PREEMPTION, FALSE); KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_EXCP_KTRAP_x86, type)) | DBG_FUNC_NONE, @@ -400,24 +648,25 @@ kernel_trap( */ map = kernel_map; - if (thread != THREAD_NULL && thread->map != kernel_map) { - vm_offset_t copy_window_base; + if (__probable(thread != THREAD_NULL && thread->map != kernel_map)) { +#if NCOPY_WINDOWS > 0 + vm_offset_t copy_window_base; vm_offset_t kvaddr; int window_index; kvaddr = (vm_offset_t)vaddr; - /* + /* * must determine if fault occurred in * the copy window while pre-emption is * disabled for this processor so that * we only need to look at the window * associated with this processor */ - copy_window_base = current_cpu_datap()->cpu_copywindow_base; + copy_window_base = current_cpu_datap()->cpu_copywindow_base; if (kvaddr >= copy_window_base && kvaddr < (copy_window_base + (NBPDE * NCOPY_WINDOWS)) ) { - window_index = (kvaddr - copy_window_base) / NBPDE; + window_index = (int)((kvaddr - copy_window_base) / NBPDE); if (thread->machine.copy_window[window_index].user_base != (user_addr_t)-1) { @@ -429,10 +678,30 @@ kernel_trap( } is_user = -1; } +#else + if (vaddr < VM_MAX_USER_PAGE_ADDRESS) { + /* fault occurred in userspace */ + map = thread->map; + is_user = -1; + /* + * If we're not sharing cr3 with the user + * and we faulted in copyio, + * then switch cr3 here and dismiss the fault. + */ + if (no_shared_cr3 && + (thread->machine.specFlags&CopyIOActive) && + map->pmap->pm_cr3 != get_cr3_base()) { + pmap_assert(current_cpu_datap()->cpu_pmap_pcid_enabled == FALSE); + set_cr3_raw(map->pmap->pm_cr3); + return; + } + } +#endif } } - KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_EXCP_KTRAP_x86, type)) | DBG_FUNC_NONE, - (int)(vaddr >> 32), (int)vaddr, is_user, kern_ip, 0); + KERNEL_DEBUG_CONSTANT( + (MACHDBG_CODE(DBG_MACH_EXCP_KTRAP_x86, type)) | DBG_FUNC_NONE, + (unsigned)(vaddr >> 32), (unsigned)vaddr, is_user, kern_ip, 0); (void) ml_set_interrupts_enabled(intr); @@ -455,11 +724,10 @@ kernel_trap( fpSSEexterrflt(); return; case T_DEBUG: -#if MACH_KDP - if ((saved_state->efl & EFL_TF) == 0 - && !kdp_has_active_watchpoints) +#ifdef __i386__ + if ((saved_state->efl & EFL_TF) == 0 && NO_WATCHPOINTS) #else - if ((saved_state->efl & EFL_TF) == 0) + if ((saved_state->isf.rflags & EFL_TF) == 0 && NO_WATCHPOINTS) #endif { /* We've somehow encountered a debug @@ -471,6 +739,10 @@ kernel_trap( return; } goto debugger_entry; +#ifdef __x86_64__ + case T_INT3: + goto debugger_entry; +#endif case T_PAGE_FAULT: /* * If the current map is a submap of the kernel map, @@ -508,7 +780,7 @@ kernel_trap( #endif /* MACH_KDB */ #if CONFIG_DTRACE - if (thread->options & TH_OPT_DTRACE) { /* Executing under dtrace_probe? */ + if (thread != THREAD_NULL && thread->options & TH_OPT_DTRACE) { /* Executing under dtrace_probe? */ if (dtrace_tally_fault(vaddr)) { /* Should a fault under dtrace be ignored? */ /* * DTrace has "anticipated" the possibility of this fault, and has @@ -539,37 +811,14 @@ look_for_watchpoints: #endif /* MACH_KDB */ if (result == KERN_SUCCESS) { - +#if NCOPY_WINDOWS > 0 if (fault_in_copy_window != -1) { - pt_entry_t *updp; - pt_entry_t *kpdp; - - /* - * in case there was no page table assigned - * for the user base address and the pmap - * got 'expanded' due to this fault, we'll - * copy in the descriptor - * - * we're either setting the page table descriptor - * to the same value or it was 0... no need - * for a TLB flush in either case - */ - - ml_set_interrupts_enabled(FALSE); - updp = pmap_pde(map->pmap, thread->machine.copy_window[fault_in_copy_window].user_base); - assert(updp); - if (0 == updp) panic("trap: updp 0"); /* XXX DEBUG */ - kpdp = current_cpu_datap()->cpu_copywindow_pdp; - kpdp += fault_in_copy_window; - -#if JOE_DEBUG - if (*kpdp && (*kpdp & PG_FRAME) != (*updp & PG_FRAME)) - panic("kernel_fault: user pdp doesn't match - updp = 0x%x, kpdp = 0x%x\n", updp, kpdp); -#endif - pmap_store_pte(kpdp, *updp); - + ml_set_interrupts_enabled(FALSE); + copy_window_fault(thread, map, + fault_in_copy_window); (void) ml_set_interrupts_enabled(intr); } +#endif /* NCOPY_WINDOWS > 0 */ return; } /* @@ -594,8 +843,8 @@ FALL_THROUGH: /* * Check thread recovery address also. */ - if (thread->recover) { - set_recovery_ip(saved_state, thread->recover); + if (thread != THREAD_NULL && thread->recover) { + set_recovery_ip(saved_state, thread->recover); thread->recover = 0; return; } @@ -605,7 +854,6 @@ FALL_THROUGH: * * fall through... */ - default: /* * Exception 15 is reserved but some chips may generate it @@ -615,19 +863,22 @@ FALL_THROUGH: kprintf("kernel_trap() ignoring spurious trap 15\n"); return; } +#if defined(__x86_64__) && DEBUG + kprint_state(saved_state); +#endif debugger_entry: /* Ensure that the i386_kernel_state at the base of the * current thread's stack (if any) is synchronized with the * context at the moment of the trap, to facilitate * access through the debugger. */ - sync_iss_to_iks(saved_state); + sync_iss_to_iks(state); #if MACH_KDB restart_debugger: #endif /* MACH_KDB */ #if MACH_KDP if (current_debugger != KDB_CUR_DB) { - if (kdp_i386_trap(type, saved_state, result, vaddr)) + if (kdp_i386_trap(type, saved_state, result, (vm_offset_t)vaddr)) return; } else { #endif /* MACH_KDP */ @@ -645,7 +896,7 @@ restart_debugger: } #endif } - + __asm__ volatile("cli":::"cc"); panic_trap(saved_state); /* * NO RETURN @@ -653,21 +904,29 @@ restart_debugger: } +#ifdef __i386__ static void set_recovery_ip(x86_saved_state32_t *saved_state, vm_offset_t ip) { saved_state->eip = ip; } +#else +static void +set_recovery_ip(x86_saved_state64_t *saved_state, vm_offset_t ip) +{ + saved_state->isf.rip = ip; +} +#endif +#ifdef __i386__ static void panic_trap(x86_saved_state32_t *regs) { const char *trapname = "Unknown"; - uint32_t cr0 = get_cr0(); - uint32_t cr2 = get_cr2(); - uint32_t cr3 = get_cr3(); - uint32_t cr4 = get_cr4(); + pal_cr_t cr0, cr2, cr3, cr4; + + pal_get_control_registers( &cr0, &cr2, &cr3, &cr4 ); /* * Issue an I/O port read if one has been requested - this is an @@ -691,7 +950,7 @@ panic_trap(x86_saved_state32_t *regs) regs->eip, regs->trapno, trapname, cr0, cr2, cr3, cr4, regs->eax,regs->ebx,regs->ecx,regs->edx, regs->cr2,regs->ebp,regs->esi,regs->edi, - regs->efl,regs->eip,regs->cs, regs->ds, regs->err); + regs->efl,regs->eip,regs->cs & 0xFFFF, regs->ds & 0xFFFF, regs->err); /* * This next statement is not executed, * but it's needed to stop the compiler using tail call optimization @@ -699,239 +958,54 @@ panic_trap(x86_saved_state32_t *regs) */ cr0 = 0; } - -extern void kprintf_break_lock(void); - - -/* - * Called from locore on a special reserved stack after a double-fault - * is taken in kernel space. - * Kernel stack overflow is one route here. - */ -void -panic_double_fault( -#if CONFIG_NO_PANIC_STRINGS - __unused int code -#else - int code -#endif - ) -{ -#if MACH_KDP || !CONFIG_NO_PANIC_STRINGS - struct i386_tss *my_ktss = current_ktss(); -#endif - - /* Set postcode (DEBUG only) */ - postcode(PANIC_DOUBLE_FAULT); - - /* - * Issue an I/O port read if one has been requested - this is an - * event logic analyzers can use as a trigger point. - */ - panic_io_port_read(); - - /* - * Break kprintf lock in case of recursion, - * and record originally faulted instruction address. - */ - kprintf_break_lock(); - -#if MACH_KDP - /* - * Print backtrace leading to first fault: - */ - panic_i386_backtrace((void *) my_ktss->ebp, 10, NULL, FALSE, NULL); -#endif - - panic("Double fault at 0x%08x, thread:%p, code:0x%x, " - "registers:\n" - "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n" - "EAX: 0x%08x, EBX: 0x%08x, ECX: 0x%08x, EDX: 0x%08x\n" - "ESP: 0x%08x, EBP: 0x%08x, ESI: 0x%08x, EDI: 0x%08x\n" - "EFL: 0x%08x, EIP: 0x%08x\n", - my_ktss->eip, current_thread(), code, - get_cr0(), get_cr2(), get_cr3(), get_cr4(), - my_ktss->eax, my_ktss->ebx, my_ktss->ecx, my_ktss->edx, - my_ktss->esp, my_ktss->ebp, my_ktss->esi, my_ktss->edi, - my_ktss->eflags, my_ktss->eip); -} - - -/* - * Called from locore on a special reserved stack after a machine-check - */ -void -panic_machine_check( -#if CONFIG_NO_PANIC_STRINGS - __unused int code #else - int code -#endif - ) -{ -#if !CONFIG_NO_PANIC_STRINGS - struct i386_tss *my_ktss = current_ktss(); -#endif - - /* Set postcode (DEBUG only) */ - postcode(PANIC_MACHINE_CHECK); - - /* - * Issue an I/O port read if one has been requested - this is an - * event logic analyzers can use as a trigger point. - */ - panic_io_port_read(); - - /* - * Break kprintf lock in case of recursion, - * and record originally faulted instruction address. - */ - kprintf_break_lock(); - - /* - * Dump the contents of the machine check MSRs (if any). - */ - mca_dump(); - - /* - * And that's all folks, we don't attempt recovery... - */ - panic("Machine-check at 0x%08x, thread:%p, code:0x%x, " - "registers:\n" - "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n" - "EAX: 0x%08x, EBX: 0x%08x, ECX: 0x%08x, EDX: 0x%08x\n" - "ESP: 0x%08x, EBP: 0x%08x, ESI: 0x%08x, EDI: 0x%08x\n" - "EFL: 0x%08x, EIP: 0x%08x\n", - my_ktss->eip, current_thread(), code, - get_cr0(), get_cr2(), get_cr3(), get_cr4(), - my_ktss->eax, my_ktss->ebx, my_ktss->ecx, my_ktss->edx, - my_ktss->esp, my_ktss->ebp, my_ktss->esi, my_ktss->edi, - my_ktss->eflags, my_ktss->eip); -} - -void -panic_double_fault64(x86_saved_state_t *esp) +static void +panic_trap(x86_saved_state64_t *regs) { - /* Set postcode (DEBUG only) */ - postcode(PANIC_DOUBLE_FAULT); + const char *trapname = "Unknown"; + pal_cr_t cr0, cr2, cr3, cr4; + pal_get_control_registers( &cr0, &cr2, &cr3, &cr4 ); + assert(ml_get_interrupts_enabled() == FALSE); + current_cpu_datap()->cpu_fatal_trap_state = regs; /* * Issue an I/O port read if one has been requested - this is an * event logic analyzers can use as a trigger point. */ panic_io_port_read(); - /* - * Break kprintf lock in case of recursion, - * and record originally faulted instruction address. - */ - kprintf_break_lock(); + kprintf("panic trap number 0x%x, rip 0x%016llx\n", + regs->isf.trapno, regs->isf.rip); + kprintf("cr0 0x%016llx cr2 0x%016llx cr3 0x%016llx cr4 0x%016llx\n", + cr0, cr2, cr3, cr4); + if (regs->isf.trapno < TRAP_TYPES) + trapname = trap_type[regs->isf.trapno]; +#undef panic + panic("Kernel trap at 0x%016llx, type %d=%s, registers:\n" + "CR0: 0x%016llx, CR2: 0x%016llx, CR3: 0x%016llx, CR4: 0x%016llx\n" + "RAX: 0x%016llx, RBX: 0x%016llx, RCX: 0x%016llx, RDX: 0x%016llx\n" + "RSP: 0x%016llx, RBP: 0x%016llx, RSI: 0x%016llx, RDI: 0x%016llx\n" + "R8: 0x%016llx, R9: 0x%016llx, R10: 0x%016llx, R11: 0x%016llx\n" + "R12: 0x%016llx, R13: 0x%016llx, R14: 0x%016llx, R15: 0x%016llx\n" + "RFL: 0x%016llx, RIP: 0x%016llx, CS: 0x%016llx, SS: 0x%016llx\n" + "CR2: 0x%016llx, Error code: 0x%016llx, Faulting CPU: 0x%x\n", + regs->isf.rip, regs->isf.trapno, trapname, + cr0, cr2, cr3, cr4, + regs->rax, regs->rbx, regs->rcx, regs->rdx, + regs->isf.rsp, regs->rbp, regs->rsi, regs->rdi, + regs->r8, regs->r9, regs->r10, regs->r11, + regs->r12, regs->r13, regs->r14, regs->r15, + regs->isf.rflags, regs->isf.rip, regs->isf.cs & 0xFFFF, + regs->isf.ss & 0xFFFF,regs->cr2, regs->isf.err, regs->isf.cpu); /* - * Dump the interrupt stack frame at last kernel entry. + * This next statement is not executed, + * but it's needed to stop the compiler using tail call optimization + * for the panic call - which confuses the subsequent backtrace. */ - if (is_saved_state64(esp)) { -#if !CONFIG_NO_PANIC_STRINGS - x86_saved_state64_t *ss64p = saved_state64(esp); -#endif - panic("Double fault thread:%p, trapno:0x%x, err:0x%qx, " - "registers:\n" - "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n" - "RAX: 0x%016qx, RBX: 0x%016qx, RCX: 0x%016qx, RDX: 0x%016qx\n" - "RSP: 0x%016qx, RBP: 0x%016qx, RSI: 0x%016qx, RDI: 0x%016qx\n" - "R8: 0x%016qx, R9: 0x%016qx, R10: 0x%016qx, R11: 0x%016qx\n" - "R12: 0x%016qx, R13: 0x%016qx, R14: 0x%016qx, R15: 0x%016qx\n" - "RFL: 0x%016qx, RIP: 0x%016qx, CR2: 0x%016qx\n", - current_thread(), ss64p->isf.trapno, ss64p->isf.err, - get_cr0(), get_cr2(), get_cr3(), get_cr4(), - ss64p->rax, ss64p->rbx, ss64p->rcx, ss64p->rdx, - ss64p->isf.rsp, ss64p->rbp, ss64p->rsi, ss64p->rdi, - ss64p->r8, ss64p->r9, ss64p->r10, ss64p->r11, - ss64p->r12, ss64p->r13, ss64p->r14, ss64p->r15, - ss64p->isf.rflags, ss64p->isf.rip, ss64p->cr2); - } else { -#if !CONFIG_NO_PANIC_STRINGS - x86_saved_state32_t *ss32p = saved_state32(esp); -#endif - panic("Double fault at 0x%08x, thread:%p, trapno:0x%x, err:0x%x)," - "registers:\n" - "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n" - "EAX: 0x%08x, EBX: 0x%08x, ECX: 0x%08x, EDX: 0x%08x\n" - "ESP: 0x%08x, EBP: 0x%08x, ESI: 0x%08x, EDI: 0x%08x\n" - "EFL: 0x%08x, EIP: 0x%08x\n", - ss32p->eip, current_thread(), ss32p->trapno, ss32p->err, - get_cr0(), get_cr2(), get_cr3(), get_cr4(), - ss32p->eax, ss32p->ebx, ss32p->ecx, ss32p->edx, - ss32p->uesp, ss32p->ebp, ss32p->esi, ss32p->edi, - ss32p->efl, ss32p->eip); - } + cr0 = 0; } - -/* - * Machine check handler for 64-bit. - */ -void -panic_machine_check64(x86_saved_state_t *esp) -{ - /* Set postcode (DEBUG only) */ - postcode(PANIC_MACHINE_CHECK); - - /* - * Issue an I/O port read if one has been requested - this is an - * event logic analyzers can use as a trigger point. - */ - panic_io_port_read(); - - /* - * Break kprintf lock in case of recursion, - * and record originally faulted instruction address. - */ - kprintf_break_lock(); - - /* - * Dump the contents of the machine check MSRs (if any). - */ - mca_dump(); - - /* - * And that's all folks, we don't attempt recovery... - */ - if (is_saved_state64(esp)) { -#if !CONFIG_NO_PANIC_STRINGS - x86_saved_state64_t *ss64p = saved_state64(esp); -#endif - panic("Machine Check thread:%p, trapno:0x%x, err:0x%qx, " - "registers:\n" - "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n" - "RAX: 0x%016qx, RBX: 0x%016qx, RCX: 0x%016qx, RDX: 0x%016qx\n" - "RSP: 0x%016qx, RBP: 0x%016qx, RSI: 0x%016qx, RDI: 0x%016qx\n" - "R8: 0x%016qx, R9: 0x%016qx, R10: 0x%016qx, R11: 0x%016qx\n" - "R12: 0x%016qx, R13: 0x%016qx, R14: 0x%016qx, R15: 0x%016qx\n" - "RFL: 0x%016qx, RIP: 0x%016qx\n", - current_thread(), ss64p->isf.trapno, ss64p->isf.err, - get_cr0(), get_cr2(), get_cr3(), get_cr4(), - ss64p->rax, ss64p->rbx, ss64p->rcx, ss64p->rdx, - ss64p->isf.rsp, ss64p->rbp, ss64p->rsi, ss64p->rdi, - ss64p->r8, ss64p->r9, ss64p->r10, ss64p->r11, - ss64p->r12, ss64p->r13, ss64p->r14, ss64p->r15, - ss64p->isf.rflags, ss64p->isf.rip); - } else { -#if !CONFIG_NO_PANIC_STRINGS - x86_saved_state32_t *ss32p = saved_state32(esp); #endif - panic("Machine Check at 0x%08x, thread:%p, trapno:0x%x, err:0x%x, " - "registers:\n" - "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n" - "EAX: 0x%08x, EBX: 0x%08x, ECX: 0x%08x, EDX: 0x%08x\n" - "ESP: 0x%08x, EBP: 0x%08x, ESI: 0x%08x, EDI: 0x%08x\n" - "EFL: 0x%08x, EIP: 0x%08x\n", - ss32p->eip, current_thread(), ss32p->trapno, ss32p->err, - get_cr0(), get_cr2(), get_cr3(), get_cr4(), - ss32p->eax, ss32p->ebx, ss32p->ecx, ss32p->edx, - ss32p->uesp, ss32p->ebp, ss32p->esi, ss32p->edi, - ss32p->efl, ss32p->eip); - } -} #if CONFIG_DTRACE extern kern_return_t dtrace_user_probe(x86_saved_state_t *); @@ -955,6 +1029,7 @@ user_trap( ast_t *myast; kern_return_t kret; user_addr_t rip; + unsigned long dr6 = 0; /* 32 bit for i386, 64 bit for x86_64 */ assert((is_saved_state32(saved_state) && !thread_is_64bit(thread)) || (is_saved_state64(saved_state) && thread_is_64bit(thread))); @@ -964,8 +1039,11 @@ user_trap( regs = saved_state64(saved_state); + /* Record cpu where state was captured */ + regs->isf.cpu = cpu_number(); + type = regs->isf.trapno; - err = regs->isf.err & 0xffff; + err = (int)regs->isf.err & 0xffff; vaddr = (user_addr_t)regs->cr2; rip = (user_addr_t)regs->isf.rip; } else { @@ -973,14 +1051,30 @@ user_trap( regs = saved_state32(saved_state); + /* Record cpu where state was captured */ + regs->cpu = cpu_number(); + type = regs->trapno; err = regs->err & 0xffff; vaddr = (user_addr_t)regs->cr2; rip = (user_addr_t)regs->eip; } - KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_EXCP_UTRAP_x86, type)) | DBG_FUNC_NONE, - (int)(vaddr>>32), (int)vaddr, (int)(rip>>32), (int)rip, 0); + if ((type == T_DEBUG) && thread->machine.ids) { + unsigned long clear = 0; + /* Stash and clear this processor's DR6 value, in the event + * this was a debug register match + */ + __asm__ volatile ("mov %%db6, %0" : "=r" (dr6)); + __asm__ volatile ("mov %0, %%db6" : : "r" (clear)); + } + + pal_sti(); + + KERNEL_DEBUG_CONSTANT( + (MACHDBG_CODE(DBG_MACH_EXCP_UTRAP_x86, type)) | DBG_FUNC_NONE, + (unsigned)(vaddr>>32), (unsigned)vaddr, + (unsigned)(rip>>32), (unsigned)rip, 0); code = 0; subcode = 0; @@ -990,18 +1084,19 @@ user_trap( kprintf("user_trap(0x%08x) type=%d vaddr=0x%016llx\n", saved_state, type, vaddr); #endif - myast = ast_pending(); - if (perfASTHook) { + + perfASTCallback astfn = perfASTHook; + if (__improbable(astfn != NULL)) { + myast = ast_pending(); if (*myast & AST_CHUD_ALL) { - perfASTHook(type, saved_state, 0, 0); + astfn(AST_CHUD_ALL, myast); } - } else { - *myast &= ~AST_CHUD_ALL; } /* Is there a hook? */ - if (perfTrapHook) { - if (perfTrapHook(type, saved_state, 0, 0) == KERN_SUCCESS) + perfCallback fn = perfTrapHook; + if (__improbable(fn != NULL)) { + if (fn(type, saved_state, 0, 0) == KERN_SUCCESS) return; /* If it succeeds, we are done... */ } @@ -1010,7 +1105,10 @@ user_trap( * Avoid needlessly calling tempDTraceTrapHook here, and let the * INT_3 case handle them. */ - + DEBUG_KPRINT_SYSCALL_MASK(1, + "user_trap: type=0x%x(%s) err=0x%x cr2=%p rip=%p\n", + type, trap_type[type], err, (void *)(long) vaddr, (void *)(long) rip); + switch (type) { case T_DIVIDE_ERROR: @@ -1021,12 +1119,11 @@ user_trap( case T_DEBUG: { pcb_t pcb; - unsigned int clear = 0; /* - * get dr6 and set it in the thread's pcb before - * returning to userland + * Update the PCB with this processor's DR6 value + * in the event this was a debug register match. */ - pcb = thread->machine.pcb; + pcb = THREAD_TO_PCB(thread); if (pcb->ids) { /* * We can get and set the status register @@ -1035,16 +1132,12 @@ user_trap( * used on x86_64 */ if (thread_is_64bit(thread)) { - uint32_t dr6; x86_debug_state64_t *ids = pcb->ids; - dr6 = (uint32_t)ids->dr6; - __asm__ volatile ("movl %%db6, %0" : "=r" (dr6)); ids->dr6 = dr6; } else { /* 32 bit thread */ x86_debug_state32_t *ids = pcb->ids; - __asm__ volatile ("movl %%db6, %0" : "=r" (ids->dr6)); + ids->dr6 = (uint32_t) dr6; } - __asm__ volatile ("movl %0, %%db6" : : "r" (clear)); } exc = EXC_BREAKPOINT; code = EXC_I386_SGL; @@ -1126,7 +1219,7 @@ user_trap( if (err & T_PF_WRITE) prot |= VM_PROT_WRITE; #if PAE - if (err & T_PF_EXECUTE) + if (__improbable(err & T_PF_EXECUTE)) prot |= VM_PROT_EXECUTE; #endif kret = vm_fault(thread->map, vm_map_trunc_page(vaddr), @@ -1182,8 +1275,6 @@ user_trap( /* * Handle AST traps for i386. - * Check for delayed floating-point exception from - * AT-bus machines. */ extern void log_thread_action (thread_t, char *); @@ -1222,6 +1313,8 @@ i386_exception( { mach_exception_data_type_t codes[EXCEPTION_CODE_MAX]; + DEBUG_KPRINT_SYSCALL_MACH("i386_exception: exc=%d code=0x%llx subcode=0x%llx\n", + exc, code, subcode); codes[0] = code; /* new exception interface */ codes[1] = subcode; exception_triage(exc, codes, 2); @@ -1229,43 +1322,6 @@ i386_exception( } -void -kernel_preempt_check(void) -{ - ast_t *myast; - boolean_t intr; - - /* - * disable interrupts to both prevent pre-emption - * and to keep the ast state from changing via - * an interrupt handler making something runnable - */ - intr = ml_set_interrupts_enabled(FALSE); - - myast = ast_pending(); - - if ((*myast & AST_URGENT) && intr == TRUE && get_interrupt_level() == 0) { - /* - * can handle interrupts and preemptions - * at this point - */ - ml_set_interrupts_enabled(intr); - - /* - * now cause the PRE-EMPTION trap - */ - __asm__ volatile (" int $0xff"); - } else { - /* - * if interrupts were already disabled or - * we're in an interrupt context, we can't - * preempt... of course if AST_URGENT - * isn't set we also don't want to - */ - ml_set_interrupts_enabled(intr); - } -} - #if MACH_KDB extern void db_i386_state(x86_saved_state32_t *regs); @@ -1306,22 +1362,27 @@ db_i386_state( */ void -sync_iss_to_iks(x86_saved_state32_t *saved_state) +sync_iss_to_iks(x86_saved_state_t *saved_state) { - struct x86_kernel_state32 *iks; + struct x86_kernel_state *iks; vm_offset_t kstack; boolean_t record_active_regs = FALSE; - if ((kstack = current_thread()->kernel_stack) != 0) { - x86_saved_state32_t *regs; + /* The PAL may have a special way to sync registers */ + if( saved_state->flavor == THREAD_STATE_NONE ) + pal_get_kern_regs( saved_state ); - regs = saved_state; + if ((kstack = current_thread()->kernel_stack) != 0) { +#ifdef __i386__ + x86_saved_state32_t *regs = saved_state32(saved_state); +#else + x86_saved_state64_t *regs = saved_state64(saved_state); +#endif iks = STACK_IKS(kstack); - /* - * Did we take the trap/interrupt in kernel mode? - */ + /* Did we take the trap/interrupt in kernel mode? */ +#ifdef __i386__ if (regs == USER_REGS32(current_thread())) record_active_regs = TRUE; else { @@ -1332,21 +1393,47 @@ sync_iss_to_iks(x86_saved_state32_t *saved_state) iks->k_esi = regs->esi; iks->k_eip = regs->eip; } +#else + if (regs == USER_REGS64(current_thread())) + record_active_regs = TRUE; + else { + iks->k_rbx = regs->rbx; + iks->k_rsp = regs->isf.rsp; + iks->k_rbp = regs->rbp; + iks->k_r12 = regs->r12; + iks->k_r13 = regs->r13; + iks->k_r14 = regs->r14; + iks->k_r15 = regs->r15; + iks->k_rip = regs->isf.rip; + } +#endif } if (record_active_regs == TRUE) { - /* - * Show the trap handler path - */ +#ifdef __i386__ + /* Show the trap handler path */ __asm__ volatile("movl %%ebx, %0" : "=m" (iks->k_ebx)); __asm__ volatile("movl %%esp, %0" : "=m" (iks->k_esp)); __asm__ volatile("movl %%ebp, %0" : "=m" (iks->k_ebp)); __asm__ volatile("movl %%edi, %0" : "=m" (iks->k_edi)); __asm__ volatile("movl %%esi, %0" : "=m" (iks->k_esi)); - /* - * "Current" instruction pointer - */ + /* "Current" instruction pointer */ __asm__ volatile("movl $1f, %0\n1:" : "=m" (iks->k_eip)); +#else + /* Show the trap handler path */ + __asm__ volatile("movq %%rbx, %0" : "=m" (iks->k_rbx)); + __asm__ volatile("movq %%rsp, %0" : "=m" (iks->k_rsp)); + __asm__ volatile("movq %%rbp, %0" : "=m" (iks->k_rbp)); + __asm__ volatile("movq %%r12, %0" : "=m" (iks->k_r12)); + __asm__ volatile("movq %%r13, %0" : "=m" (iks->k_r13)); + __asm__ volatile("movq %%r14, %0" : "=m" (iks->k_r14)); + __asm__ volatile("movq %%r15, %0" : "=m" (iks->k_r15)); + /* "Current" instruction pointer */ + __asm__ volatile("leaq 1f(%%rip), %%rax; mov %%rax, %0\n1:" + : "=m" (iks->k_rip) + : + : "rax"); +#endif } } @@ -1358,22 +1445,31 @@ sync_iss_to_iks(x86_saved_state32_t *saved_state) */ void sync_iss_to_iks_unconditionally(__unused x86_saved_state_t *saved_state) { - struct x86_kernel_state32 *iks; + struct x86_kernel_state *iks; vm_offset_t kstack; if ((kstack = current_thread()->kernel_stack) != 0) { iks = STACK_IKS(kstack); - /* - * Display the trap handler path. - */ +#ifdef __i386__ + /* Display the trap handler path */ __asm__ volatile("movl %%ebx, %0" : "=m" (iks->k_ebx)); __asm__ volatile("movl %%esp, %0" : "=m" (iks->k_esp)); __asm__ volatile("movl %%ebp, %0" : "=m" (iks->k_ebp)); __asm__ volatile("movl %%edi, %0" : "=m" (iks->k_edi)); __asm__ volatile("movl %%esi, %0" : "=m" (iks->k_esi)); - /* - * "Current" instruction pointer. - */ + /* "Current" instruction pointer */ __asm__ volatile("movl $1f, %0\n1:" : "=m" (iks->k_eip)); +#else + /* Display the trap handler path */ + __asm__ volatile("movq %%rbx, %0" : "=m" (iks->k_rbx)); + __asm__ volatile("movq %%rsp, %0" : "=m" (iks->k_rsp)); + __asm__ volatile("movq %%rbp, %0" : "=m" (iks->k_rbp)); + __asm__ volatile("movq %%r12, %0" : "=m" (iks->k_r12)); + __asm__ volatile("movq %%r13, %0" : "=m" (iks->k_r13)); + __asm__ volatile("movq %%r14, %0" : "=m" (iks->k_r14)); + __asm__ volatile("movq %%r15, %0" : "=m" (iks->k_r15)); + /* "Current" instruction pointer */ + __asm__ volatile("leaq 1f(%%rip), %%rax; mov %%rax, %0\n1:" : "=m" (iks->k_rip)::"rax"); +#endif } }