+extern void PE_incoming_interrupt(int interrupt);
+
+#if defined(__x86_64__) && DEBUG
+void
+kprint_state(x86_saved_state64_t *saved_state)
+{
+ kprintf("current_cpu_datap() 0x%lx\n", (uintptr_t)current_cpu_datap());
+ kprintf("Current GS base MSR 0x%llx\n", rdmsr64(MSR_IA32_GS_BASE));
+ kprintf("Kernel GS base MSR 0x%llx\n", rdmsr64(MSR_IA32_KERNEL_GS_BASE));
+ kprintf("state at 0x%lx:\n", (uintptr_t) saved_state);
+
+ kprintf(" rdi 0x%llx\n", saved_state->rdi);
+ kprintf(" rsi 0x%llx\n", saved_state->rsi);
+ kprintf(" rdx 0x%llx\n", saved_state->rdx);
+ kprintf(" r10 0x%llx\n", saved_state->r10);
+ kprintf(" r8 0x%llx\n", saved_state->r8);
+ kprintf(" r9 0x%llx\n", saved_state->r9);
+ kprintf(" v_arg6 0x%llx\n", saved_state->v_arg6);
+ kprintf(" v_arg7 0x%llx\n", saved_state->v_arg7);
+ kprintf(" v_arg8 0x%llx\n", saved_state->v_arg8);
+
+ kprintf(" cr2 0x%llx\n", saved_state->cr2);
+ kprintf("real cr2 0x%lx\n", get_cr2());
+ kprintf(" r15 0x%llx\n", saved_state->r15);
+ kprintf(" r14 0x%llx\n", saved_state->r14);
+ kprintf(" r13 0x%llx\n", saved_state->r13);
+ kprintf(" r12 0x%llx\n", saved_state->r12);
+ kprintf(" r11 0x%llx\n", saved_state->r11);
+ kprintf(" rbp 0x%llx\n", saved_state->rbp);
+ kprintf(" rbx 0x%llx\n", saved_state->rbx);
+ kprintf(" rcx 0x%llx\n", saved_state->rcx);
+ kprintf(" rax 0x%llx\n", saved_state->rax);
+
+ kprintf(" gs 0x%x\n", saved_state->gs);
+ kprintf(" fs 0x%x\n", saved_state->fs);
+
+ kprintf(" isf.trapno 0x%x\n", saved_state->isf.trapno);
+ kprintf(" isf._pad 0x%x\n", saved_state->isf._pad);
+ kprintf(" isf.trapfn 0x%llx\n", saved_state->isf.trapfn);
+ kprintf(" isf.err 0x%llx\n", saved_state->isf.err);
+ kprintf(" isf.rip 0x%llx\n", saved_state->isf.rip);
+ kprintf(" isf.cs 0x%llx\n", saved_state->isf.cs);
+ kprintf(" isf.rflags 0x%llx\n", saved_state->isf.rflags);
+ kprintf(" isf.rsp 0x%llx\n", saved_state->isf.rsp);
+ kprintf(" isf.ss 0x%llx\n", saved_state->isf.ss);
+}
+#endif
+
+
+/*
+ * Non-zero indicates latency assert is enabled and capped at valued
+ * absolute time units.
+ */
+
+uint64_t interrupt_latency_cap = 0;
+boolean_t ilat_assert = FALSE;
+
+void
+interrupt_latency_tracker_setup(void) {
+ uint32_t ilat_cap_us;
+ if (PE_parse_boot_argn("interrupt_latency_cap_us", &ilat_cap_us, sizeof(ilat_cap_us))) {
+ interrupt_latency_cap = ilat_cap_us * NSEC_PER_USEC;
+ nanoseconds_to_absolutetime(interrupt_latency_cap, &interrupt_latency_cap);
+ } else {
+ interrupt_latency_cap = LockTimeOut;
+ }
+ PE_parse_boot_argn("-interrupt_latency_assert_enable", &ilat_assert, sizeof(ilat_assert));
+}
+
+void interrupt_reset_latency_stats(void) {
+ uint32_t i;
+ for (i = 0; i < real_ncpus; i++) {
+ cpu_data_ptr[i]->cpu_max_observed_int_latency =
+ cpu_data_ptr[i]->cpu_max_observed_int_latency_vector = 0;
+ }
+}
+
+void interrupt_populate_latency_stats(char *buf, unsigned bufsize) {
+ uint32_t i, tcpu = ~0;
+ uint64_t cur_max = 0;
+
+ for (i = 0; i < real_ncpus; i++) {
+ if (cur_max < cpu_data_ptr[i]->cpu_max_observed_int_latency) {
+ cur_max = cpu_data_ptr[i]->cpu_max_observed_int_latency;
+ tcpu = i;
+ }
+ }
+
+ if (tcpu < real_ncpus)
+ snprintf(buf, bufsize, "0x%x 0x%x 0x%llx", tcpu, cpu_data_ptr[tcpu]->cpu_max_observed_int_latency_vector, cpu_data_ptr[tcpu]->cpu_max_observed_int_latency);
+}
+
+/*
+ * Handle interrupts:
+ * - local APIC interrupts (IPIs, timers, etc) are handled by the kernel,
+ * - device interrupts go to the platform expert.
+ */
+void
+interrupt(x86_saved_state_t *state)
+{
+ uint64_t rip;
+ uint64_t rsp;
+ int interrupt_num;
+ boolean_t user_mode = FALSE;
+ int ipl;
+ int cnum = cpu_number();
+ int itype = 0;
+
+ if (is_saved_state64(state) == TRUE) {
+ x86_saved_state64_t *state64;
+
+ state64 = saved_state64(state);
+ rip = state64->isf.rip;
+ rsp = state64->isf.rsp;
+ interrupt_num = state64->isf.trapno;
+#ifdef __x86_64__
+ if(state64->isf.cs & 0x03)
+#endif
+ user_mode = TRUE;
+ } else {
+ x86_saved_state32_t *state32;
+
+ state32 = saved_state32(state);
+ if (state32->cs & 0x03)
+ user_mode = TRUE;
+ rip = state32->eip;
+ rsp = state32->uesp;
+ interrupt_num = state32->trapno;
+ }
+
+ if (cpu_data_ptr[cnum]->lcpu.package->num_idle == topoParms.nLThreadsPerPackage)
+ cpu_data_ptr[cnum]->cpu_hwIntpexits[interrupt_num]++;
+
+ if (interrupt_num == (LAPIC_DEFAULT_INTERRUPT_BASE + LAPIC_INTERPROCESSOR_INTERRUPT))
+ itype = 1;
+ else if (interrupt_num == (LAPIC_DEFAULT_INTERRUPT_BASE + LAPIC_TIMER_INTERRUPT))
+ itype = 2;
+ else
+ itype = 3;
+
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+ MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_START,
+ interrupt_num,
+ (user_mode ? rip : VM_KERNEL_UNSLIDE(rip)),
+ user_mode, itype, 0);
+
+ SCHED_STATS_INTERRUPT(current_processor());
+
+ ipl = get_preemption_level();
+
+ /*
+ * Handle local APIC interrupts
+ * else call platform expert for devices.
+ */
+ if (!lapic_interrupt(interrupt_num, state))
+ PE_incoming_interrupt(interrupt_num);
+
+ if (__improbable(get_preemption_level() != ipl)) {
+ panic("Preemption level altered by interrupt vector 0x%x: initial 0x%x, final: 0x%x\n", interrupt_num, ipl, get_preemption_level());
+ }
+
+
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+ MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_END,
+ interrupt_num, 0, 0, 0, 0);
+
+ if (cpu_data_ptr[cnum]->cpu_nested_istack) {
+ cpu_data_ptr[cnum]->cpu_nested_istack_events++;
+ }
+ else {
+ uint64_t int_latency = mach_absolute_time() - cpu_data_ptr[cnum]->cpu_int_event_time;
+ if (ilat_assert && (int_latency > interrupt_latency_cap) && !machine_timeout_suspended()) {
+ panic("Interrupt vector 0x%x exceeded interrupt latency threshold, 0x%llx absolute time delta, prior signals: 0x%x, current signals: 0x%x", interrupt_num, int_latency, cpu_data_ptr[cnum]->cpu_prior_signals, cpu_data_ptr[cnum]->cpu_signals);
+ }
+ if (int_latency > cpu_data_ptr[cnum]->cpu_max_observed_int_latency) {
+ cpu_data_ptr[cnum]->cpu_max_observed_int_latency = int_latency;
+ cpu_data_ptr[cnum]->cpu_max_observed_int_latency_vector = interrupt_num;
+ }
+ }
+
+ /*
+ * Having serviced the interrupt first, look at the interrupted stack depth.
+ */
+ if (!user_mode) {
+ uint64_t depth = cpu_data_ptr[cnum]->cpu_kernel_stack
+ + sizeof(struct x86_kernel_state)
+ + sizeof(struct i386_exception_link *)
+ - rsp;
+ if (depth > kernel_stack_depth_max) {
+ kernel_stack_depth_max = (vm_offset_t)depth;
+ KERNEL_DEBUG_CONSTANT(
+ MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_DEPTH),
+ (long) depth, (long) VM_KERNEL_UNSLIDE(rip), 0, 0, 0);
+ }
+ }
+}
projects / apple / xnu.git / blobdiff