+
+uint64_t interrupt_latency_cap = 0;
+boolean_t ilat_assert = FALSE;
+
+void
+interrupt_latency_tracker_setup(void) {
+ uint32_t ilat_cap_us;
+ if (PE_parse_boot_argn("interrupt_latency_cap_us", &ilat_cap_us, sizeof(ilat_cap_us))) {
+ interrupt_latency_cap = ilat_cap_us * NSEC_PER_USEC;
+ nanoseconds_to_absolutetime(interrupt_latency_cap, &interrupt_latency_cap);
+ } else {
+ interrupt_latency_cap = LockTimeOut;
+ }
+ PE_parse_boot_argn("-interrupt_latency_assert_enable", &ilat_assert, sizeof(ilat_assert));
+}
+
+void interrupt_reset_latency_stats(void) {
+ uint32_t i;
+ for (i = 0; i < real_ncpus; i++) {
+ cpu_data_ptr[i]->cpu_max_observed_int_latency =
+ cpu_data_ptr[i]->cpu_max_observed_int_latency_vector = 0;
+ }
+}
+
+void interrupt_populate_latency_stats(char *buf, unsigned bufsize) {
+ uint32_t i, tcpu = ~0;
+ uint64_t cur_max = 0;
+
+ for (i = 0; i < real_ncpus; i++) {
+ if (cur_max < cpu_data_ptr[i]->cpu_max_observed_int_latency) {
+ cur_max = cpu_data_ptr[i]->cpu_max_observed_int_latency;
+ tcpu = i;
+ }
+ }
+
+ if (tcpu < real_ncpus)
+ snprintf(buf, bufsize, "0x%x 0x%x 0x%llx", tcpu, cpu_data_ptr[tcpu]->cpu_max_observed_int_latency_vector, cpu_data_ptr[tcpu]->cpu_max_observed_int_latency);
+}
+
+uint32_t interrupt_timer_coalescing_enabled = 1;
+uint64_t interrupt_coalesced_timers;
+
+/*
+ * Handle interrupts:
+ * - local APIC interrupts (IPIs, timers, etc) are handled by the kernel,
+ * - device interrupts go to the platform expert.
+ */
+void
+interrupt(x86_saved_state_t *state)
+{
+ uint64_t rip;
+ uint64_t rsp;
+ int interrupt_num;
+ boolean_t user_mode = FALSE;
+ int ipl;
+ int cnum = cpu_number();
+ cpu_data_t *cdp = cpu_data_ptr[cnum];
+ int itype = DBG_INTR_TYPE_UNKNOWN;
+
+ x86_saved_state64_t *state64 = saved_state64(state);
+ rip = state64->isf.rip;
+ rsp = state64->isf.rsp;
+ interrupt_num = state64->isf.trapno;
+ if(state64->isf.cs & 0x03)
+ user_mode = TRUE;
+
+ if (cpu_data_ptr[cnum]->lcpu.package->num_idle == topoParms.nLThreadsPerPackage)
+ cpu_data_ptr[cnum]->cpu_hwIntpexits[interrupt_num]++;
+
+ if (interrupt_num == (LAPIC_DEFAULT_INTERRUPT_BASE + LAPIC_INTERPROCESSOR_INTERRUPT))
+ itype = DBG_INTR_TYPE_IPI;
+ else if (interrupt_num == (LAPIC_DEFAULT_INTERRUPT_BASE + LAPIC_TIMER_INTERRUPT))
+ itype = DBG_INTR_TYPE_TIMER;
+ else
+ itype = DBG_INTR_TYPE_OTHER;
+
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+ MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_START,
+ interrupt_num,
+ (user_mode ? rip : VM_KERNEL_UNSLIDE(rip)),
+ user_mode, itype, 0);
+
+ SCHED_STATS_INTERRUPT(current_processor());
+
+#if CONFIG_TELEMETRY
+ if (telemetry_needs_record) {
+ telemetry_mark_curthread(user_mode, FALSE);
+ }
+#endif
+
+ ipl = get_preemption_level();
+
+ /*
+ * Handle local APIC interrupts
+ * else call platform expert for devices.
+ */
+ if (!lapic_interrupt(interrupt_num, state)) {
+ PE_incoming_interrupt(interrupt_num);
+ }
+
+ if (__improbable(get_preemption_level() != ipl)) {
+ panic("Preemption level altered by interrupt vector 0x%x: initial 0x%x, final: 0x%x\n", interrupt_num, ipl, get_preemption_level());
+ }
+
+
+ if (__improbable(cdp->cpu_nested_istack)) {
+ cdp->cpu_nested_istack_events++;
+ }
+ else {
+ uint64_t ctime = mach_absolute_time();
+ uint64_t int_latency = ctime - cdp->cpu_int_event_time;
+ uint64_t esdeadline, ehdeadline;
+ /* Attempt to process deferred timers in the context of
+ * this interrupt, unless interrupt time has already exceeded
+ * TCOAL_ILAT_THRESHOLD.
+ */
+#define TCOAL_ILAT_THRESHOLD (30000ULL)
+
+ if ((int_latency < TCOAL_ILAT_THRESHOLD) &&
+ interrupt_timer_coalescing_enabled) {
+ esdeadline = cdp->rtclock_timer.queue.earliest_soft_deadline;
+ ehdeadline = cdp->rtclock_timer.deadline;
+ if ((ctime >= esdeadline) && (ctime < ehdeadline)) {
+ interrupt_coalesced_timers++;
+ TCOAL_DEBUG(0x88880000 | DBG_FUNC_START, ctime, esdeadline, ehdeadline, interrupt_coalesced_timers, 0);
+ rtclock_intr(state);
+ TCOAL_DEBUG(0x88880000 | DBG_FUNC_END, ctime, esdeadline, interrupt_coalesced_timers, 0, 0);
+ } else {
+ TCOAL_DEBUG(0x77770000, ctime, cdp->rtclock_timer.queue.earliest_soft_deadline, cdp->rtclock_timer.deadline, interrupt_coalesced_timers, 0);
+ }
+ }
+
+ if (__improbable(ilat_assert && (int_latency > interrupt_latency_cap) && !machine_timeout_suspended())) {
+ panic("Interrupt vector 0x%x exceeded interrupt latency threshold, 0x%llx absolute time delta, prior signals: 0x%x, current signals: 0x%x", interrupt_num, int_latency, cdp->cpu_prior_signals, cdp->cpu_signals);
+ }
+
+ if (__improbable(int_latency > cdp->cpu_max_observed_int_latency)) {
+ cdp->cpu_max_observed_int_latency = int_latency;
+ cdp->cpu_max_observed_int_latency_vector = interrupt_num;
+ }
+ }
+
+ /*
+ * Having serviced the interrupt first, look at the interrupted stack depth.
+ */
+ if (!user_mode) {
+ uint64_t depth = cdp->cpu_kernel_stack
+ + sizeof(struct thread_kernel_state)
+ + sizeof(struct i386_exception_link *)
+ - rsp;
+ if (__improbable(depth > kernel_stack_depth_max)) {
+ kernel_stack_depth_max = (vm_offset_t)depth;
+ KERNEL_DEBUG_CONSTANT(
+ MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_DEPTH),
+ (long) depth, (long) VM_KERNEL_UNSLIDE(rip), 0, 0, 0);
+ }
+ }
+
+ if (cnum == master_cpu)
+ ml_entropy_collect();
projects / apple / xnu.git / blobdiff