]>
Commit | Line | Data |
---|---|---|
1c79356b | 1 | /* |
d26ffc64 | 2 | * Copyright (c) 2000-2018 Apple Inc. All rights reserved. |
1c79356b | 3 | * |
2d21ac55 | 4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
0a7de745 | 5 | * |
2d21ac55 A |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License | |
8 | * Version 2.0 (the 'License'). You may not use this file except in | |
9 | * compliance with the License. The rights granted to you under the License | |
10 | * may not be used to create, or enable the creation or redistribution of, | |
11 | * unlawful or unlicensed copies of an Apple operating system, or to | |
12 | * circumvent, violate, or enable the circumvention or violation of, any | |
13 | * terms of an Apple operating system software license agreement. | |
0a7de745 | 14 | * |
2d21ac55 A |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. | |
0a7de745 | 17 | * |
2d21ac55 A |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER | |
8f6c56a5 A |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, | |
2d21ac55 A |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and | |
24 | * limitations under the License. | |
0a7de745 | 25 | * |
2d21ac55 | 26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
1c79356b A |
27 | */ |
28 | /* | |
0a7de745 A |
29 | * @OSF_COPYRIGHT@ |
30 | */ | |
1c79356b | 31 | /* |
0a7de745 A |
32 | * Mach Operating System |
33 | * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University | |
34 | * All Rights Reserved. | |
35 | * | |
36 | * Permission to use, copy, modify and distribute this software and its | |
37 | * documentation is hereby granted, provided that both the copyright | |
38 | * notice and this permission notice appear in all copies of the | |
39 | * software, derivative works or modified versions, and any portions | |
40 | * thereof, and that both notices appear in supporting documentation. | |
41 | * | |
42 | * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" | |
43 | * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR | |
44 | * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. | |
45 | * | |
46 | * Carnegie Mellon requests users of this software to return to | |
47 | * | |
48 | * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU | |
49 | * School of Computer Science | |
50 | * Carnegie Mellon University | |
51 | * Pittsburgh PA 15213-3890 | |
52 | * | |
53 | * any improvements or extensions that they make and grant Carnegie Mellon | |
54 | * the rights to redistribute these changes. | |
55 | */ | |
56 | /* | |
57 | */ | |
2d21ac55 | 58 | |
1c79356b | 59 | /* |
0a7de745 | 60 | * Hardware trap/fault handler. |
1c79356b A |
61 | */ |
62 | ||
1c79356b A |
63 | #include <mach_kdp.h> |
64 | #include <mach_ldebug.h> | |
65 | ||
66 | #include <types.h> | |
67 | #include <i386/eflags.h> | |
68 | #include <i386/trap.h> | |
69 | #include <i386/pmap.h> | |
70 | #include <i386/fpu.h> | |
0c530ab8 | 71 | #include <i386/misc_protos.h> /* panic_io_port_read() */ |
b0d623f7 | 72 | #include <i386/lapic.h> |
1c79356b A |
73 | |
74 | #include <mach/exception.h> | |
75 | #include <mach/kern_return.h> | |
76 | #include <mach/vm_param.h> | |
77 | #include <mach/i386/thread_status.h> | |
78 | ||
79 | #include <vm/vm_kern.h> | |
80 | #include <vm/vm_fault.h> | |
81 | ||
1c79356b | 82 | #include <kern/kern_types.h> |
91447636 | 83 | #include <kern/processor.h> |
1c79356b A |
84 | #include <kern/thread.h> |
85 | #include <kern/task.h> | |
86 | #include <kern/sched.h> | |
87 | #include <kern/sched_prim.h> | |
88 | #include <kern/exception.h> | |
89 | #include <kern/spl.h> | |
90 | #include <kern/misc_protos.h> | |
b0d623f7 | 91 | #include <kern/debug.h> |
39236c6e A |
92 | #if CONFIG_TELEMETRY |
93 | #include <kern/telemetry.h> | |
94 | #endif | |
0c530ab8 | 95 | #include <sys/kdebug.h> |
d9a64523 | 96 | #include <kperf/kperf.h> |
fe8ab488 | 97 | #include <prng/random.h> |
0c530ab8 | 98 | |
1c79356b A |
99 | #include <string.h> |
100 | ||
0c530ab8 A |
101 | #include <i386/postcode.h> |
102 | #include <i386/mp_desc.h> | |
103 | #include <i386/proc_reg.h> | |
b0d623f7 | 104 | #if CONFIG_MCA |
0c530ab8 | 105 | #include <i386/machine_check.h> |
b0d623f7 | 106 | #endif |
0c530ab8 | 107 | #include <mach/i386/syscall_sw.h> |
1c79356b | 108 | |
b0d623f7 | 109 | #include <libkern/OSDebug.h> |
bd504ef0 | 110 | #include <i386/cpu_threads.h> |
6d2010ae | 111 | #include <machine/pal_routines.h> |
593a1d5f | 112 | |
6d2010ae A |
113 | extern void throttle_lowpri_io(int); |
114 | extern void kprint_state(x86_saved_state64_t *saved_state); | |
b0d623f7 | 115 | |
1c79356b A |
116 | /* |
117 | * Forward declarations | |
118 | */ | |
cb323159 | 119 | static void panic_trap(x86_saved_state64_t *saved_state, uint32_t pl, kern_return_t fault_result) __dead2; |
b0d623f7 | 120 | static void set_recovery_ip(x86_saved_state64_t *saved_state, vm_offset_t ip); |
6601e61a | 121 | |
2d21ac55 A |
122 | #if CONFIG_DTRACE |
123 | /* See <rdar://problem/4613924> */ | |
124 | perfCallback tempDTraceTrapHook = NULL; /* Pointer to DTrace fbt trap hook routine */ | |
125 | ||
126 | extern boolean_t dtrace_tally_fault(user_addr_t); | |
127 | #endif | |
128 | ||
13f56ec4 | 129 | extern boolean_t pmap_smep_enabled; |
fe8ab488 | 130 | extern boolean_t pmap_smap_enabled; |
7ddcb079 | 131 | |
39037602 | 132 | __attribute__((noreturn)) |
1c79356b A |
133 | void |
134 | thread_syscall_return( | |
0a7de745 | 135 | kern_return_t ret) |
1c79356b | 136 | { |
0a7de745 A |
137 | thread_t thr_act = current_thread(); |
138 | boolean_t is_mach; | |
139 | int code; | |
b0d623f7 | 140 | |
6d2010ae | 141 | pal_register_cache_state(thr_act, DIRTY); |
0c530ab8 | 142 | |
0a7de745 A |
143 | if (thread_is_64bit_addr(thr_act)) { |
144 | x86_saved_state64_t *regs; | |
145 | ||
0c530ab8 A |
146 | regs = USER_REGS64(thr_act); |
147 | ||
b0d623f7 A |
148 | code = (int) (regs->rax & SYSCALL_NUMBER_MASK); |
149 | is_mach = (regs->rax & SYSCALL_CLASS_MASK) | |
0a7de745 | 150 | == (SYSCALL_CLASS_MACH << SYSCALL_CLASS_SHIFT); |
b0d623f7 | 151 | if (kdebug_enable && is_mach) { |
0a7de745 A |
152 | /* Mach trap */ |
153 | KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, | |
154 | MACHDBG_CODE(DBG_MACH_EXCP_SC, code) | DBG_FUNC_END, | |
155 | ret, 0, 0, 0, 0); | |
0c530ab8 A |
156 | } |
157 | regs->rax = ret; | |
b0d623f7 | 158 | #if DEBUG |
0a7de745 | 159 | if (is_mach) { |
b0d623f7 A |
160 | DEBUG_KPRINT_SYSCALL_MACH( |
161 | "thread_syscall_return: 64-bit mach ret=%u\n", | |
162 | ret); | |
0a7de745 | 163 | } else { |
b0d623f7 A |
164 | DEBUG_KPRINT_SYSCALL_UNIX( |
165 | "thread_syscall_return: 64-bit unix ret=%u\n", | |
166 | ret); | |
0a7de745 | 167 | } |
b0d623f7 | 168 | #endif |
0c530ab8 | 169 | } else { |
0a7de745 A |
170 | x86_saved_state32_t *regs; |
171 | ||
0c530ab8 A |
172 | regs = USER_REGS32(thr_act); |
173 | ||
b0d623f7 A |
174 | code = ((int) regs->eax); |
175 | is_mach = (code < 0); | |
176 | if (kdebug_enable && is_mach) { | |
0a7de745 A |
177 | /* Mach trap */ |
178 | KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, | |
179 | MACHDBG_CODE(DBG_MACH_EXCP_SC, -code) | DBG_FUNC_END, | |
180 | ret, 0, 0, 0, 0); | |
0c530ab8 A |
181 | } |
182 | regs->eax = ret; | |
b0d623f7 | 183 | #if DEBUG |
0a7de745 | 184 | if (is_mach) { |
b0d623f7 A |
185 | DEBUG_KPRINT_SYSCALL_MACH( |
186 | "thread_syscall_return: 32-bit mach ret=%u\n", | |
187 | ret); | |
0a7de745 | 188 | } else { |
b0d623f7 A |
189 | DEBUG_KPRINT_SYSCALL_UNIX( |
190 | "thread_syscall_return: 32-bit unix ret=%u\n", | |
191 | ret); | |
0a7de745 | 192 | } |
b0d623f7 | 193 | #endif |
0c530ab8 | 194 | } |
5ba3f43e A |
195 | |
196 | #if DEBUG || DEVELOPMENT | |
197 | kern_allocation_name_t | |
198 | prior __assert_only = thread_get_kernel_state(thr_act)->allocation_name; | |
199 | assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior)); | |
200 | #endif /* DEBUG || DEVELOPMENT */ | |
201 | ||
39236c6e | 202 | throttle_lowpri_io(1); |
593a1d5f A |
203 | |
204 | thread_exception_return(); | |
0a7de745 | 205 | /*NOTREACHED*/ |
1c79356b A |
206 | } |
207 | ||
1c79356b A |
208 | /* |
209 | * Fault recovery in copyin/copyout routines. | |
210 | */ | |
211 | struct recovery { | |
0a7de745 A |
212 | uintptr_t fault_addr; |
213 | uintptr_t recover_addr; | |
1c79356b A |
214 | }; |
215 | ||
0a7de745 A |
216 | extern struct recovery recover_table[]; |
217 | extern struct recovery recover_table_end[]; | |
1c79356b | 218 | |
0a7de745 A |
219 | const char * trap_type[] = {TRAP_NAMES}; |
220 | unsigned TRAP_TYPES = sizeof(trap_type) / sizeof(trap_type[0]); | |
91447636 | 221 | |
0a7de745 | 222 | extern void PE_incoming_interrupt(int interrupt); |
6d2010ae | 223 | |
b0d623f7 | 224 | #if defined(__x86_64__) && DEBUG |
6d2010ae | 225 | void |
0a7de745 | 226 | kprint_state(x86_saved_state64_t *saved_state) |
b0d623f7 A |
227 | { |
228 | kprintf("current_cpu_datap() 0x%lx\n", (uintptr_t)current_cpu_datap()); | |
229 | kprintf("Current GS base MSR 0x%llx\n", rdmsr64(MSR_IA32_GS_BASE)); | |
230 | kprintf("Kernel GS base MSR 0x%llx\n", rdmsr64(MSR_IA32_KERNEL_GS_BASE)); | |
231 | kprintf("state at 0x%lx:\n", (uintptr_t) saved_state); | |
232 | ||
0a7de745 A |
233 | kprintf(" rdi 0x%llx\n", saved_state->rdi); |
234 | kprintf(" rsi 0x%llx\n", saved_state->rsi); | |
b0d623f7 A |
235 | kprintf(" rdx 0x%llx\n", saved_state->rdx); |
236 | kprintf(" r10 0x%llx\n", saved_state->r10); | |
237 | kprintf(" r8 0x%llx\n", saved_state->r8); | |
0a7de745 | 238 | kprintf(" r9 0x%llx\n", saved_state->r9); |
b0d623f7 A |
239 | |
240 | kprintf(" cr2 0x%llx\n", saved_state->cr2); | |
241 | kprintf("real cr2 0x%lx\n", get_cr2()); | |
242 | kprintf(" r15 0x%llx\n", saved_state->r15); | |
243 | kprintf(" r14 0x%llx\n", saved_state->r14); | |
244 | kprintf(" r13 0x%llx\n", saved_state->r13); | |
245 | kprintf(" r12 0x%llx\n", saved_state->r12); | |
246 | kprintf(" r11 0x%llx\n", saved_state->r11); | |
247 | kprintf(" rbp 0x%llx\n", saved_state->rbp); | |
248 | kprintf(" rbx 0x%llx\n", saved_state->rbx); | |
249 | kprintf(" rcx 0x%llx\n", saved_state->rcx); | |
250 | kprintf(" rax 0x%llx\n", saved_state->rax); | |
251 | ||
252 | kprintf(" gs 0x%x\n", saved_state->gs); | |
253 | kprintf(" fs 0x%x\n", saved_state->fs); | |
254 | ||
255 | kprintf(" isf.trapno 0x%x\n", saved_state->isf.trapno); | |
256 | kprintf(" isf._pad 0x%x\n", saved_state->isf._pad); | |
257 | kprintf(" isf.trapfn 0x%llx\n", saved_state->isf.trapfn); | |
258 | kprintf(" isf.err 0x%llx\n", saved_state->isf.err); | |
259 | kprintf(" isf.rip 0x%llx\n", saved_state->isf.rip); | |
260 | kprintf(" isf.cs 0x%llx\n", saved_state->isf.cs); | |
261 | kprintf(" isf.rflags 0x%llx\n", saved_state->isf.rflags); | |
262 | kprintf(" isf.rsp 0x%llx\n", saved_state->isf.rsp); | |
263 | kprintf(" isf.ss 0x%llx\n", saved_state->isf.ss); | |
264 | } | |
b0d623f7 A |
265 | #endif |
266 | ||
060df5ea | 267 | |
060df5ea A |
268 | /* |
269 | * Non-zero indicates latency assert is enabled and capped at valued | |
270 | * absolute time units. | |
271 | */ | |
0a7de745 | 272 | |
060df5ea A |
273 | uint64_t interrupt_latency_cap = 0; |
274 | boolean_t ilat_assert = FALSE; | |
275 | ||
276 | void | |
0a7de745 A |
277 | interrupt_latency_tracker_setup(void) |
278 | { | |
060df5ea A |
279 | uint32_t ilat_cap_us; |
280 | if (PE_parse_boot_argn("interrupt_latency_cap_us", &ilat_cap_us, sizeof(ilat_cap_us))) { | |
281 | interrupt_latency_cap = ilat_cap_us * NSEC_PER_USEC; | |
282 | nanoseconds_to_absolutetime(interrupt_latency_cap, &interrupt_latency_cap); | |
283 | } else { | |
284 | interrupt_latency_cap = LockTimeOut; | |
285 | } | |
286 | PE_parse_boot_argn("-interrupt_latency_assert_enable", &ilat_assert, sizeof(ilat_assert)); | |
287 | } | |
288 | ||
0a7de745 A |
289 | void |
290 | interrupt_reset_latency_stats(void) | |
291 | { | |
060df5ea A |
292 | uint32_t i; |
293 | for (i = 0; i < real_ncpus; i++) { | |
294 | cpu_data_ptr[i]->cpu_max_observed_int_latency = | |
295 | cpu_data_ptr[i]->cpu_max_observed_int_latency_vector = 0; | |
296 | } | |
297 | } | |
298 | ||
0a7de745 A |
299 | void |
300 | interrupt_populate_latency_stats(char *buf, unsigned bufsize) | |
301 | { | |
060df5ea A |
302 | uint32_t i, tcpu = ~0; |
303 | uint64_t cur_max = 0; | |
304 | ||
305 | for (i = 0; i < real_ncpus; i++) { | |
306 | if (cur_max < cpu_data_ptr[i]->cpu_max_observed_int_latency) { | |
307 | cur_max = cpu_data_ptr[i]->cpu_max_observed_int_latency; | |
308 | tcpu = i; | |
309 | } | |
310 | } | |
311 | ||
0a7de745 | 312 | if (tcpu < real_ncpus) { |
060df5ea | 313 | snprintf(buf, bufsize, "0x%x 0x%x 0x%llx", tcpu, cpu_data_ptr[tcpu]->cpu_max_observed_int_latency_vector, cpu_data_ptr[tcpu]->cpu_max_observed_int_latency); |
0a7de745 | 314 | } |
060df5ea | 315 | } |
b0d623f7 | 316 | |
39236c6e A |
317 | uint32_t interrupt_timer_coalescing_enabled = 1; |
318 | uint64_t interrupt_coalesced_timers; | |
319 | ||
b0d623f7 A |
320 | /* |
321 | * Handle interrupts: | |
322 | * - local APIC interrupts (IPIs, timers, etc) are handled by the kernel, | |
323 | * - device interrupts go to the platform expert. | |
324 | */ | |
325 | void | |
326 | interrupt(x86_saved_state_t *state) | |
327 | { | |
0a7de745 A |
328 | uint64_t rip; |
329 | uint64_t rsp; | |
330 | int interrupt_num; | |
331 | boolean_t user_mode = FALSE; | |
332 | int ipl; | |
333 | int cnum = cpu_number(); | |
334 | cpu_data_t *cdp = cpu_data_ptr[cnum]; | |
335 | int itype = DBG_INTR_TYPE_UNKNOWN; | |
336 | int handled; | |
337 | ||
338 | x86_saved_state64_t *state64 = saved_state64(state); | |
5ba3f43e A |
339 | rip = state64->isf.rip; |
340 | rsp = state64->isf.rsp; | |
341 | interrupt_num = state64->isf.trapno; | |
0a7de745 | 342 | if (state64->isf.cs & 0x03) { |
5ba3f43e | 343 | user_mode = TRUE; |
0a7de745 | 344 | } |
b0d623f7 | 345 | |
94ff46dc A |
346 | #if DEVELOPMENT || DEBUG |
347 | uint64_t frameptr = is_saved_state64(state) ? state64->rbp : saved_state32(state)->ebp; | |
348 | uint32_t traptrace_index = traptrace_start(interrupt_num, rip, mach_absolute_time(), frameptr); | |
349 | #endif | |
350 | ||
0a7de745 | 351 | if (cpu_data_ptr[cnum]->lcpu.package->num_idle == topoParms.nLThreadsPerPackage) { |
bd504ef0 | 352 | cpu_data_ptr[cnum]->cpu_hwIntpexits[interrupt_num]++; |
0a7de745 | 353 | } |
bd504ef0 | 354 | |
0a7de745 | 355 | if (interrupt_num == (LAPIC_DEFAULT_INTERRUPT_BASE + LAPIC_INTERPROCESSOR_INTERRUPT)) { |
5ba3f43e | 356 | itype = DBG_INTR_TYPE_IPI; |
0a7de745 | 357 | } else if (interrupt_num == (LAPIC_DEFAULT_INTERRUPT_BASE + LAPIC_TIMER_INTERRUPT)) { |
5ba3f43e | 358 | itype = DBG_INTR_TYPE_TIMER; |
0a7de745 | 359 | } else { |
5ba3f43e | 360 | itype = DBG_INTR_TYPE_OTHER; |
0a7de745 | 361 | } |
316670eb | 362 | |
0a7de745 A |
363 | KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, |
364 | MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_START, | |
365 | interrupt_num, | |
366 | (user_mode ? rip : VM_KERNEL_UNSLIDE(rip)), | |
367 | user_mode, itype, 0); | |
6d2010ae A |
368 | |
369 | SCHED_STATS_INTERRUPT(current_processor()); | |
370 | ||
39236c6e | 371 | #if CONFIG_TELEMETRY |
3e170ce0 | 372 | if (telemetry_needs_record) { |
d9a64523 | 373 | telemetry_mark_curthread(user_mode, FALSE); |
39236c6e A |
374 | } |
375 | #endif | |
376 | ||
6d2010ae | 377 | ipl = get_preemption_level(); |
0a7de745 | 378 | |
b0d623f7 A |
379 | /* |
380 | * Handle local APIC interrupts | |
381 | * else call platform expert for devices. | |
6d2010ae | 382 | */ |
0a7de745 A |
383 | handled = lapic_interrupt(interrupt_num, state); |
384 | ||
385 | if (!handled) { | |
386 | if (interrupt_num == (LAPIC_DEFAULT_INTERRUPT_BASE + LAPIC_CMCI_INTERRUPT)) { | |
387 | /* | |
388 | * CMCI can be signalled on any logical processor, and the kexts | |
389 | * that implement handling CMCI use IOKit to register handlers for | |
390 | * the CMCI vector, so if we see a CMCI, do not encode a CPU | |
391 | * number in bits 8:31 (since the vector is the same regardless of | |
392 | * the handling CPU). | |
393 | */ | |
394 | PE_incoming_interrupt(interrupt_num); | |
395 | } else if (cnum <= lapic_max_interrupt_cpunum) { | |
396 | PE_incoming_interrupt((cnum << 8) | interrupt_num); | |
397 | } | |
fe8ab488 | 398 | } |
6d2010ae A |
399 | |
400 | if (__improbable(get_preemption_level() != ipl)) { | |
401 | panic("Preemption level altered by interrupt vector 0x%x: initial 0x%x, final: 0x%x\n", interrupt_num, ipl, get_preemption_level()); | |
060df5ea | 402 | } |
b0d623f7 | 403 | |
316670eb | 404 | |
0a7de745 A |
405 | if (__improbable(cdp->cpu_nested_istack)) { |
406 | cdp->cpu_nested_istack_events++; | |
407 | } else { | |
39236c6e A |
408 | uint64_t ctime = mach_absolute_time(); |
409 | uint64_t int_latency = ctime - cdp->cpu_int_event_time; | |
410 | uint64_t esdeadline, ehdeadline; | |
411 | /* Attempt to process deferred timers in the context of | |
412 | * this interrupt, unless interrupt time has already exceeded | |
413 | * TCOAL_ILAT_THRESHOLD. | |
414 | */ | |
415 | #define TCOAL_ILAT_THRESHOLD (30000ULL) | |
416 | ||
417 | if ((int_latency < TCOAL_ILAT_THRESHOLD) && | |
418 | interrupt_timer_coalescing_enabled) { | |
419 | esdeadline = cdp->rtclock_timer.queue.earliest_soft_deadline; | |
420 | ehdeadline = cdp->rtclock_timer.deadline; | |
421 | if ((ctime >= esdeadline) && (ctime < ehdeadline)) { | |
422 | interrupt_coalesced_timers++; | |
423 | TCOAL_DEBUG(0x88880000 | DBG_FUNC_START, ctime, esdeadline, ehdeadline, interrupt_coalesced_timers, 0); | |
424 | rtclock_intr(state); | |
425 | TCOAL_DEBUG(0x88880000 | DBG_FUNC_END, ctime, esdeadline, interrupt_coalesced_timers, 0, 0); | |
426 | } else { | |
427 | TCOAL_DEBUG(0x77770000, ctime, cdp->rtclock_timer.queue.earliest_soft_deadline, cdp->rtclock_timer.deadline, interrupt_coalesced_timers, 0); | |
428 | } | |
060df5ea | 429 | } |
39236c6e A |
430 | |
431 | if (__improbable(ilat_assert && (int_latency > interrupt_latency_cap) && !machine_timeout_suspended())) { | |
432 | panic("Interrupt vector 0x%x exceeded interrupt latency threshold, 0x%llx absolute time delta, prior signals: 0x%x, current signals: 0x%x", interrupt_num, int_latency, cdp->cpu_prior_signals, cdp->cpu_signals); | |
433 | } | |
434 | ||
435 | if (__improbable(int_latency > cdp->cpu_max_observed_int_latency)) { | |
436 | cdp->cpu_max_observed_int_latency = int_latency; | |
437 | cdp->cpu_max_observed_int_latency_vector = interrupt_num; | |
060df5ea A |
438 | } |
439 | } | |
440 | ||
b0d623f7 A |
441 | /* |
442 | * Having serviced the interrupt first, look at the interrupted stack depth. | |
443 | */ | |
444 | if (!user_mode) { | |
39236c6e | 445 | uint64_t depth = cdp->cpu_kernel_stack |
0a7de745 A |
446 | + sizeof(struct thread_kernel_state) |
447 | + sizeof(struct i386_exception_link *) | |
448 | - rsp; | |
39236c6e | 449 | if (__improbable(depth > kernel_stack_depth_max)) { |
b0d623f7 A |
450 | kernel_stack_depth_max = (vm_offset_t)depth; |
451 | KERNEL_DEBUG_CONSTANT( | |
452 | MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_DEPTH), | |
316670eb | 453 | (long) depth, (long) VM_KERNEL_UNSLIDE(rip), 0, 0, 0); |
b0d623f7 A |
454 | } |
455 | } | |
d9a64523 | 456 | |
0a7de745 | 457 | if (cnum == master_cpu) { |
fe8ab488 | 458 | ml_entropy_collect(); |
0a7de745 | 459 | } |
fe8ab488 | 460 | |
d9a64523 A |
461 | #if KPERF |
462 | kperf_interrupt(); | |
463 | #endif /* KPERF */ | |
464 | ||
465 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_END, | |
0a7de745 | 466 | interrupt_num); |
39236c6e | 467 | |
3e170ce0 | 468 | assert(ml_get_interrupts_enabled() == FALSE); |
94ff46dc A |
469 | |
470 | #if DEVELOPMENT || DEBUG | |
471 | if (traptrace_index != TRAPTRACE_INVALID_INDEX) { | |
472 | traptrace_end(traptrace_index, mach_absolute_time()); | |
473 | } | |
474 | #endif | |
b0d623f7 | 475 | } |
2d21ac55 | 476 | |
0c530ab8 A |
477 | static inline void |
478 | reset_dr7(void) | |
479 | { | |
b0d623f7 | 480 | long dr7 = 0x400; /* magic dr7 reset value; 32 bit on i386, 64 bit on x86_64 */ |
0a7de745 | 481 | __asm__ volatile ("mov %0,%%dr7" : : "r" (dr7)); |
0c530ab8 A |
482 | } |
483 | #if MACH_KDP | |
484 | unsigned kdp_has_active_watchpoints = 0; | |
b0d623f7 A |
485 | #define NO_WATCHPOINTS (!kdp_has_active_watchpoints) |
486 | #else | |
487 | #define NO_WATCHPOINTS 1 | |
0c530ab8 | 488 | #endif |
1c79356b A |
489 | /* |
490 | * Trap from kernel mode. Only page-fault errors are recoverable, | |
491 | * and then only in special circumstances. All other errors are | |
492 | * fatal. Return value indicates if trap was handled. | |
493 | */ | |
b0d623f7 | 494 | |
0c530ab8 | 495 | void |
1c79356b | 496 | kernel_trap( |
0a7de745 | 497 | x86_saved_state_t *state, |
6d2010ae | 498 | uintptr_t *lo_spp) |
1c79356b | 499 | { |
0a7de745 A |
500 | x86_saved_state64_t *saved_state; |
501 | int code; | |
502 | user_addr_t vaddr; | |
503 | int type; | |
504 | vm_map_t map = 0; /* protected by T_PAGE_FAULT */ | |
505 | kern_return_t result = KERN_FAILURE; | |
506 | kern_return_t fault_result = KERN_SUCCESS; | |
507 | thread_t thread; | |
0c530ab8 | 508 | boolean_t intr; |
0a7de745 A |
509 | vm_prot_t prot; |
510 | struct recovery *rp; | |
511 | vm_offset_t kern_ip; | |
b0d623f7 | 512 | #if NCOPY_WINDOWS > 0 |
0a7de745 | 513 | int fault_in_copy_window = -1; |
b0d623f7 | 514 | #endif |
0a7de745 A |
515 | int is_user; |
516 | int trap_pl = get_preemption_level(); | |
3e170ce0 | 517 | |
1c79356b | 518 | thread = current_thread(); |
1c79356b | 519 | |
0a7de745 | 520 | if (__improbable(is_saved_state32(state))) { |
b0d623f7 | 521 | panic("kernel_trap(%p) with 32-bit state", state); |
0a7de745 | 522 | } |
b0d623f7 | 523 | saved_state = saved_state64(state); |
6d2010ae A |
524 | |
525 | /* Record cpu where state was captured */ | |
526 | saved_state->isf.cpu = cpu_number(); | |
527 | ||
b0d623f7 A |
528 | vaddr = (user_addr_t)saved_state->cr2; |
529 | type = saved_state->isf.trapno; | |
530 | code = (int)(saved_state->isf.err & 0xffff); | |
0a7de745 | 531 | intr = (saved_state->isf.rflags & EFL_IF) != 0; /* state of ints at trap */ |
b0d623f7 | 532 | kern_ip = (vm_offset_t)saved_state->isf.rip; |
0c530ab8 | 533 | |
39037602 A |
534 | is_user = (vaddr < VM_MAX_USER_PAGE_ADDRESS); |
535 | ||
94ff46dc A |
536 | #if DEVELOPMENT || DEBUG |
537 | uint32_t traptrace_index = traptrace_start(type, kern_ip, mach_absolute_time(), saved_state->rbp); | |
538 | #endif | |
539 | ||
2d21ac55 | 540 | #if CONFIG_DTRACE |
fe8ab488 A |
541 | /* |
542 | * Is there a DTrace hook? | |
0a7de745 | 543 | */ |
6d2010ae A |
544 | if (__improbable(tempDTraceTrapHook != NULL)) { |
545 | if (tempDTraceTrapHook(type, state, lo_spp, 0) == KERN_SUCCESS) { | |
2d21ac55 A |
546 | /* |
547 | * If it succeeds, we are done... | |
548 | */ | |
94ff46dc | 549 | goto common_return; |
2d21ac55 A |
550 | } |
551 | } | |
552 | #endif /* CONFIG_DTRACE */ | |
553 | ||
0c530ab8 A |
554 | /* |
555 | * we come here with interrupts off as we don't want to recurse | |
556 | * on preemption below. but we do want to re-enable interrupts | |
557 | * as soon we possibly can to hold latency down | |
558 | */ | |
6d2010ae | 559 | if (__improbable(T_PREEMPT == type)) { |
5ba3f43e | 560 | ast_taken_kernel(); |
0c530ab8 | 561 | |
0a7de745 A |
562 | KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, |
563 | (MACHDBG_CODE(DBG_MACH_EXCP_KTRAP_x86, type)) | DBG_FUNC_NONE, | |
564 | 0, 0, 0, VM_KERNEL_UNSLIDE(kern_ip), 0); | |
94ff46dc A |
565 | |
566 | goto common_return; | |
0c530ab8 | 567 | } |
39037602 | 568 | |
0a7de745 | 569 | user_addr_t kd_vaddr = is_user ? vaddr : VM_KERNEL_UNSLIDE(vaddr); |
39037602 | 570 | KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, |
0a7de745 A |
571 | (MACHDBG_CODE(DBG_MACH_EXCP_KTRAP_x86, type)) | DBG_FUNC_NONE, |
572 | (unsigned)(kd_vaddr >> 32), (unsigned)kd_vaddr, is_user, | |
573 | VM_KERNEL_UNSLIDE(kern_ip), 0); | |
39037602 A |
574 | |
575 | ||
0c530ab8 A |
576 | if (T_PAGE_FAULT == type) { |
577 | /* | |
578 | * assume we're faulting in the kernel map | |
579 | */ | |
580 | map = kernel_map; | |
581 | ||
6d2010ae | 582 | if (__probable(thread != THREAD_NULL && thread->map != kernel_map)) { |
b0d623f7 | 583 | #if NCOPY_WINDOWS > 0 |
0a7de745 A |
584 | vm_offset_t copy_window_base; |
585 | vm_offset_t kvaddr; | |
586 | int window_index; | |
0c530ab8 A |
587 | |
588 | kvaddr = (vm_offset_t)vaddr; | |
b0d623f7 | 589 | /* |
0c530ab8 A |
590 | * must determine if fault occurred in |
591 | * the copy window while pre-emption is | |
592 | * disabled for this processor so that | |
593 | * we only need to look at the window | |
594 | * associated with this processor | |
595 | */ | |
6d2010ae | 596 | copy_window_base = current_cpu_datap()->cpu_copywindow_base; |
0c530ab8 | 597 | |
0a7de745 | 598 | if (kvaddr >= copy_window_base && kvaddr < (copy_window_base + (NBPDE * NCOPY_WINDOWS))) { |
6d2010ae | 599 | window_index = (int)((kvaddr - copy_window_base) / NBPDE); |
0c530ab8 A |
600 | |
601 | if (thread->machine.copy_window[window_index].user_base != (user_addr_t)-1) { | |
0a7de745 A |
602 | kvaddr -= (copy_window_base + (NBPDE * window_index)); |
603 | vaddr = thread->machine.copy_window[window_index].user_base + kvaddr; | |
0c530ab8 A |
604 | |
605 | map = thread->map; | |
606 | fault_in_copy_window = window_index; | |
607 | } | |
0c530ab8 | 608 | } |
b0d623f7 | 609 | #else |
7ddcb079 | 610 | if (__probable(vaddr < VM_MAX_USER_PAGE_ADDRESS)) { |
b0d623f7 A |
611 | /* fault occurred in userspace */ |
612 | map = thread->map; | |
7ddcb079 | 613 | |
13f56ec4 A |
614 | /* Intercept a potential Supervisor Mode Execute |
615 | * Protection fault. These criteria identify | |
616 | * both NX faults and SMEP faults, but both | |
617 | * are fatal. We avoid checking PTEs (racy). | |
618 | * (The VM could just redrive a SMEP fault, hence | |
619 | * the intercept). | |
620 | */ | |
39037602 | 621 | if (__improbable((code == (T_PF_PROT | T_PF_EXECUTE)) && |
0a7de745 | 622 | (pmap_smep_enabled) && (saved_state->isf.rip == vaddr))) { |
13f56ec4 A |
623 | goto debugger_entry; |
624 | } | |
625 | ||
04b8595b A |
626 | /* |
627 | * Additionally check for SMAP faults... | |
628 | * which are characterized by page-present and | |
629 | * the AC bit unset (i.e. not from copyin/out path). | |
630 | */ | |
631 | if (__improbable(code & T_PF_PROT && | |
0a7de745 A |
632 | pmap_smap_enabled && |
633 | (saved_state->isf.rflags & EFL_AC) == 0)) { | |
04b8595b A |
634 | goto debugger_entry; |
635 | } | |
636 | ||
b0d623f7 A |
637 | /* |
638 | * If we're not sharing cr3 with the user | |
639 | * and we faulted in copyio, | |
640 | * then switch cr3 here and dismiss the fault. | |
641 | */ | |
642 | if (no_shared_cr3 && | |
0a7de745 | 643 | (thread->machine.specFlags & CopyIOActive) && |
6d2010ae A |
644 | map->pmap->pm_cr3 != get_cr3_base()) { |
645 | pmap_assert(current_cpu_datap()->cpu_pmap_pcid_enabled == FALSE); | |
646 | set_cr3_raw(map->pmap->pm_cr3); | |
b0d623f7 A |
647 | return; |
648 | } | |
39037602 A |
649 | if (__improbable(vaddr < PAGE_SIZE) && |
650 | ((thread->machine.specFlags & CopyIOActive) == 0)) { | |
651 | goto debugger_entry; | |
652 | } | |
b0d623f7 A |
653 | } |
654 | #endif | |
0c530ab8 A |
655 | } |
656 | } | |
0c530ab8 A |
657 | |
658 | (void) ml_set_interrupts_enabled(intr); | |
659 | ||
1c79356b | 660 | switch (type) { |
0a7de745 | 661 | case T_NO_FPU: |
1c79356b | 662 | fpnoextflt(); |
94ff46dc | 663 | goto common_return; |
1c79356b | 664 | |
0a7de745 | 665 | case T_FPU_FAULT: |
1c79356b | 666 | fpextovrflt(); |
94ff46dc | 667 | goto common_return; |
1c79356b | 668 | |
0a7de745 | 669 | case T_FLOATING_POINT_ERROR: |
1c79356b | 670 | fpexterrflt(); |
94ff46dc | 671 | goto common_return; |
1c79356b | 672 | |
0a7de745 | 673 | case T_SSE_FLOAT_ERROR: |
d26ffc64 | 674 | fpSSEexterrflt(); |
94ff46dc | 675 | goto common_return; |
d26ffc64 | 676 | |
0a7de745 | 677 | case T_INVALID_OPCODE: |
d26ffc64 A |
678 | fpUDflt(kern_ip); |
679 | goto debugger_entry; | |
680 | ||
0a7de745 A |
681 | case T_DEBUG: |
682 | if ((saved_state->isf.rflags & EFL_TF) == 0 && NO_WATCHPOINTS) { | |
683 | /* We've somehow encountered a debug | |
684 | * register match that does not belong | |
685 | * to the kernel debugger. | |
686 | * This isn't supposed to happen. | |
687 | */ | |
688 | reset_dr7(); | |
94ff46dc | 689 | goto common_return; |
0a7de745 A |
690 | } |
691 | goto debugger_entry; | |
692 | case T_INT3: | |
693 | goto debugger_entry; | |
694 | case T_PAGE_FAULT: | |
0c530ab8 | 695 | |
2d21ac55 | 696 | #if CONFIG_DTRACE |
cb323159 | 697 | if (thread != THREAD_NULL && thread->t_dtrace_inprobe) { /* Executing under dtrace_probe? */ |
2d21ac55 A |
698 | if (dtrace_tally_fault(vaddr)) { /* Should a fault under dtrace be ignored? */ |
699 | /* | |
700 | * DTrace has "anticipated" the possibility of this fault, and has | |
701 | * established the suitable recovery state. Drop down now into the | |
0a7de745 | 702 | * recovery handling code in "case T_GENERAL_PROTECTION:". |
2d21ac55 A |
703 | */ |
704 | goto FALL_THROUGH; | |
705 | } | |
706 | } | |
707 | #endif /* CONFIG_DTRACE */ | |
0a7de745 | 708 | |
7ddcb079 A |
709 | prot = VM_PROT_READ; |
710 | ||
0a7de745 A |
711 | if (code & T_PF_WRITE) { |
712 | prot |= VM_PROT_WRITE; | |
713 | } | |
714 | if (code & T_PF_EXECUTE) { | |
715 | prot |= VM_PROT_EXECUTE; | |
716 | } | |
7ddcb079 | 717 | |
d190cdc3 | 718 | fault_result = result = vm_fault(map, |
0a7de745 A |
719 | vaddr, |
720 | prot, | |
721 | FALSE, VM_KERN_MEMORY_NONE, | |
722 | THREAD_UNINT, NULL, 0); | |
0c530ab8 | 723 | |
1c79356b | 724 | if (result == KERN_SUCCESS) { |
b0d623f7 | 725 | #if NCOPY_WINDOWS > 0 |
0c530ab8 | 726 | if (fault_in_copy_window != -1) { |
6d2010ae A |
727 | ml_set_interrupts_enabled(FALSE); |
728 | copy_window_fault(thread, map, | |
0a7de745 | 729 | fault_in_copy_window); |
0c530ab8 | 730 | (void) ml_set_interrupts_enabled(intr); |
1c79356b | 731 | } |
b0d623f7 | 732 | #endif /* NCOPY_WINDOWS > 0 */ |
94ff46dc | 733 | goto common_return; |
1c79356b | 734 | } |
0c530ab8 A |
735 | /* |
736 | * fall through | |
737 | */ | |
2d21ac55 A |
738 | #if CONFIG_DTRACE |
739 | FALL_THROUGH: | |
740 | #endif /* CONFIG_DTRACE */ | |
1c79356b | 741 | |
0a7de745 | 742 | case T_GENERAL_PROTECTION: |
1c79356b A |
743 | /* |
744 | * If there is a failure recovery address | |
745 | * for this fault, go there. | |
746 | */ | |
0a7de745 A |
747 | for (rp = recover_table; rp < recover_table_end; rp++) { |
748 | if (kern_ip == rp->fault_addr) { | |
749 | set_recovery_ip(saved_state, rp->recover_addr); | |
94ff46dc | 750 | goto common_return; |
1c79356b | 751 | } |
1c79356b A |
752 | } |
753 | ||
754 | /* | |
0c530ab8 | 755 | * Check thread recovery address also. |
1c79356b | 756 | */ |
6d2010ae | 757 | if (thread != THREAD_NULL && thread->recover) { |
b0d623f7 | 758 | set_recovery_ip(saved_state, thread->recover); |
0c530ab8 | 759 | thread->recover = 0; |
94ff46dc | 760 | goto common_return; |
1c79356b | 761 | } |
0a7de745 A |
762 | /* |
763 | * Unanticipated page-fault errors in kernel | |
764 | * should not happen. | |
765 | * | |
766 | * fall through... | |
767 | */ | |
768 | default: | |
91447636 A |
769 | /* |
770 | * Exception 15 is reserved but some chips may generate it | |
771 | * spuriously. Seen at startup on AMD Athlon-64. | |
772 | */ | |
0a7de745 A |
773 | if (type == 15) { |
774 | kprintf("kernel_trap() ignoring spurious trap 15\n"); | |
94ff46dc | 775 | goto common_return; |
91447636 | 776 | } |
0c530ab8 A |
777 | debugger_entry: |
778 | /* Ensure that the i386_kernel_state at the base of the | |
779 | * current thread's stack (if any) is synchronized with the | |
780 | * context at the moment of the trap, to facilitate | |
781 | * access through the debugger. | |
1c79356b | 782 | */ |
b0d623f7 | 783 | sync_iss_to_iks(state); |
1c79356b | 784 | #if MACH_KDP |
0a7de745 | 785 | if (kdp_i386_trap(type, saved_state, result, (vm_offset_t)vaddr)) { |
94ff46dc | 786 | goto common_return; |
0a7de745 | 787 | } |
2d21ac55 | 788 | #endif |
4452a7af | 789 | } |
316670eb | 790 | pal_cli(); |
d190cdc3 | 791 | panic_trap(saved_state, trap_pl, fault_result); |
0c530ab8 A |
792 | /* |
793 | * NO RETURN | |
794 | */ | |
94ff46dc A |
795 | |
796 | common_return: | |
797 | #if DEVELOPMENT || DEBUG | |
798 | if (traptrace_index != TRAPTRACE_INVALID_INDEX) { | |
799 | traptrace_end(traptrace_index, mach_absolute_time()); | |
800 | } | |
801 | #endif | |
802 | return; | |
0c530ab8 A |
803 | } |
804 | ||
b0d623f7 A |
805 | static void |
806 | set_recovery_ip(x86_saved_state64_t *saved_state, vm_offset_t ip) | |
807 | { | |
0a7de745 | 808 | saved_state->isf.rip = ip; |
b0d623f7 | 809 | } |
0c530ab8 | 810 | |
b0d623f7 | 811 | static void |
d190cdc3 | 812 | panic_trap(x86_saved_state64_t *regs, uint32_t pl, kern_return_t fault_result) |
4452a7af | 813 | { |
0a7de745 A |
814 | const char *trapname = "Unknown"; |
815 | pal_cr_t cr0, cr2, cr3, cr4; | |
816 | boolean_t potential_smep_fault = FALSE, potential_kernel_NX_fault = FALSE; | |
817 | boolean_t potential_smap_fault = FALSE; | |
0c530ab8 | 818 | |
6d2010ae A |
819 | pal_get_control_registers( &cr0, &cr2, &cr3, &cr4 ); |
820 | assert(ml_get_interrupts_enabled() == FALSE); | |
821 | current_cpu_datap()->cpu_fatal_trap_state = regs; | |
2d21ac55 A |
822 | /* |
823 | * Issue an I/O port read if one has been requested - this is an | |
824 | * event logic analyzers can use as a trigger point. | |
825 | */ | |
0c530ab8 A |
826 | panic_io_port_read(); |
827 | ||
5c9f4661 A |
828 | kprintf("CPU %d panic trap number 0x%x, rip 0x%016llx\n", |
829 | cpu_number(), regs->isf.trapno, regs->isf.rip); | |
b0d623f7 | 830 | kprintf("cr0 0x%016llx cr2 0x%016llx cr3 0x%016llx cr4 0x%016llx\n", |
0a7de745 | 831 | cr0, cr2, cr3, cr4); |
4452a7af | 832 | |
0a7de745 A |
833 | if (regs->isf.trapno < TRAP_TYPES) { |
834 | trapname = trap_type[regs->isf.trapno]; | |
835 | } | |
7ddcb079 | 836 | |
316670eb A |
837 | if ((regs->isf.trapno == T_PAGE_FAULT) && (regs->isf.err == (T_PF_PROT | T_PF_EXECUTE)) && (regs->isf.rip == regs->cr2)) { |
838 | if (pmap_smep_enabled && (regs->isf.rip < VM_MAX_USER_PAGE_ADDRESS)) { | |
839 | potential_smep_fault = TRUE; | |
840 | } else if (regs->isf.rip >= VM_MIN_KERNEL_AND_KEXT_ADDRESS) { | |
841 | potential_kernel_NX_fault = TRUE; | |
842 | } | |
04b8595b | 843 | } else if (pmap_smap_enabled && |
0a7de745 A |
844 | regs->isf.trapno == T_PAGE_FAULT && |
845 | regs->isf.err & T_PF_PROT && | |
846 | regs->cr2 < VM_MAX_USER_PAGE_ADDRESS && | |
847 | regs->isf.rip >= VM_MIN_KERNEL_AND_KEXT_ADDRESS) { | |
04b8595b | 848 | potential_smap_fault = TRUE; |
13f56ec4 A |
849 | } |
850 | ||
b0d623f7 A |
851 | #undef panic |
852 | panic("Kernel trap at 0x%016llx, type %d=%s, registers:\n" | |
0a7de745 A |
853 | "CR0: 0x%016llx, CR2: 0x%016llx, CR3: 0x%016llx, CR4: 0x%016llx\n" |
854 | "RAX: 0x%016llx, RBX: 0x%016llx, RCX: 0x%016llx, RDX: 0x%016llx\n" | |
855 | "RSP: 0x%016llx, RBP: 0x%016llx, RSI: 0x%016llx, RDI: 0x%016llx\n" | |
856 | "R8: 0x%016llx, R9: 0x%016llx, R10: 0x%016llx, R11: 0x%016llx\n" | |
857 | "R12: 0x%016llx, R13: 0x%016llx, R14: 0x%016llx, R15: 0x%016llx\n" | |
858 | "RFL: 0x%016llx, RIP: 0x%016llx, CS: 0x%016llx, SS: 0x%016llx\n" | |
859 | "Fault CR2: 0x%016llx, Error code: 0x%016llx, Fault CPU: 0x%x%s%s%s%s, PL: %d, VF: %d\n", | |
860 | regs->isf.rip, regs->isf.trapno, trapname, | |
861 | cr0, cr2, cr3, cr4, | |
862 | regs->rax, regs->rbx, regs->rcx, regs->rdx, | |
863 | regs->isf.rsp, regs->rbp, regs->rsi, regs->rdi, | |
864 | regs->r8, regs->r9, regs->r10, regs->r11, | |
865 | regs->r12, regs->r13, regs->r14, regs->r15, | |
866 | regs->isf.rflags, regs->isf.rip, regs->isf.cs & 0xFFFF, | |
867 | regs->isf.ss & 0xFFFF, regs->cr2, regs->isf.err, regs->isf.cpu, | |
868 | virtualized ? " VMM" : "", | |
869 | potential_kernel_NX_fault ? " Kernel NX fault" : "", | |
870 | potential_smep_fault ? " SMEP/User NX fault" : "", | |
871 | potential_smap_fault ? " SMAP fault" : "", | |
872 | pl, | |
873 | fault_result); | |
4452a7af A |
874 | } |
875 | ||
2d21ac55 A |
876 | #if CONFIG_DTRACE |
877 | extern kern_return_t dtrace_user_probe(x86_saved_state_t *); | |
878 | #endif | |
879 | ||
d26ffc64 A |
880 | #if DEBUG |
881 | uint32_t fsigs[2]; | |
882 | uint32_t fsigns, fsigcs; | |
883 | #endif | |
884 | ||
1c79356b A |
885 | /* |
886 | * Trap from user mode. | |
887 | */ | |
888 | void | |
889 | user_trap( | |
0c530ab8 | 890 | x86_saved_state_t *saved_state) |
1c79356b | 891 | { |
0a7de745 A |
892 | int exc; |
893 | int err; | |
894 | mach_exception_code_t code; | |
2d21ac55 | 895 | mach_exception_subcode_t subcode; |
0a7de745 A |
896 | int type; |
897 | user_addr_t vaddr; | |
898 | vm_prot_t prot; | |
899 | thread_t thread = current_thread(); | |
900 | kern_return_t kret; | |
901 | user_addr_t rip; | |
902 | unsigned long dr6 = 0; /* 32 bit for i386, 64 bit for x86_64 */ | |
94ff46dc A |
903 | #if DEVELOPMENT || DEBUG |
904 | uint32_t traptrace_index; | |
905 | #endif | |
d9a64523 | 906 | assert((is_saved_state32(saved_state) && !thread_is_64bit_addr(thread)) || |
0a7de745 | 907 | (is_saved_state64(saved_state) && thread_is_64bit_addr(thread))); |
0c530ab8 A |
908 | |
909 | if (is_saved_state64(saved_state)) { | |
0a7de745 | 910 | x86_saved_state64_t *regs; |
0c530ab8 A |
911 | |
912 | regs = saved_state64(saved_state); | |
913 | ||
6d2010ae A |
914 | /* Record cpu where state was captured */ |
915 | regs->isf.cpu = cpu_number(); | |
916 | ||
0c530ab8 | 917 | type = regs->isf.trapno; |
b0d623f7 | 918 | err = (int)regs->isf.err & 0xffff; |
0c530ab8 A |
919 | vaddr = (user_addr_t)regs->cr2; |
920 | rip = (user_addr_t)regs->isf.rip; | |
94ff46dc A |
921 | #if DEVELOPMENT || DEBUG |
922 | traptrace_index = traptrace_start(type, rip, mach_absolute_time(), regs->rbp); | |
923 | #endif | |
0c530ab8 | 924 | } else { |
0a7de745 | 925 | x86_saved_state32_t *regs; |
0c530ab8 A |
926 | |
927 | regs = saved_state32(saved_state); | |
928 | ||
6d2010ae A |
929 | /* Record cpu where state was captured */ |
930 | regs->cpu = cpu_number(); | |
931 | ||
0c530ab8 A |
932 | type = regs->trapno; |
933 | err = regs->err & 0xffff; | |
934 | vaddr = (user_addr_t)regs->cr2; | |
935 | rip = (user_addr_t)regs->eip; | |
94ff46dc A |
936 | #if DEVELOPMENT || DEBUG |
937 | traptrace_index = traptrace_start(type, rip, mach_absolute_time(), regs->ebp); | |
938 | #endif | |
1c79356b A |
939 | } |
940 | ||
94ff46dc | 941 | |
6d2010ae A |
942 | if ((type == T_DEBUG) && thread->machine.ids) { |
943 | unsigned long clear = 0; | |
944 | /* Stash and clear this processor's DR6 value, in the event | |
945 | * this was a debug register match | |
946 | */ | |
0a7de745 | 947 | __asm__ volatile ("mov %%db6, %0" : "=r" (dr6)); |
6d2010ae A |
948 | __asm__ volatile ("mov %0, %%db6" : : "r" (clear)); |
949 | } | |
950 | ||
951 | pal_sti(); | |
952 | ||
0a7de745 A |
953 | KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, |
954 | (MACHDBG_CODE(DBG_MACH_EXCP_UTRAP_x86, type)) | DBG_FUNC_NONE, | |
955 | (unsigned)(vaddr >> 32), (unsigned)vaddr, | |
956 | (unsigned)(rip >> 32), (unsigned)rip, 0); | |
0c530ab8 | 957 | |
1c79356b A |
958 | code = 0; |
959 | subcode = 0; | |
91447636 | 960 | exc = 0; |
1c79356b | 961 | |
fe8ab488 | 962 | #if CONFIG_DTRACE |
2d21ac55 A |
963 | /* |
964 | * DTrace does not consume all user traps, only INT_3's for now. | |
965 | * Avoid needlessly calling tempDTraceTrapHook here, and let the | |
966 | * INT_3 case handle them. | |
967 | */ | |
fe8ab488 | 968 | #endif |
0a7de745 | 969 | |
b0d623f7 | 970 | DEBUG_KPRINT_SYSCALL_MASK(1, |
0a7de745 A |
971 | "user_trap: type=0x%x(%s) err=0x%x cr2=%p rip=%p\n", |
972 | type, trap_type[type], err, (void *)(long) vaddr, (void *)(long) rip); | |
1c79356b | 973 | |
0a7de745 A |
974 | switch (type) { |
975 | case T_DIVIDE_ERROR: | |
1c79356b A |
976 | exc = EXC_ARITHMETIC; |
977 | code = EXC_I386_DIV; | |
978 | break; | |
979 | ||
0a7de745 A |
980 | case T_DEBUG: |
981 | { | |
982 | pcb_t pcb; | |
983 | /* | |
984 | * Update the PCB with this processor's DR6 value | |
985 | * in the event this was a debug register match. | |
986 | */ | |
987 | pcb = THREAD_TO_PCB(thread); | |
988 | if (pcb->ids) { | |
0c530ab8 | 989 | /* |
0a7de745 A |
990 | * We can get and set the status register |
991 | * in 32-bit mode even on a 64-bit thread | |
992 | * because the high order bits are not | |
993 | * used on x86_64 | |
0c530ab8 | 994 | */ |
0a7de745 A |
995 | if (thread_is_64bit_addr(thread)) { |
996 | x86_debug_state64_t *ids = pcb->ids; | |
997 | ids->dr6 = dr6; | |
998 | } else { /* 32 bit thread */ | |
999 | x86_debug_state32_t *ids = pcb->ids; | |
1000 | ids->dr6 = (uint32_t) dr6; | |
0c530ab8 | 1001 | } |
0c530ab8 | 1002 | } |
0a7de745 A |
1003 | exc = EXC_BREAKPOINT; |
1004 | code = EXC_I386_SGL; | |
1005 | break; | |
1006 | } | |
1007 | case T_INT3: | |
2d21ac55 | 1008 | #if CONFIG_DTRACE |
0a7de745 | 1009 | if (dtrace_user_probe(saved_state) == KERN_SUCCESS) { |
2d21ac55 | 1010 | return; /* If it succeeds, we are done... */ |
0a7de745 | 1011 | } |
2d21ac55 | 1012 | #endif |
1c79356b A |
1013 | exc = EXC_BREAKPOINT; |
1014 | code = EXC_I386_BPT; | |
1015 | break; | |
1016 | ||
0a7de745 | 1017 | case T_OVERFLOW: |
1c79356b A |
1018 | exc = EXC_ARITHMETIC; |
1019 | code = EXC_I386_INTO; | |
1020 | break; | |
1021 | ||
0a7de745 | 1022 | case T_OUT_OF_BOUNDS: |
1c79356b A |
1023 | exc = EXC_SOFTWARE; |
1024 | code = EXC_I386_BOUND; | |
1025 | break; | |
1026 | ||
0a7de745 | 1027 | case T_INVALID_OPCODE: |
94ff46dc A |
1028 | if (fpUDflt(rip) == 1) { |
1029 | exc = EXC_BAD_INSTRUCTION; | |
1030 | code = EXC_I386_INVOP; | |
1031 | } | |
1c79356b A |
1032 | break; |
1033 | ||
0a7de745 | 1034 | case T_NO_FPU: |
1c79356b | 1035 | fpnoextflt(); |
94ff46dc | 1036 | break; |
1c79356b | 1037 | |
0a7de745 | 1038 | case T_FPU_FAULT: |
94ff46dc A |
1039 | fpextovrflt(); |
1040 | /* | |
1041 | * Raise exception. | |
1042 | */ | |
1043 | exc = EXC_BAD_ACCESS; | |
1044 | code = VM_PROT_READ | VM_PROT_EXECUTE; | |
1045 | subcode = 0; | |
1046 | break; | |
1c79356b | 1047 | |
0a7de745 | 1048 | case T_INVALID_TSS: /* invalid TSS == iret with NT flag set */ |
1c79356b A |
1049 | exc = EXC_BAD_INSTRUCTION; |
1050 | code = EXC_I386_INVTSSFLT; | |
0c530ab8 | 1051 | subcode = err; |
1c79356b A |
1052 | break; |
1053 | ||
0a7de745 | 1054 | case T_SEGMENT_NOT_PRESENT: |
1c79356b A |
1055 | exc = EXC_BAD_INSTRUCTION; |
1056 | code = EXC_I386_SEGNPFLT; | |
0c530ab8 | 1057 | subcode = err; |
1c79356b A |
1058 | break; |
1059 | ||
0a7de745 | 1060 | case T_STACK_FAULT: |
1c79356b A |
1061 | exc = EXC_BAD_INSTRUCTION; |
1062 | code = EXC_I386_STKFLT; | |
0c530ab8 | 1063 | subcode = err; |
1c79356b A |
1064 | break; |
1065 | ||
0a7de745 | 1066 | case T_GENERAL_PROTECTION: |
2d21ac55 A |
1067 | /* |
1068 | * There's a wide range of circumstances which generate this | |
1069 | * class of exception. From user-space, many involve bad | |
1070 | * addresses (such as a non-canonical 64-bit address). | |
1071 | * So we map this to EXC_BAD_ACCESS (and thereby SIGSEGV). | |
1072 | * The trouble is cr2 doesn't contain the faulting address; | |
1073 | * we'd need to decode the faulting instruction to really | |
1074 | * determine this. We'll leave that to debuggers. | |
1075 | * However, attempted execution of privileged instructions | |
1076 | * (e.g. cli) also generate GP faults and so we map these to | |
1077 | * to EXC_BAD_ACCESS (and thence SIGSEGV) also - rather than | |
1078 | * EXC_BAD_INSTRUCTION which is more accurate. We just can't | |
1079 | * win! | |
0a7de745 | 1080 | */ |
2d21ac55 | 1081 | exc = EXC_BAD_ACCESS; |
1c79356b | 1082 | code = EXC_I386_GPFLT; |
0c530ab8 | 1083 | subcode = err; |
1c79356b A |
1084 | break; |
1085 | ||
0a7de745 A |
1086 | case T_PAGE_FAULT: |
1087 | { | |
1088 | prot = VM_PROT_READ; | |
0c530ab8 | 1089 | |
0a7de745 A |
1090 | if (err & T_PF_WRITE) { |
1091 | prot |= VM_PROT_WRITE; | |
1092 | } | |
1093 | if (__improbable(err & T_PF_EXECUTE)) { | |
1094 | prot |= VM_PROT_EXECUTE; | |
1095 | } | |
d26ffc64 A |
1096 | #if DEVELOPMENT || DEBUG |
1097 | uint32_t fsig = 0; | |
1098 | fsig = thread_fpsimd_hash(thread); | |
1099 | #if DEBUG | |
1100 | fsigs[0] = fsig; | |
1101 | #endif | |
1102 | #endif | |
39236c6e | 1103 | kret = vm_fault(thread->map, |
0a7de745 A |
1104 | vaddr, |
1105 | prot, FALSE, VM_KERN_MEMORY_NONE, | |
1106 | THREAD_ABORTSAFE, NULL, 0); | |
d26ffc64 A |
1107 | #if DEVELOPMENT || DEBUG |
1108 | if (fsig) { | |
1109 | uint32_t fsig2 = thread_fpsimd_hash(thread); | |
1110 | #if DEBUG | |
1111 | fsigcs++; | |
1112 | fsigs[1] = fsig2; | |
1113 | #endif | |
1114 | if (fsig != fsig2) { | |
1115 | panic("FP/SIMD state hash mismatch across fault thread: %p 0x%x->0x%x", thread, fsig, fsig2); | |
1116 | } | |
1117 | } else { | |
1118 | #if DEBUG | |
1119 | fsigns++; | |
1120 | #endif | |
1121 | } | |
1122 | #endif | |
db609669 | 1123 | if (__probable((kret == KERN_SUCCESS) || (kret == KERN_ABORTED))) { |
94ff46dc A |
1124 | break; |
1125 | } else if (__improbable(kret == KERN_FAILURE)) { | |
1126 | /* | |
1127 | * For a user trap, vm_fault() should never return KERN_FAILURE. | |
1128 | * If it does, we're leaking preemption disables somewhere in the kernel. | |
1129 | */ | |
cb323159 A |
1130 | panic("vm_fault() KERN_FAILURE from user fault on thread %p", thread); |
1131 | } | |
1132 | ||
94ff46dc A |
1133 | /* PAL debug hook (empty on x86) */ |
1134 | pal_dbg_page_fault(thread, vaddr, kret); | |
1135 | exc = EXC_BAD_ACCESS; | |
1136 | code = kret; | |
1137 | subcode = vaddr; | |
1138 | } | |
0a7de745 | 1139 | break; |
1c79356b | 1140 | |
0a7de745 | 1141 | case T_SSE_FLOAT_ERROR: |
94ff46dc A |
1142 | fpSSEexterrflt(); |
1143 | exc = EXC_ARITHMETIC; | |
1144 | code = EXC_I386_SSEEXTERR; | |
1145 | subcode = ((struct x86_fx_thread_state *)thread->machine.ifps)->fx_MXCSR; | |
1146 | break; | |
0c530ab8 A |
1147 | |
1148 | ||
0a7de745 | 1149 | case T_FLOATING_POINT_ERROR: |
94ff46dc A |
1150 | fpexterrflt(); |
1151 | exc = EXC_ARITHMETIC; | |
1152 | code = EXC_I386_EXTERR; | |
1153 | subcode = ((struct x86_fx_thread_state *)thread->machine.ifps)->fx_status; | |
1154 | break; | |
1c79356b | 1155 | |
0a7de745 | 1156 | case T_DTRACE_RET: |
2d21ac55 | 1157 | #if CONFIG_DTRACE |
0a7de745 | 1158 | if (dtrace_user_probe(saved_state) == KERN_SUCCESS) { |
2d21ac55 | 1159 | return; /* If it succeeds, we are done... */ |
0a7de745 | 1160 | } |
2d21ac55 A |
1161 | #endif |
1162 | /* | |
1163 | * If we get an INT 0x7f when we do not expect to, | |
1164 | * treat it as an illegal instruction | |
1165 | */ | |
1166 | exc = EXC_BAD_INSTRUCTION; | |
1167 | code = EXC_I386_INVOP; | |
1168 | break; | |
1169 | ||
0a7de745 | 1170 | default: |
2d21ac55 | 1171 | panic("Unexpected user trap, type %d", type); |
1c79356b | 1172 | } |
94ff46dc A |
1173 | |
1174 | #if DEVELOPMENT || DEBUG | |
1175 | if (traptrace_index != TRAPTRACE_INVALID_INDEX) { | |
1176 | traptrace_end(traptrace_index, mach_absolute_time()); | |
1177 | } | |
1178 | #endif | |
1179 | ||
1180 | if (exc != 0) { | |
1181 | /* | |
1182 | * Note: Codepaths that directly return from user_trap() have pending | |
1183 | * ASTs processed in locore | |
1184 | */ | |
1185 | i386_exception(exc, code, subcode); | |
1186 | /* NOTREACHED */ | |
1187 | } | |
1c79356b A |
1188 | } |
1189 | ||
1c79356b A |
1190 | /* |
1191 | * Handle exceptions for i386. | |
1192 | * | |
1193 | * If we are an AT bus machine, we must turn off the AST for a | |
1194 | * delayed floating-point exception. | |
1195 | * | |
1196 | * If we are providing floating-point emulation, we may have | |
1197 | * to retrieve the real register values from the floating point | |
1198 | * emulator. | |
1199 | */ | |
1200 | void | |
1201 | i386_exception( | |
0a7de745 | 1202 | int exc, |
2d21ac55 A |
1203 | mach_exception_code_t code, |
1204 | mach_exception_subcode_t subcode) | |
1c79356b | 1205 | { |
2d21ac55 | 1206 | mach_exception_data_type_t codes[EXCEPTION_CODE_MAX]; |
1c79356b | 1207 | |
b0d623f7 | 1208 | DEBUG_KPRINT_SYSCALL_MACH("i386_exception: exc=%d code=0x%llx subcode=0x%llx\n", |
0a7de745 A |
1209 | exc, code, subcode); |
1210 | codes[0] = code; /* new exception interface */ | |
1c79356b | 1211 | codes[1] = subcode; |
91447636 | 1212 | exception_triage(exc, codes, 2); |
1c79356b A |
1213 | /*NOTREACHED*/ |
1214 | } | |
1215 | ||
0c530ab8 | 1216 | |
fe8ab488 A |
1217 | /* Synchronize a thread's x86_kernel_state (if any) with the given |
1218 | * x86_saved_state_t obtained from the trap/IPI handler; called in | |
0c530ab8 | 1219 | * kernel_trap() prior to entering the debugger, and when receiving |
fe8ab488 A |
1220 | * an "MP_KDP" IPI. Called with null saved_state if an incoming IPI |
1221 | * was detected from the kernel while spinning with interrupts masked. | |
0c530ab8 | 1222 | */ |
0a7de745 | 1223 | |
0c530ab8 | 1224 | void |
b0d623f7 | 1225 | sync_iss_to_iks(x86_saved_state_t *saved_state) |
0c530ab8 | 1226 | { |
5ba3f43e | 1227 | struct x86_kernel_state *iks = NULL; |
0c530ab8 A |
1228 | vm_offset_t kstack; |
1229 | boolean_t record_active_regs = FALSE; | |
1230 | ||
6d2010ae | 1231 | /* The PAL may have a special way to sync registers */ |
0a7de745 | 1232 | if (saved_state && saved_state->flavor == THREAD_STATE_NONE) { |
6d2010ae | 1233 | pal_get_kern_regs( saved_state ); |
0a7de745 | 1234 | } |
6d2010ae | 1235 | |
0a7de745 | 1236 | if (current_thread() != NULL && |
5ba3f43e | 1237 | (kstack = current_thread()->kernel_stack) != 0) { |
0a7de745 | 1238 | x86_saved_state64_t *regs = saved_state64(saved_state); |
0c530ab8 A |
1239 | |
1240 | iks = STACK_IKS(kstack); | |
1241 | ||
6d2010ae | 1242 | /* Did we take the trap/interrupt in kernel mode? */ |
fe8ab488 | 1243 | if (saved_state == NULL || /* NULL => polling in kernel */ |
0a7de745 A |
1244 | regs == USER_REGS64(current_thread())) { |
1245 | record_active_regs = TRUE; | |
1246 | } else { | |
b0d623f7 A |
1247 | iks->k_rbx = regs->rbx; |
1248 | iks->k_rsp = regs->isf.rsp; | |
1249 | iks->k_rbp = regs->rbp; | |
1250 | iks->k_r12 = regs->r12; | |
1251 | iks->k_r13 = regs->r13; | |
1252 | iks->k_r14 = regs->r14; | |
1253 | iks->k_r15 = regs->r15; | |
1254 | iks->k_rip = regs->isf.rip; | |
1255 | } | |
0c530ab8 A |
1256 | } |
1257 | ||
1258 | if (record_active_regs == TRUE) { | |
b0d623f7 | 1259 | /* Show the trap handler path */ |
0a7de745 A |
1260 | __asm__ volatile ("movq %%rbx, %0" : "=m" (iks->k_rbx)); |
1261 | __asm__ volatile ("movq %%rsp, %0" : "=m" (iks->k_rsp)); | |
1262 | __asm__ volatile ("movq %%rbp, %0" : "=m" (iks->k_rbp)); | |
1263 | __asm__ volatile ("movq %%r12, %0" : "=m" (iks->k_r12)); | |
1264 | __asm__ volatile ("movq %%r13, %0" : "=m" (iks->k_r13)); | |
1265 | __asm__ volatile ("movq %%r14, %0" : "=m" (iks->k_r14)); | |
1266 | __asm__ volatile ("movq %%r15, %0" : "=m" (iks->k_r15)); | |
b0d623f7 | 1267 | /* "Current" instruction pointer */ |
0a7de745 A |
1268 | __asm__ volatile ("leaq 1f(%%rip), %%rax; mov %%rax, %0\n1:" |
1269 | : "=m" (iks->k_rip) | |
1270 | : | |
1271 | : "rax"); | |
0c530ab8 A |
1272 | } |
1273 | } | |
1274 | ||
1275 | /* | |
1276 | * This is used by the NMI interrupt handler (from mp.c) to | |
1277 | * uncondtionally sync the trap handler context to the IKS | |
1278 | * irrespective of whether the NMI was fielded in kernel | |
1279 | * or user space. | |
1280 | */ | |
1281 | void | |
0a7de745 A |
1282 | sync_iss_to_iks_unconditionally(__unused x86_saved_state_t *saved_state) |
1283 | { | |
b0d623f7 | 1284 | struct x86_kernel_state *iks; |
0c530ab8 | 1285 | vm_offset_t kstack; |
0c530ab8 A |
1286 | |
1287 | if ((kstack = current_thread()->kernel_stack) != 0) { | |
0c530ab8 | 1288 | iks = STACK_IKS(kstack); |
b0d623f7 | 1289 | /* Display the trap handler path */ |
0a7de745 A |
1290 | __asm__ volatile ("movq %%rbx, %0" : "=m" (iks->k_rbx)); |
1291 | __asm__ volatile ("movq %%rsp, %0" : "=m" (iks->k_rsp)); | |
1292 | __asm__ volatile ("movq %%rbp, %0" : "=m" (iks->k_rbp)); | |
1293 | __asm__ volatile ("movq %%r12, %0" : "=m" (iks->k_r12)); | |
1294 | __asm__ volatile ("movq %%r13, %0" : "=m" (iks->k_r13)); | |
1295 | __asm__ volatile ("movq %%r14, %0" : "=m" (iks->k_r14)); | |
1296 | __asm__ volatile ("movq %%r15, %0" : "=m" (iks->k_r15)); | |
b0d623f7 | 1297 | /* "Current" instruction pointer */ |
0a7de745 | 1298 | __asm__ volatile ("leaq 1f(%%rip), %%rax; mov %%rax, %0\n1:" : "=m" (iks->k_rip)::"rax"); |
0c530ab8 A |
1299 | } |
1300 | } | |
00867663 A |
1301 | |
1302 | #if DEBUG | |
5c9f4661 A |
1303 | #define TERI 1 |
1304 | #endif | |
1305 | ||
1306 | #if TERI | |
0a7de745 | 1307 | extern void thread_exception_return_internal(void) __dead2; |
00867663 | 1308 | |
0a7de745 A |
1309 | void |
1310 | thread_exception_return(void) | |
1311 | { | |
00867663 A |
1312 | thread_t thread = current_thread(); |
1313 | ml_set_interrupts_enabled(FALSE); | |
d9a64523 | 1314 | if (thread_is_64bit_addr(thread) != task_has_64Bit_addr(thread->task)) { |
0a7de745 | 1315 | panic("Task/thread bitness mismatch %p %p, task: %d, thread: %d", thread, thread->task, thread_is_64bit_addr(thread), task_has_64Bit_addr(thread->task)); |
00867663 A |
1316 | } |
1317 | ||
d9a64523 | 1318 | if (thread_is_64bit_addr(thread)) { |
00867663 A |
1319 | if ((gdt_desc_p(USER64_CS)->access & ACC_PL_U) == 0) { |
1320 | panic("64-GDT mismatch %p, descriptor: %p", thread, gdt_desc_p(USER64_CS)); | |
1321 | } | |
1322 | } else { | |
0a7de745 A |
1323 | if ((gdt_desc_p(USER_CS)->access & ACC_PL_U) == 0) { |
1324 | panic("32-GDT mismatch %p, descriptor: %p", thread, gdt_desc_p(USER_CS)); | |
00867663 A |
1325 | } |
1326 | } | |
0a7de745 | 1327 | assert(get_preemption_level() == 0); |
00867663 A |
1328 | thread_exception_return_internal(); |
1329 | } | |
1330 | #endif |