]> git.saurik.com Git - apple/xnu.git/blob - osfmk/arm64/sleh.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-4570.71.2.tar.gz
[apple/xnu.git] / osfmk / arm64 / sleh.c
1 /*
2 * Copyright (c) 2012-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <arm/caches_internal.h>
30 #include <arm/cpu_data.h>
31 #include <arm/cpu_data_internal.h>
32 #include <arm/misc_protos.h>
33 #include <arm/thread.h>
34 #include <arm/rtclock.h>
35 #include <arm/trap.h> /* for IS_ARM_GDB_TRAP() et al */
36 #include <arm64/proc_reg.h>
37 #include <arm64/machine_machdep.h>
38 #include <arm64/monotonic.h>
39
40 #include <kern/debug.h>
41 #include <kern/thread.h>
42 #include <mach/exception.h>
43 #include <mach/vm_types.h>
44 #include <mach/machine/thread_status.h>
45
46 #include <machine/atomic.h>
47 #include <machine/machlimits.h>
48
49 #include <pexpert/arm/protos.h>
50
51 #include <vm/vm_page.h>
52 #include <vm/pmap.h>
53 #include <vm/vm_fault.h>
54 #include <vm/vm_kern.h>
55
56 #include <sys/kdebug.h>
57
58 #include <kern/policy_internal.h>
59 #if CONFIG_TELEMETRY
60 #include <kern/telemetry.h>
61 #endif
62
63 #include <prng/random.h>
64
65 #ifndef __arm64__
66 #error Should only be compiling for arm64.
67 #endif
68
69 #define TEST_CONTEXT32_SANITY(context) \
70 (context->ss.ash.flavor == ARM_SAVED_STATE32 && context->ss.ash.count == ARM_SAVED_STATE32_COUNT && \
71 context->ns.nsh.flavor == ARM_NEON_SAVED_STATE32 && context->ns.nsh.count == ARM_NEON_SAVED_STATE32_COUNT)
72
73 #define TEST_CONTEXT64_SANITY(context) \
74 (context->ss.ash.flavor == ARM_SAVED_STATE64 && context->ss.ash.count == ARM_SAVED_STATE64_COUNT && \
75 context->ns.nsh.flavor == ARM_NEON_SAVED_STATE64 && context->ns.nsh.count == ARM_NEON_SAVED_STATE64_COUNT)
76
77 #define ASSERT_CONTEXT_SANITY(context) \
78 assert(TEST_CONTEXT32_SANITY(context) || TEST_CONTEXT64_SANITY(context))
79
80
81 #define COPYIN(src, dst, size) \
82 (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) ? \
83 copyin_kern(src, dst, size) \
84 : \
85 copyin(src, dst, size)
86
87 #define COPYOUT(src, dst, size) \
88 (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) ? \
89 copyout_kern(src, dst, size) \
90 : \
91 copyout(src, dst, size)
92
93 // Below is for concatenating a string param to a string literal
94 #define STR1(x) #x
95 #define STR(x) STR1(x)
96
97 void panic_with_thread_kernel_state(const char *msg, arm_saved_state_t *ss);
98
99 void sleh_synchronous_sp1(arm_context_t *, uint32_t, vm_offset_t);
100 void sleh_synchronous(arm_context_t *, uint32_t, vm_offset_t);
101 void sleh_irq(arm_saved_state_t *);
102 void sleh_fiq(arm_saved_state_t *);
103 void sleh_serror(arm_context_t *context, uint32_t esr, vm_offset_t far);
104 void sleh_invalid_stack(arm_context_t *context, uint32_t esr, vm_offset_t far);
105
106 static void sleh_interrupt_handler_prologue(arm_saved_state_t *, unsigned int type);
107 static void sleh_interrupt_handler_epilogue(void);
108
109 static void handle_svc(arm_saved_state_t *);
110 static void handle_mach_absolute_time_trap(arm_saved_state_t *);
111 static void handle_mach_continuous_time_trap(arm_saved_state_t *);
112
113 static void handle_msr_trap(arm_saved_state_t *state, uint32_t iss);
114
115 extern kern_return_t arm_fast_fault(pmap_t, vm_map_address_t, vm_prot_t, boolean_t);
116
117 static void handle_uncategorized(arm_saved_state_t *, boolean_t);
118 static void handle_breakpoint(arm_saved_state_t *);
119
120 typedef void(*abort_inspector_t)(uint32_t, fault_status_t *, vm_prot_t *);
121 static void inspect_instruction_abort(uint32_t, fault_status_t *, vm_prot_t *);
122 static void inspect_data_abort(uint32_t, fault_status_t *, vm_prot_t *);
123
124 static int is_vm_fault(fault_status_t);
125 static int is_alignment_fault(fault_status_t);
126
127 typedef void(*abort_handler_t)(arm_saved_state_t *, uint32_t, vm_offset_t, fault_status_t, vm_prot_t, vm_offset_t);
128 static void handle_user_abort(arm_saved_state_t *, uint32_t, vm_offset_t, fault_status_t, vm_prot_t, vm_offset_t);
129 static void handle_kernel_abort(arm_saved_state_t *, uint32_t, vm_offset_t, fault_status_t, vm_prot_t, vm_offset_t);
130
131 static void handle_pc_align(arm_saved_state_t *ss);
132 static void handle_sp_align(arm_saved_state_t *ss);
133 static void handle_sw_step_debug(arm_saved_state_t *ss);
134 static void handle_wf_trap(arm_saved_state_t *ss);
135
136 static void handle_watchpoint(vm_offset_t fault_addr);
137
138 static void handle_abort(arm_saved_state_t *, uint32_t, vm_offset_t, vm_offset_t, abort_inspector_t, abort_handler_t);
139
140 static void handle_user_trapped_instruction32(arm_saved_state_t *, uint32_t esr);
141
142 static void handle_simd_trap(arm_saved_state_t *, uint32_t esr);
143
144 extern void mach_kauth_cred_uthread_update(void);
145 void mach_syscall_trace_exit(unsigned int retval, unsigned int call_number);
146
147 struct uthread;
148 struct proc;
149
150 extern void
151 unix_syscall(struct arm_saved_state * regs, thread_t thread_act,
152 struct uthread * uthread, struct proc * proc);
153
154 extern void
155 mach_syscall(struct arm_saved_state*);
156
157 #if CONFIG_DTRACE
158 extern kern_return_t dtrace_user_probe(arm_saved_state_t* regs);
159 extern boolean_t dtrace_tally_fault(user_addr_t);
160
161 /* Traps for userland processing. Can't include bsd/sys/fasttrap_isa.h, so copy and paste the trap instructions
162 over from that file. Need to keep these in sync! */
163 #define FASTTRAP_ARM32_INSTR 0xe7ffdefc
164 #define FASTTRAP_THUMB32_INSTR 0xdefc
165 #define FASTTRAP_ARM64_INSTR 0xe7eeee7e
166
167 #define FASTTRAP_ARM32_RET_INSTR 0xe7ffdefb
168 #define FASTTRAP_THUMB32_RET_INSTR 0xdefb
169 #define FASTTRAP_ARM64_RET_INSTR 0xe7eeee7d
170
171 /* See <rdar://problem/4613924> */
172 perfCallback tempDTraceTrapHook = NULL; /* Pointer to DTrace fbt trap hook routine */
173 #endif
174
175 #if CONFIG_PGTRACE
176 extern boolean_t pgtrace_enabled;
177 #endif
178
179 #if __ARM_PAN_AVAILABLE__
180 #endif
181
182 #if defined(APPLECYCLONE)
183 #define CPU_NAME "Cyclone"
184 #elif defined(APPLETYPHOON)
185 #define CPU_NAME "Typhoon"
186 #elif defined(APPLETWISTER)
187 #define CPU_NAME "Twister"
188 #elif defined(APPLEHURRICANE)
189 #define CPU_NAME "Hurricane"
190 #else
191 #define CPU_NAME "Unknown"
192 #endif
193
194 #if (CONFIG_KERNEL_INTEGRITY && defined(KERNEL_INTEGRITY_WT))
195 #define ESR_WT_SERROR(esr) (((esr) & 0xffffff00) == 0xbf575400)
196 #define ESR_WT_REASON(esr) ((esr) & 0xff)
197
198 #define WT_REASON_NONE 0
199 #define WT_REASON_INTEGRITY_FAIL 1
200 #define WT_REASON_BAD_SYSCALL 2
201 #define WT_REASON_NOT_LOCKED 3
202 #define WT_REASON_ALREADY_LOCKED 4
203 #define WT_REASON_SW_REQ 5
204 #define WT_REASON_PT_INVALID 6
205 #define WT_REASON_PT_VIOLATION 7
206 #define WT_REASON_REG_VIOLATION 8
207 #endif
208
209
210 static inline unsigned
211 __ror(unsigned value, unsigned shift)
212 {
213 return (((unsigned)(value) >> (unsigned)(shift)) |
214 (unsigned)(value) << ((unsigned)(sizeof(unsigned) * CHAR_BIT) - (unsigned)(shift)));
215 }
216
217 static void
218 arm64_implementation_specific_error(arm_saved_state_t *state, uint32_t esr, vm_offset_t far)
219 {
220 #if defined(APPLE_ARM64_ARCH_FAMILY)
221 uint64_t fed_err_sts, mmu_err_sts, lsu_err_sts;
222 #if defined(NO_ECORE)
223 uint64_t l2c_err_sts, l2c_err_adr, l2c_err_inf;
224
225 mmu_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_MMU_ERR_STS));
226 l2c_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_L2C_ERR_STS));
227 l2c_err_adr = __builtin_arm_rsr64(STR(ARM64_REG_L2C_ERR_ADR));
228 l2c_err_inf = __builtin_arm_rsr64(STR(ARM64_REG_L2C_ERR_INF));
229 lsu_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_LSU_ERR_STS));
230 fed_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_FED_ERR_STS));
231
232 panic_plain("Unhandled " CPU_NAME
233 " implementation specific error. state=%p esr=%#x far=%p\n"
234 "\tlsu_err_sts:%p, fed_err_sts:%p, mmu_err_sts:%p\n"
235 "\tl2c_err_sts:%p, l2c_err_adr:%p, l2c_err_inf:%p\n",
236 state, esr, (void *)far,
237 (void *)lsu_err_sts, (void *)fed_err_sts, (void *)mmu_err_sts,
238 (void *)l2c_err_sts, (void *)l2c_err_adr, (void *)l2c_err_inf);
239
240 #elif defined(HAS_MIGSTS)
241 uint64_t l2c_err_sts, l2c_err_adr, l2c_err_inf, mpidr, migsts;
242
243 mpidr = __builtin_arm_rsr64("MPIDR_EL1");
244 migsts = __builtin_arm_rsr64(STR(ARM64_REG_MIGSTS_EL1));
245 mmu_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_MMU_ERR_STS));
246 l2c_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_L2C_ERR_STS));
247 l2c_err_adr = __builtin_arm_rsr64(STR(ARM64_REG_L2C_ERR_ADR));
248 l2c_err_inf = __builtin_arm_rsr64(STR(ARM64_REG_L2C_ERR_INF));
249 lsu_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_LSU_ERR_STS));
250 fed_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_FED_ERR_STS));
251
252 panic_plain("Unhandled " CPU_NAME
253 " implementation specific error. state=%p esr=%#x far=%p p-core?%d migsts=%p\n"
254 "\tlsu_err_sts:%p, fed_err_sts:%p, mmu_err_sts:%p\n"
255 "\tl2c_err_sts:%p, l2c_err_adr:%p, l2c_err_inf:%p\n",
256 state, esr, (void *)far, !!(mpidr & MPIDR_PNE), (void *)migsts,
257 (void *)lsu_err_sts, (void *)fed_err_sts, (void *)mmu_err_sts,
258 (void *)l2c_err_sts, (void *)l2c_err_adr, (void *)l2c_err_inf);
259 #else // !defined(NO_ECORE) && !defined(HAS_MIGSTS)
260 uint64_t llc_err_sts, llc_err_adr, llc_err_inf, mpidr;
261
262 mpidr = __builtin_arm_rsr64("MPIDR_EL1");
263
264 if (mpidr & MPIDR_PNE) {
265 mmu_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_MMU_ERR_STS));
266 lsu_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_LSU_ERR_STS));
267 fed_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_FED_ERR_STS));
268 } else {
269 mmu_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_E_MMU_ERR_STS));
270 lsu_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_E_LSU_ERR_STS));
271 fed_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_E_FED_ERR_STS));
272 }
273
274 llc_err_sts = __builtin_arm_rsr64(STR(ARM64_REG_L2C_ERR_STS));
275 llc_err_adr = __builtin_arm_rsr64(STR(ARM64_REG_L2C_ERR_ADR));
276 llc_err_inf = __builtin_arm_rsr64(STR(ARM64_REG_L2C_ERR_INF));
277
278 panic_plain("Unhandled " CPU_NAME
279 " implementation specific error. state=%p esr=%#x far=%p p-core?%d\n"
280 "\tlsu_err_sts:%p, fed_err_sts:%p, mmu_err_sts:%p\n"
281 "\tllc_err_sts:%p, llc_err_adr:%p, llc_err_inf:%p\n",
282 state, esr, (void *)far, !!(mpidr & MPIDR_PNE),
283 (void *)lsu_err_sts, (void *)fed_err_sts, (void *)mmu_err_sts,
284 (void *)llc_err_sts, (void *)llc_err_adr, (void *)llc_err_inf);
285 #endif
286 #else // !defined(APPLE_ARM64_ARCH_FAMILY)
287 panic_plain("Unhandled implementation specific error\n");
288 #endif
289 }
290
291 #if CONFIG_KERNEL_INTEGRITY
292 #pragma clang diagnostic push
293 #pragma clang diagnostic ignored "-Wunused-parameter"
294 static void
295 kernel_integrity_error_handler(uint32_t esr, vm_offset_t far) {
296 #if defined(KERNEL_INTEGRITY_WT)
297 #if (DEVELOPMENT || DEBUG)
298 if (ESR_WT_SERROR(esr)) {
299 switch (ESR_WT_REASON(esr)) {
300 case WT_REASON_INTEGRITY_FAIL:
301 panic_plain("Kernel integrity, violation in frame 0x%016lx.", far);
302 case WT_REASON_BAD_SYSCALL:
303 panic_plain("Kernel integrity, bad syscall.");
304 case WT_REASON_NOT_LOCKED:
305 panic_plain("Kernel integrity, not locked.");
306 case WT_REASON_ALREADY_LOCKED:
307 panic_plain("Kernel integrity, already locked.");
308 case WT_REASON_SW_REQ:
309 panic_plain("Kernel integrity, software request.");
310 case WT_REASON_PT_INVALID:
311 panic_plain("Kernel integrity, encountered invalid TTE/PTE while "
312 "walking 0x%016lx.", far);
313 case WT_REASON_PT_VIOLATION:
314 panic_plain("Kernel integrity, violation in mapping 0x%016lx.",
315 far);
316 case WT_REASON_REG_VIOLATION:
317 panic_plain("Kernel integrity, violation in system register %d.",
318 (unsigned) far);
319 default:
320 panic_plain("Kernel integrity, unknown (esr=0x%08x).", esr);
321 }
322 }
323 #else
324 if (ESR_WT_SERROR(esr)) {
325 panic_plain("SError esr: 0x%08x far: 0x%016lx.", esr, far);
326 }
327 #endif
328 #endif
329 }
330 #pragma clang diagnostic pop
331 #endif
332
333 static void
334 arm64_platform_error(arm_saved_state_t *state, uint32_t esr, vm_offset_t far)
335 {
336 cpu_data_t *cdp = getCpuDatap();
337
338 #if CONFIG_KERNEL_INTEGRITY
339 kernel_integrity_error_handler(esr, far);
340 #endif
341
342 if (cdp->platform_error_handler != (platform_error_handler_t) NULL)
343 (*(platform_error_handler_t)cdp->platform_error_handler) (cdp->cpu_id, far);
344 else
345 arm64_implementation_specific_error(state, esr, far);
346 }
347
348 void
349 panic_with_thread_kernel_state(const char *msg, arm_saved_state_t *ss)
350 {
351 boolean_t ss_valid;
352
353 ss_valid = is_saved_state64(ss);
354 arm_saved_state64_t *state = saved_state64(ss);
355
356 panic_plain("%s (saved state: %p%s)\n"
357 "\t x0: 0x%016llx x1: 0x%016llx x2: 0x%016llx x3: 0x%016llx\n"
358 "\t x4: 0x%016llx x5: 0x%016llx x6: 0x%016llx x7: 0x%016llx\n"
359 "\t x8: 0x%016llx x9: 0x%016llx x10: 0x%016llx x11: 0x%016llx\n"
360 "\t x12: 0x%016llx x13: 0x%016llx x14: 0x%016llx x15: 0x%016llx\n"
361 "\t x16: 0x%016llx x17: 0x%016llx x18: 0x%016llx x19: 0x%016llx\n"
362 "\t x20: 0x%016llx x21: 0x%016llx x22: 0x%016llx x23: 0x%016llx\n"
363 "\t x24: 0x%016llx x25: 0x%016llx x26: 0x%016llx x27: 0x%016llx\n"
364 "\t x28: 0x%016llx fp: 0x%016llx lr: 0x%016llx sp: 0x%016llx\n"
365 "\t pc: 0x%016llx cpsr: 0x%08x esr: 0x%08x far: 0x%016llx\n",
366 msg, ss, (ss_valid ? "" : " INVALID"),
367 state->x[0], state->x[1], state->x[2], state->x[3],
368 state->x[4], state->x[5], state->x[6], state->x[7],
369 state->x[8], state->x[9], state->x[10], state->x[11],
370 state->x[12], state->x[13], state->x[14], state->x[15],
371 state->x[16], state->x[17], state->x[18], state->x[19],
372 state->x[20], state->x[21], state->x[22], state->x[23],
373 state->x[24], state->x[25], state->x[26], state->x[27],
374 state->x[28], state->fp, state->lr, state->sp,
375 state->pc, state->cpsr, state->esr, state->far);
376 }
377
378
379 void
380 sleh_synchronous_sp1(arm_context_t *context, uint32_t esr, vm_offset_t far __unused)
381 {
382 esr_exception_class_t class = ESR_EC(esr);
383 arm_saved_state_t *state = &context->ss;
384
385 switch (class) {
386 case ESR_EC_UNCATEGORIZED:
387 {
388 uint32_t instr = *((uint32_t*)get_saved_state_pc(state));
389 if (IS_ARM_GDB_TRAP(instr))
390 DebuggerCall(EXC_BREAKPOINT, state);
391 // Intentionally fall through to panic if we return from the debugger
392 }
393 default:
394 panic_with_thread_kernel_state("Synchronous exception taken while SP1 selected", state);
395 }
396 }
397
398 void
399 sleh_synchronous(arm_context_t *context, uint32_t esr, vm_offset_t far)
400 {
401 esr_exception_class_t class = ESR_EC(esr);
402 arm_saved_state_t *state = &context->ss;
403 vm_offset_t recover = 0;
404 thread_t thread = current_thread();
405
406 ASSERT_CONTEXT_SANITY(context);
407
408 /* Don't run exception handler with recover handler set in case of double fault */
409 if (thread->recover) {
410 recover = thread->recover;
411 thread->recover = (vm_offset_t)NULL;
412 }
413
414 /* Inherit the interrupt masks from previous context */
415 if (SPSR_INTERRUPTS_ENABLED(get_saved_state_cpsr(state)))
416 ml_set_interrupts_enabled(TRUE);
417
418 switch (class) {
419 case ESR_EC_SVC_64:
420 if (!is_saved_state64(state) || !PSR64_IS_USER(get_saved_state_cpsr(state))) {
421 panic("Invalid SVC_64 context");
422 }
423
424 handle_svc(state);
425 break;
426
427 case ESR_EC_DABORT_EL0:
428 handle_abort(state, esr, far, recover, inspect_data_abort, handle_user_abort);
429 assert(0); /* Unreachable */
430
431 case ESR_EC_MSR_TRAP:
432 handle_msr_trap(state, ESR_ISS(esr));
433 break;
434
435 case ESR_EC_IABORT_EL0:
436 handle_abort(state, esr, far, recover, inspect_instruction_abort, handle_user_abort);
437 assert(0); /* Unreachable */
438
439 case ESR_EC_IABORT_EL1:
440 panic("Kernel instruction fetch abort: pc=%p iss=0x%x far=%p. Note: the faulting frame may be missing in the backtrace.",
441 (void *)get_saved_state_pc(state), ESR_ISS(esr), (void*)far);
442
443 case ESR_EC_PC_ALIGN:
444 handle_pc_align(state);
445 assert(0); /* Unreachable */
446 break;
447
448 case ESR_EC_DABORT_EL1:
449 handle_abort(state, esr, far, recover, inspect_data_abort, handle_kernel_abort);
450 break;
451
452 case ESR_EC_UNCATEGORIZED:
453 assert(!ESR_ISS(esr));
454
455 handle_uncategorized(&context->ss, ESR_INSTR_IS_2BYTES(esr));
456 /* TODO: Uncomment this after stackshot uses a brk instruction
457 * rather than an undefined instruction, as stackshot is the
458 * only case where we want to return to the first-level handler.
459 */
460 //assert(0); /* Unreachable */
461 break;
462
463 case ESR_EC_SP_ALIGN:
464 handle_sp_align(state);
465 assert(0); /* Unreachable */
466 break;
467
468 case ESR_EC_BKPT_AARCH32:
469 handle_breakpoint(state);
470 assert(0); /* Unreachable */
471 break;
472
473 case ESR_EC_BRK_AARCH64:
474 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
475
476 kprintf("Breakpoint instruction exception from kernel. Hanging here (by design).\n");
477 for (;;);
478
479 __unreachable_ok_push
480 DebuggerCall(EXC_BREAKPOINT, &context->ss);
481 break;
482 __unreachable_ok_pop
483 } else {
484 handle_breakpoint(state);
485 assert(0); /* Unreachable */
486 }
487
488 case ESR_EC_BKPT_REG_MATCH_EL0:
489 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
490 handle_breakpoint(state);
491 assert(0); /* Unreachable */
492 }
493 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
494 class, state, class, esr, (void *)far);
495 assert(0); /* Unreachable */
496 break;
497
498 case ESR_EC_BKPT_REG_MATCH_EL1:
499 if (!PE_i_can_has_debugger(NULL) && FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
500 kprintf("Hardware Breakpoint Debug exception from kernel. Hanging here (by design).\n");
501 for (;;);
502
503 __unreachable_ok_push
504 DebuggerCall(EXC_BREAKPOINT, &context->ss);
505 break;
506 __unreachable_ok_pop
507 }
508 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
509 class, state, class, esr, (void *)far);
510 assert(0); /* Unreachable */
511 break;
512
513 case ESR_EC_SW_STEP_DEBUG_EL0:
514 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
515 handle_sw_step_debug(state);
516 assert(0); /* Unreachable */
517 }
518 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
519 class, state, class, esr, (void *)far);
520 assert(0); /* Unreachable */
521 break;
522
523 case ESR_EC_SW_STEP_DEBUG_EL1:
524 if (!PE_i_can_has_debugger(NULL) && FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
525 kprintf("Software Step Debug exception from kernel. Hanging here (by design).\n");
526 for (;;);
527
528 __unreachable_ok_push
529 DebuggerCall(EXC_BREAKPOINT, &context->ss);
530 break;
531 __unreachable_ok_pop
532 }
533 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
534 class, state, class, esr, (void *)far);
535 assert(0); /* Unreachable */
536 break;
537
538 case ESR_EC_WATCHPT_MATCH_EL0:
539 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
540 handle_watchpoint(far);
541 assert(0); /* Unreachable */
542 }
543 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
544 class, state, class, esr, (void *)far);
545 assert(0); /* Unreachable */
546 break;
547
548 case ESR_EC_WATCHPT_MATCH_EL1:
549 /*
550 * If we hit a watchpoint in kernel mode, probably in a copyin/copyout which we don't want to
551 * abort. Turn off watchpoints and keep going; we'll turn them back on in return_from_exception..
552 */
553 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
554 arm_debug_set(NULL);
555 break; /* return to first level handler */
556 }
557 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
558 class, state, class, esr, (void *)far);
559 assert(0); /* Unreachable */
560 break;
561
562 case ESR_EC_TRAP_SIMD_FP:
563 handle_simd_trap(state, esr);
564 assert(0);
565 break;
566
567 case ESR_EC_ILLEGAL_INSTR_SET:
568 if (EXCB_ACTION_RERUN !=
569 ex_cb_invoke(EXCB_CLASS_ILLEGAL_INSTR_SET, far)) {
570 // instruction is not re-executed
571 panic("Illegal instruction set exception. state=%p class=%u esr=%u far=%p spsr=0x%x",
572 state, class, esr, (void *)far, get_saved_state_cpsr(state));
573 assert(0);
574 }
575 // must clear this fault in PSR to re-run
576 set_saved_state_cpsr(state, get_saved_state_cpsr(state) & (~PSR64_IL));
577 break;
578
579 case ESR_EC_MCR_MRC_CP15_TRAP:
580 case ESR_EC_MCRR_MRRC_CP15_TRAP:
581 case ESR_EC_MCR_MRC_CP14_TRAP:
582 case ESR_EC_LDC_STC_CP14_TRAP:
583 case ESR_EC_MCRR_MRRC_CP14_TRAP:
584 handle_user_trapped_instruction32(state, esr);
585 assert(0);
586 break;
587
588 case ESR_EC_WFI_WFE:
589 // Use of WFI or WFE instruction when they have been disabled for EL0
590 handle_wf_trap(state);
591 assert(0); /* Unreachable */
592 break;
593
594 default:
595 panic("Unsupported synchronous exception. state=%p class=%u esr=%u far=%p",
596 state, class, esr, (void *)far);
597 assert(0); /* Unreachable */
598 break;
599 }
600
601 if (recover)
602 thread->recover = recover;
603 }
604
605 /*
606 * Uncategorized exceptions are a catch-all for general execution errors.
607 * ARM64_TODO: For now, we assume this is for undefined instruction exceptions.
608 */
609 static void
610 handle_uncategorized(arm_saved_state_t *state, boolean_t instrLen2)
611 {
612 exception_type_t exception = EXC_BAD_INSTRUCTION;
613 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
614 mach_msg_type_number_t numcodes = 2;
615 uint32_t instr;
616
617 if (instrLen2) {
618 uint16_t instr16;
619 COPYIN(get_saved_state_pc(state), (char *)&instr16, sizeof(instr16));
620
621 instr = instr16;
622 } else {
623 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
624 }
625
626 #if CONFIG_DTRACE
627 if (tempDTraceTrapHook && (tempDTraceTrapHook(exception, state, 0, 0) == KERN_SUCCESS)) {
628 return;
629 }
630
631 if (PSR64_IS_USER64(get_saved_state_cpsr(state))) {
632 /*
633 * For a 64bit user process, we care about all 4 bytes of the
634 * instr.
635 */
636 if (instr == FASTTRAP_ARM64_INSTR || instr == FASTTRAP_ARM64_RET_INSTR) {
637 if (dtrace_user_probe(state) == KERN_SUCCESS)
638 return;
639 }
640 } else if (PSR64_IS_USER32(get_saved_state_cpsr(state))) {
641 /*
642 * For a 32bit user process, we check for thumb mode, in
643 * which case we only care about a 2 byte instruction length.
644 * For non-thumb mode, we care about all 4 bytes of the instructin.
645 */
646 if (get_saved_state_cpsr(state) & PSR64_MODE_USER32_THUMB) {
647 if (((uint16_t)instr == FASTTRAP_THUMB32_INSTR) ||
648 ((uint16_t)instr == FASTTRAP_THUMB32_RET_INSTR)) {
649 if (dtrace_user_probe(state) == KERN_SUCCESS) {
650 return;
651 }
652 }
653 } else {
654 if ((instr == FASTTRAP_ARM32_INSTR) ||
655 (instr == FASTTRAP_ARM32_RET_INSTR)) {
656 if (dtrace_user_probe(state) == KERN_SUCCESS) {
657 return;
658 }
659 }
660 }
661 }
662
663 #endif /* CONFIG_DTRACE */
664
665 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
666 if (IS_ARM_GDB_TRAP(instr)) {
667 boolean_t interrupt_state;
668 vm_offset_t kstackptr;
669 exception = EXC_BREAKPOINT;
670
671 interrupt_state = ml_set_interrupts_enabled(FALSE);
672
673 /* Save off the context here (so that the debug logic
674 * can see the original state of this thread).
675 */
676 kstackptr = (vm_offset_t) current_thread()->machine.kstackptr;
677 if (kstackptr) {
678 ((thread_kernel_state_t) kstackptr)->machine.ss = *state;
679 }
680
681 /* Hop into the debugger (typically either due to a
682 * fatal exception, an explicit panic, or a stackshot
683 * request.
684 */
685 DebuggerCall(exception, state);
686
687 (void) ml_set_interrupts_enabled(interrupt_state);
688 return;
689 } else {
690 panic("Undefined kernel instruction: pc=%p instr=%x\n", (void*)get_saved_state_pc(state), instr);
691 }
692 }
693
694 /*
695 * Check for GDB breakpoint via illegal opcode.
696 */
697 if (instrLen2) {
698 if (IS_THUMB_GDB_TRAP(instr)) {
699 exception = EXC_BREAKPOINT;
700 codes[0] = EXC_ARM_BREAKPOINT;
701 codes[1] = instr;
702 } else {
703 codes[1] = instr;
704 }
705 } else {
706 if (IS_ARM_GDB_TRAP(instr)) {
707 exception = EXC_BREAKPOINT;
708 codes[0] = EXC_ARM_BREAKPOINT;
709 codes[1] = instr;
710 } else if (IS_THUMB_GDB_TRAP((instr & 0xFFFF))) {
711 exception = EXC_BREAKPOINT;
712 codes[0] = EXC_ARM_BREAKPOINT;
713 codes[1] = instr & 0xFFFF;
714 } else if (IS_THUMB_GDB_TRAP((instr >> 16))) {
715 exception = EXC_BREAKPOINT;
716 codes[0] = EXC_ARM_BREAKPOINT;
717 codes[1] = instr >> 16;
718 } else {
719 codes[1] = instr;
720 }
721 }
722
723 exception_triage(exception, codes, numcodes);
724 assert(0); /* NOTREACHED */
725 }
726
727 static void
728 handle_breakpoint(arm_saved_state_t *state)
729 {
730 exception_type_t exception = EXC_BREAKPOINT;
731 mach_exception_data_type_t codes[2] = {EXC_ARM_BREAKPOINT};
732 mach_msg_type_number_t numcodes = 2;
733
734 codes[1] = get_saved_state_pc(state);
735 exception_triage(exception, codes, numcodes);
736 assert(0); /* NOTREACHED */
737 }
738
739 static void
740 handle_watchpoint(vm_offset_t fault_addr)
741 {
742 exception_type_t exception = EXC_BREAKPOINT;
743 mach_exception_data_type_t codes[2] = {EXC_ARM_DA_DEBUG};
744 mach_msg_type_number_t numcodes = 2;
745
746 codes[1] = fault_addr;
747 exception_triage(exception, codes, numcodes);
748 assert(0); /* NOTREACHED */
749 }
750
751 static void
752 handle_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr, vm_offset_t recover,
753 abort_inspector_t inspect_abort, abort_handler_t handler)
754 {
755 fault_status_t fault_code;
756 vm_prot_t fault_type;
757
758 inspect_abort(ESR_ISS(esr), &fault_code, &fault_type);
759 handler(state, esr, fault_addr, fault_code, fault_type, recover);
760 }
761
762 static void
763 inspect_instruction_abort(uint32_t iss, fault_status_t *fault_code, vm_prot_t *fault_type)
764 {
765 getCpuDatap()->cpu_stat.instr_ex_cnt++;
766 *fault_code = ISS_IA_FSC(iss);
767 *fault_type = (VM_PROT_READ | VM_PROT_EXECUTE);
768 }
769
770 static void
771 inspect_data_abort(uint32_t iss, fault_status_t *fault_code, vm_prot_t *fault_type)
772 {
773 getCpuDatap()->cpu_stat.data_ex_cnt++;
774 *fault_code = ISS_DA_FSC(iss);
775
776 /* Cache operations report faults as write access. Change these to read access. */
777 if ((iss & ISS_DA_WNR) && !(iss & ISS_DA_CM)) {
778 *fault_type = (VM_PROT_READ | VM_PROT_WRITE);
779 } else {
780 *fault_type = (VM_PROT_READ);
781 }
782 }
783
784 static void
785 handle_pc_align(arm_saved_state_t *ss)
786 {
787 exception_type_t exc;
788 mach_exception_data_type_t codes[2];
789 mach_msg_type_number_t numcodes = 2;
790
791 if (!PSR64_IS_USER(get_saved_state_cpsr(ss))) {
792 panic_with_thread_kernel_state("PC alignment exception from kernel.", ss);
793 }
794
795 exc = EXC_BAD_ACCESS;
796 codes[0] = EXC_ARM_DA_ALIGN;
797 codes[1] = get_saved_state_pc(ss);
798
799 exception_triage(exc, codes, numcodes);
800 assert(0); /* NOTREACHED */
801 }
802
803 static void
804 handle_sp_align(arm_saved_state_t *ss)
805 {
806 exception_type_t exc;
807 mach_exception_data_type_t codes[2];
808 mach_msg_type_number_t numcodes = 2;
809
810 if (!PSR64_IS_USER(get_saved_state_cpsr(ss))) {
811 panic_with_thread_kernel_state("SP alignment exception from kernel.", ss);
812 }
813
814 exc = EXC_BAD_ACCESS;
815 codes[0] = EXC_ARM_SP_ALIGN;
816 codes[1] = get_saved_state_sp(ss);
817
818 exception_triage(exc, codes, numcodes);
819 assert(0); /* NOTREACHED */
820 }
821
822 static void
823 handle_wf_trap(arm_saved_state_t *ss)
824 {
825 exception_type_t exc;
826 mach_exception_data_type_t codes[2];
827 mach_msg_type_number_t numcodes = 2;
828
829 exc = EXC_BAD_INSTRUCTION;
830 codes[0] = EXC_ARM_UNDEFINED;
831 codes[1] = get_saved_state_sp(ss);
832
833 exception_triage(exc, codes, numcodes);
834 assert(0); /* NOTREACHED */
835 }
836
837
838 static void
839 handle_sw_step_debug(arm_saved_state_t *state)
840 {
841 thread_t thread = current_thread();
842 exception_type_t exc;
843 mach_exception_data_type_t codes[2];
844 mach_msg_type_number_t numcodes = 2;
845
846 if (!PSR64_IS_USER(get_saved_state_cpsr(state))) {
847 panic_with_thread_kernel_state("SW_STEP_DEBUG exception from kernel.", state);
848 }
849
850 // Disable single step and unmask interrupts (in the saved state, anticipating next exception return)
851 if (thread->machine.DebugData != NULL) {
852 thread->machine.DebugData->uds.ds64.mdscr_el1 &= ~0x1;
853 } else {
854 panic_with_thread_kernel_state("SW_STEP_DEBUG exception thread DebugData is NULL.", state);
855 }
856
857 set_saved_state_cpsr((thread->machine.upcb),
858 get_saved_state_cpsr((thread->machine.upcb)) & ~(PSR64_SS | DAIF_IRQF | DAIF_FIQF));
859
860 // Special encoding for gdb single step event on ARM
861 exc = EXC_BREAKPOINT;
862 codes[0] = 1;
863 codes[1] = 0;
864
865 exception_triage(exc, codes, numcodes);
866 assert(0); /* NOTREACHED */
867 }
868
869 static int
870 is_vm_fault(fault_status_t status)
871 {
872 switch (status) {
873 case FSC_TRANSLATION_FAULT_L0:
874 case FSC_TRANSLATION_FAULT_L1:
875 case FSC_TRANSLATION_FAULT_L2:
876 case FSC_TRANSLATION_FAULT_L3:
877 case FSC_ACCESS_FLAG_FAULT_L1:
878 case FSC_ACCESS_FLAG_FAULT_L2:
879 case FSC_ACCESS_FLAG_FAULT_L3:
880 case FSC_PERMISSION_FAULT_L1:
881 case FSC_PERMISSION_FAULT_L2:
882 case FSC_PERMISSION_FAULT_L3:
883 return TRUE;
884 default:
885 return FALSE;
886 }
887 }
888
889 #if __ARM_PAN_AVAILABLE__
890 static int
891 is_permission_fault(fault_status_t status)
892 {
893 switch (status) {
894 case FSC_PERMISSION_FAULT_L1:
895 case FSC_PERMISSION_FAULT_L2:
896 case FSC_PERMISSION_FAULT_L3:
897 return TRUE;
898 default:
899 return FALSE;
900 }
901 }
902 #endif
903
904 static int
905 is_alignment_fault(fault_status_t status)
906 {
907 return (status == FSC_ALIGNMENT_FAULT);
908 }
909
910 static int
911 is_parity_error(fault_status_t status)
912 {
913 switch (status) {
914 case FSC_SYNC_PARITY:
915 case FSC_ASYNC_PARITY:
916 case FSC_SYNC_PARITY_TT_L1:
917 case FSC_SYNC_PARITY_TT_L2:
918 case FSC_SYNC_PARITY_TT_L3:
919 return TRUE;
920 default:
921 return FALSE;
922 }
923 }
924
925 static void
926 handle_user_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr,
927 fault_status_t fault_code, vm_prot_t fault_type, vm_offset_t recover)
928 {
929 exception_type_t exc = EXC_BAD_ACCESS;
930 mach_exception_data_type_t codes[2];
931 mach_msg_type_number_t numcodes = 2;
932 thread_t thread = current_thread();
933
934 (void)esr;
935 (void)state;
936
937 if (ml_at_interrupt_context())
938 panic_with_thread_kernel_state("Apparently on interrupt stack when taking user abort!\n", state);
939
940 thread->iotier_override = THROTTLE_LEVEL_NONE; /* Reset IO tier override before handling abort from userspace */
941
942 if (is_vm_fault(fault_code)) {
943 kern_return_t result = KERN_FAILURE;
944 vm_map_t map = thread->map;
945 vm_offset_t vm_fault_addr = fault_addr;
946
947 assert(map != kernel_map);
948
949 if (!(fault_type & VM_PROT_EXECUTE) && user_tbi_enabled())
950 vm_fault_addr = tbi_clear(fault_addr);
951
952 #if CONFIG_DTRACE
953 if (thread->options & TH_OPT_DTRACE) { /* Executing under dtrace_probe? */
954 if (dtrace_tally_fault(vm_fault_addr)) { /* Should a user mode fault under dtrace be ignored? */
955 if (recover) {
956 set_saved_state_pc(state, recover);
957 } else {
958 boolean_t intr = ml_set_interrupts_enabled(FALSE);
959 panic_with_thread_kernel_state("copyin/out has no recovery point", state);
960 (void) ml_set_interrupts_enabled(intr);
961 }
962 return;
963 } else {
964 boolean_t intr = ml_set_interrupts_enabled(FALSE);
965 panic_with_thread_kernel_state("Unexpected UMW page fault under dtrace_probe", state);
966 (void) ml_set_interrupts_enabled(intr);
967 return;
968 }
969 }
970 #else
971 (void)recover;
972 #endif
973
974 #if CONFIG_PGTRACE
975 if (pgtrace_enabled) {
976 /* Check to see if trace bit is set */
977 result = pmap_pgtrace_fault(map->pmap, fault_addr, state);
978 if (result == KERN_SUCCESS) return;
979 }
980 #endif
981
982 /* check to see if it is just a pmap ref/modify fault */
983
984 if (result != KERN_SUCCESS) {
985 result = arm_fast_fault(map->pmap, trunc_page(vm_fault_addr), fault_type, TRUE);
986 }
987 if (result != KERN_SUCCESS) {
988
989 {
990 /* We have to fault the page in */
991 result = vm_fault(map, vm_fault_addr, fault_type,
992 /* change_wiring */ FALSE, VM_KERN_MEMORY_NONE, THREAD_ABORTSAFE,
993 /* caller_pmap */ NULL, /* caller_pmap_addr */ 0);
994 }
995 }
996 if (result == KERN_SUCCESS || result == KERN_ABORTED) {
997 thread_exception_return();
998 /* NOTREACHED */
999 }
1000
1001 codes[0] = result;
1002 } else if (is_alignment_fault(fault_code)) {
1003 codes[0] = EXC_ARM_DA_ALIGN;
1004 } else if (is_parity_error(fault_code)) {
1005 #if defined(APPLE_ARM64_ARCH_FAMILY)
1006 if (fault_code == FSC_SYNC_PARITY) {
1007 arm64_platform_error(state, esr, fault_addr);
1008 thread_exception_return();
1009 /* NOTREACHED */
1010 }
1011 #else
1012 panic("User parity error.");
1013 #endif
1014 } else {
1015 codes[0] = KERN_FAILURE;
1016 }
1017
1018 codes[1] = fault_addr;
1019 exception_triage(exc, codes, numcodes);
1020 assert(0); /* NOTREACHED */
1021 }
1022
1023 #if __ARM_PAN_AVAILABLE__
1024 static int
1025 is_pan_fault(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr, fault_status_t fault_code)
1026 {
1027 // PAN (Privileged Access Never) fault occurs for data read/write in EL1 to
1028 // virtual address that is readable/writeable from both EL1 and EL0
1029
1030 // To check for PAN fault, we evaluate if the following conditions are true:
1031 // 1. This is a permission fault
1032 // 2. PAN is enabled
1033 // 3. AT instruction (on which PAN has no effect) on the same faulting address
1034 // succeeds
1035
1036 vm_offset_t pa;
1037
1038 if (!(is_permission_fault(fault_code) && get_saved_state_cpsr(state) & PSR64_PAN)) {
1039 return FALSE;
1040 }
1041
1042 if (esr & ISS_DA_WNR) {
1043 pa = mmu_kvtop_wpreflight(fault_addr);
1044 } else {
1045 pa = mmu_kvtop(fault_addr);
1046 }
1047 return (pa)? TRUE: FALSE;
1048 }
1049 #endif
1050
1051 static void
1052 handle_kernel_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr,
1053 fault_status_t fault_code, vm_prot_t fault_type, vm_offset_t recover)
1054 {
1055 thread_t thread = current_thread();
1056 (void)esr;
1057
1058 #if CONFIG_DTRACE
1059 if (is_vm_fault(fault_code) && thread->options & TH_OPT_DTRACE) { /* Executing under dtrace_probe? */
1060 if (dtrace_tally_fault(fault_addr)) { /* Should a fault under dtrace be ignored? */
1061 /*
1062 * Point to next instruction, or recovery handler if set.
1063 */
1064 if (recover) {
1065 set_saved_state_pc(state, recover);
1066 } else {
1067 set_saved_state_pc(state, get_saved_state_pc(state) + 4);
1068 }
1069 return;
1070 } else {
1071 boolean_t intr = ml_set_interrupts_enabled(FALSE);
1072 panic_with_thread_kernel_state("Unexpected page fault under dtrace_probe", state);
1073 (void) ml_set_interrupts_enabled(intr);
1074 return;
1075 }
1076 }
1077 #endif
1078
1079 #if !CONFIG_PGTRACE /* This will be moved next to pgtrace fault evaluation */
1080 if (ml_at_interrupt_context())
1081 panic_with_thread_kernel_state("Unexpected abort while on interrupt stack.", state);
1082 #endif
1083
1084 if (is_vm_fault(fault_code)) {
1085 kern_return_t result;
1086 vm_map_t map;
1087 int interruptible;
1088
1089 /*
1090 * Ensure no faults in the physical aperture. This could happen if
1091 * a page table is incorrectly allocated from the read only region
1092 * when running with KTRR.
1093 */
1094
1095
1096 if (fault_addr >= gVirtBase && fault_addr < (gVirtBase+gPhysSize)) {
1097 panic_with_thread_kernel_state("Unexpected fault in kernel static region\n",state);
1098 }
1099
1100 if (VM_KERNEL_ADDRESS(fault_addr) || thread == THREAD_NULL) {
1101 map = kernel_map;
1102 interruptible = THREAD_UNINT;
1103 } else {
1104 map = thread->map;
1105 interruptible = THREAD_ABORTSAFE;
1106 }
1107
1108 #if CONFIG_PGTRACE
1109 if (pgtrace_enabled) {
1110 /* Check to see if trace bit is set */
1111 result = pmap_pgtrace_fault(map->pmap, fault_addr, state);
1112 if (result == KERN_SUCCESS) return;
1113 }
1114
1115 if (ml_at_interrupt_context())
1116 panic_with_thread_kernel_state("Unexpected abort while on interrupt stack.", state);
1117 #endif
1118
1119 /* check to see if it is just a pmap ref/modify fault */
1120 result = arm_fast_fault(map->pmap, trunc_page(fault_addr), fault_type, FALSE);
1121 if (result == KERN_SUCCESS) return;
1122
1123 {
1124 /*
1125 * We have to "fault" the page in.
1126 */
1127 result = vm_fault(map, fault_addr, fault_type,
1128 /* change_wiring */ FALSE, VM_KERN_MEMORY_NONE, interruptible,
1129 /* caller_pmap */ NULL, /* caller_pmap_addr */ 0);
1130 }
1131
1132 if (result == KERN_SUCCESS) return;
1133
1134 /*
1135 * If we have a recover handler, invoke it now.
1136 */
1137 if (recover) {
1138 set_saved_state_pc(state, recover);
1139 return;
1140 }
1141
1142 #if __ARM_PAN_AVAILABLE__
1143 if (is_pan_fault(state, esr, fault_addr, fault_code)) {
1144 panic_with_thread_kernel_state("Privileged access never abort.", state);
1145 }
1146 #endif
1147
1148 #if CONFIG_PGTRACE
1149 } else if (ml_at_interrupt_context()) {
1150 panic_with_thread_kernel_state("Unexpected abort while on interrupt stack.", state);
1151 #endif
1152 } else if (is_alignment_fault(fault_code)) {
1153 panic_with_thread_kernel_state("Unaligned kernel data abort.", state);
1154 } else if (is_parity_error(fault_code)) {
1155 #if defined(APPLE_ARM64_ARCH_FAMILY)
1156 if (fault_code == FSC_SYNC_PARITY) {
1157 arm64_platform_error(state, esr, fault_addr);
1158 return;
1159 }
1160 #else
1161 panic_with_thread_kernel_state("Kernel parity error.", state);
1162 #endif
1163 } else {
1164 kprintf("Unclassified kernel abort (fault_code=0x%x)\n", fault_code);
1165 }
1166
1167 panic_with_thread_kernel_state("Kernel data abort.", state);
1168 }
1169
1170 extern void syscall_trace(struct arm_saved_state * regs);
1171
1172 static void
1173 handle_svc(arm_saved_state_t *state)
1174 {
1175 int trap_no = get_saved_state_svc_number(state);
1176 thread_t thread = current_thread();
1177 struct proc *p;
1178
1179 #define handle_svc_kprintf(x...) /* kprintf("handle_svc: " x) */
1180
1181 #define TRACE_SYSCALL 1
1182 #if TRACE_SYSCALL
1183 syscall_trace(state);
1184 #endif
1185
1186 thread->iotier_override = THROTTLE_LEVEL_NONE; /* Reset IO tier override before handling SVC from userspace */
1187
1188 if (trap_no == (int)PLATFORM_SYSCALL_TRAP_NO) {
1189 platform_syscall(state);
1190 panic("Returned from platform_syscall()?");
1191 }
1192
1193 mach_kauth_cred_uthread_update();
1194
1195 if (trap_no < 0) {
1196 if (trap_no == -3) {
1197 handle_mach_absolute_time_trap(state);
1198 return;
1199 } else if (trap_no == -4) {
1200 handle_mach_continuous_time_trap(state);
1201 return;
1202 }
1203
1204 /* Counting perhaps better in the handler, but this is how it's been done */
1205 thread->syscalls_mach++;
1206 mach_syscall(state);
1207 } else {
1208 /* Counting perhaps better in the handler, but this is how it's been done */
1209 thread->syscalls_unix++;
1210 p = get_bsdthreadtask_info(thread);
1211
1212 assert(p);
1213
1214 unix_syscall(state, thread, (struct uthread*)thread->uthread, p);
1215 }
1216 }
1217
1218 static void
1219 handle_mach_absolute_time_trap(arm_saved_state_t *state)
1220 {
1221 uint64_t now = mach_absolute_time();
1222 saved_state64(state)->x[0] = now;
1223 }
1224
1225 static void
1226 handle_mach_continuous_time_trap(arm_saved_state_t *state)
1227 {
1228 uint64_t now = mach_continuous_time();
1229 saved_state64(state)->x[0] = now;
1230 }
1231
1232 static void
1233 handle_msr_trap(arm_saved_state_t *state, uint32_t iss)
1234 {
1235 exception_type_t exception = EXC_BAD_INSTRUCTION;
1236 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
1237 mach_msg_type_number_t numcodes = 2;
1238 uint32_t instr;
1239
1240 (void)iss;
1241
1242 if (!is_saved_state64(state)) {
1243 panic("MSR/MRS trap (EC 0x%x) from 32-bit state\n", ESR_EC_MSR_TRAP);
1244 }
1245
1246 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1247 panic("MSR/MRS trap (EC 0x%x) from kernel\n", ESR_EC_MSR_TRAP);
1248 }
1249
1250 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1251 codes[1] = instr;
1252
1253 exception_triage(exception, codes, numcodes);
1254 }
1255
1256 static void
1257 handle_user_trapped_instruction32(arm_saved_state_t *state, uint32_t esr)
1258 {
1259 exception_type_t exception = EXC_BAD_INSTRUCTION;
1260 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
1261 mach_msg_type_number_t numcodes = 2;
1262 uint32_t instr;
1263
1264 if (is_saved_state64(state)) {
1265 panic("ESR (0x%x) for instruction trapped from U32, but saved state is 64-bit.", esr);
1266 }
1267
1268 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1269 panic("ESR (0x%x) for instruction trapped from U32, actually came from kernel?", esr);
1270 }
1271
1272 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1273 codes[1] = instr;
1274
1275 exception_triage(exception, codes, numcodes);
1276 }
1277
1278 static void
1279 handle_simd_trap(arm_saved_state_t *state, uint32_t esr)
1280 {
1281 exception_type_t exception = EXC_BAD_INSTRUCTION;
1282 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
1283 mach_msg_type_number_t numcodes = 2;
1284 uint32_t instr;
1285
1286 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1287 panic("ESR (0x%x) for SIMD trap from userland, actually came from kernel?", esr);
1288 }
1289
1290 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1291 codes[1] = instr;
1292
1293 exception_triage(exception, codes, numcodes);
1294 }
1295
1296 void
1297 sleh_irq(arm_saved_state_t *state)
1298 {
1299 uint64_t timestamp = 0;
1300 uint32_t old_entropy_data = 0;
1301 uint32_t * old_entropy_data_ptr = NULL;
1302 uint32_t * new_entropy_data_ptr = NULL;
1303 cpu_data_t * cdp = getCpuDatap();
1304
1305 sleh_interrupt_handler_prologue(state, DBG_INTR_TYPE_OTHER);
1306
1307 /* Run the registered interrupt handler. */
1308 cdp->interrupt_handler(cdp->interrupt_target,
1309 cdp->interrupt_refCon,
1310 cdp->interrupt_nub,
1311 cdp->interrupt_source);
1312
1313 /* We use interrupt timing as an entropy source. */
1314 timestamp = ml_get_timebase();
1315
1316 /*
1317 * The buffer index is subject to races, but as these races should only
1318 * result in multiple CPUs updating the same location, the end result
1319 * should be that noise gets written into the entropy buffer. As this
1320 * is the entire point of the entropy buffer, we will not worry about
1321 * these races for now.
1322 */
1323 old_entropy_data_ptr = EntropyData.index_ptr;
1324 new_entropy_data_ptr = old_entropy_data_ptr + 1;
1325
1326 if (new_entropy_data_ptr >= &EntropyData.buffer[ENTROPY_BUFFER_SIZE]) {
1327 new_entropy_data_ptr = EntropyData.buffer;
1328 }
1329
1330 EntropyData.index_ptr = new_entropy_data_ptr;
1331
1332 /* Mix the timestamp data and the old data together. */
1333 old_entropy_data = *old_entropy_data_ptr;
1334 *old_entropy_data_ptr = (uint32_t)timestamp ^ __ror(old_entropy_data, 9);
1335
1336 sleh_interrupt_handler_epilogue();
1337 }
1338
1339 void
1340 sleh_fiq(arm_saved_state_t *state)
1341 {
1342 unsigned int type = DBG_INTR_TYPE_UNKNOWN;
1343 if (ml_get_timer_pending()) {
1344 type = DBG_INTR_TYPE_TIMER;
1345 }
1346
1347 sleh_interrupt_handler_prologue(state, type);
1348
1349 {
1350 /*
1351 * We don't know that this is a timer, but we don't have insight into
1352 * the other interrupts that go down this path.
1353 */
1354
1355
1356 cpu_data_t *cdp = getCpuDatap();
1357
1358 cdp->cpu_decrementer = -1; /* Large */
1359
1360 /*
1361 * ARM64_TODO: whether we're coming from userland is ignored right now.
1362 * We can easily thread it through, but not bothering for the
1363 * moment (AArch32 doesn't either).
1364 */
1365 rtclock_intr(TRUE);
1366 }
1367
1368 sleh_interrupt_handler_epilogue();
1369 }
1370
1371 void
1372 sleh_serror(arm_context_t *context, uint32_t esr, vm_offset_t far)
1373 {
1374 arm_saved_state_t *state = &context->ss;
1375
1376 ASSERT_CONTEXT_SANITY(context);
1377 arm64_platform_error(state, esr, far);
1378 }
1379
1380 void
1381 mach_syscall_trace_exit(
1382 unsigned int retval,
1383 unsigned int call_number)
1384 {
1385 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
1386 MACHDBG_CODE(DBG_MACH_EXCP_SC, (call_number)) | DBG_FUNC_END,
1387 retval, 0, 0, 0, 0);
1388 }
1389
1390 __attribute__((noreturn))
1391 void
1392 thread_syscall_return(kern_return_t error)
1393 {
1394 thread_t thread;
1395 struct arm_saved_state *state;
1396
1397 thread = current_thread();
1398 state = get_user_regs(thread);
1399
1400 assert(is_saved_state64(state));
1401 saved_state64(state)->x[0] = error;
1402
1403 #if DEBUG || DEVELOPMENT
1404 kern_allocation_name_t
1405 prior __assert_only = thread_get_kernel_state(thread)->allocation_name;
1406 assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior));
1407 #endif /* DEBUG || DEVELOPMENT */
1408
1409 if (kdebug_enable) {
1410 /* Invert syscall number (negative for a mach syscall) */
1411 mach_syscall_trace_exit(error, (-1) * get_saved_state_svc_number(state));
1412 }
1413
1414 thread_exception_return();
1415 }
1416
1417 void
1418 syscall_trace(
1419 struct arm_saved_state * regs __unused)
1420 {
1421 /* kprintf("syscall: %d\n", saved_state64(regs)->x[16]); */
1422 }
1423
1424 static void
1425 sleh_interrupt_handler_prologue(arm_saved_state_t *state, unsigned int type)
1426 {
1427 uint64_t is_user = PSR64_IS_USER(get_saved_state_cpsr(state));
1428
1429 uint64_t pc = is_user ? get_saved_state_pc(state) :
1430 VM_KERNEL_UNSLIDE(get_saved_state_pc(state));
1431
1432 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_START,
1433 0, pc, is_user, type);
1434
1435 #if CONFIG_TELEMETRY
1436 if (telemetry_needs_record) {
1437 telemetry_mark_curthread((boolean_t)is_user);
1438 }
1439 #endif /* CONFIG_TELEMETRY */
1440 }
1441
1442 static void
1443 sleh_interrupt_handler_epilogue(void)
1444 {
1445 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_END);
1446 }
1447
1448 void
1449 sleh_invalid_stack(arm_context_t *context, uint32_t esr __unused, vm_offset_t far __unused)
1450 {
1451 thread_t thread = current_thread();
1452 vm_offset_t kernel_stack_bottom, sp;
1453
1454 sp = get_saved_state_sp(&context->ss);
1455 kernel_stack_bottom = round_page(thread->machine.kstackptr) - KERNEL_STACK_SIZE;
1456
1457 if ((sp < kernel_stack_bottom) && (sp >= (kernel_stack_bottom - PAGE_SIZE))) {
1458 panic_with_thread_kernel_state("Invalid kernel stack pointer (probable overflow).", &context->ss);
1459 }
1460
1461 panic_with_thread_kernel_state("Invalid kernel stack pointer (probable corruption).", &context->ss);
1462 }
1463
mirror of XNU