]> git.saurik.com Git - apple/xnu.git/blob - bsd/dev/arm64/fbt_arm.c
projects / apple / xnu.git / blob
? search:


083f98665aff2eea07e0ec934a8f7814cd94521f
[apple/xnu.git] / bsd / dev / arm64 / fbt_arm.c
1 /*
2 * Copyright (c) 2007 Apple Inc. All rights reserved.
3 */
4 /*
5 * CDDL HEADER START
6 *
7 * The contents of this file are subject to the terms of the
8 * Common Development and Distribution License, Version 1.0 only
9 * (the "License"). You may not use this file except in compliance
10 * with the License.
11 *
12 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
13 * or http://www.opensolaris.org/os/licensing.
14 * See the License for the specific language governing permissions
15 * and limitations under the License.
16 *
17 * When distributing Covered Code, include this CDDL HEADER in each
18 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
19 * If applicable, add the following below this CDDL HEADER, with the
20 * fields enclosed by brackets "[]" replaced with your own identifying
21 * information: Portions Copyright [yyyy] [name of copyright owner]
22 *
23 * CDDL HEADER END
24 */
25 /*
26 * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
27 * Use is subject to license terms.
28 */
29
30 /* #pragma ident "@(#)fbt.c 1.15 05/09/19 SMI" */
31
32 #ifdef KERNEL
33 #ifndef _KERNEL
34 #define _KERNEL /* Solaris vs. Darwin */
35 #endif
36 #endif
37
38 #define MACH__POSIX_C_SOURCE_PRIVATE 1 /* pulls in suitable savearea from
39 * mach/ppc/thread_status.h */
40 #include <kern/thread.h>
41 #include <mach/thread_status.h>
42 #include <arm/proc_reg.h>
43 #include <arm/caches_internal.h>
44
45 #include <mach-o/loader.h>
46 #include <mach-o/nlist.h>
47 #include <libkern/kernel_mach_header.h>
48
49 #include <sys/param.h>
50 #include <sys/systm.h>
51 #include <sys/errno.h>
52 #include <sys/stat.h>
53 #include <sys/ioctl.h>
54 #include <sys/conf.h>
55 #include <sys/fcntl.h>
56 #include <miscfs/devfs/devfs.h>
57
58 #include <sys/dtrace.h>
59 #include <sys/dtrace_impl.h>
60 #include <sys/fbt.h>
61
62 #include <sys/dtrace_glue.h>
63
64 #if __has_include(<ptrauth.h>)
65 #include <ptrauth.h>
66 #endif
67
68 #define DTRACE_INVOP_PUSH_FRAME 11
69
70 #define DTRACE_INVOP_NOP_SKIP 4
71 #define DTRACE_INVOP_ADD_FP_SP_SKIP 4
72
73 #define DTRACE_INVOP_POP_PC_SKIP 2
74
75 /*
76 * stp fp, lr, [sp, #val]
77 * stp fp, lr, [sp, #val]!
78 */
79 #define FBT_IS_ARM64_FRAME_PUSH(x) \
80 (((x) & 0xffc07fff) == 0xa9007bfd || ((x) & 0xffc07fff) == 0xa9807bfd)
81
82 /*
83 * stp Xt1, Xt2, [sp, #val]
84 * stp Xt1, Xt2, [sp, #val]!
85 */
86 #define FBT_IS_ARM64_PUSH(x) \
87 (((x) & 0xffc003e0) == 0xa90003e0 || ((x) & 0xffc003e0) == 0xa98003e0)
88
89 /*
90 * ldp fp, lr, [sp, #val]
91 * ldp fp, lr, [sp], #val
92 */
93 #define FBT_IS_ARM64_FRAME_POP(x) \
94 (((x) & 0xffc07fff) == 0xa9407bfd || ((x) & 0xffc07fff) == 0xa8c07bfd)
95
96 #define FBT_IS_ARM64_ADD_FP_SP(x) (((x) & 0xffc003ff) == 0x910003fd) /* add fp, sp, #val (add fp, sp, #0 == mov fp, sp) */
97 #define FBT_IS_ARM64_RET(x) (((x) == 0xd65f03c0) || ((x) == 0xd65f0fff)) /* ret, retab */
98
99
100 #define FBT_B_MASK 0xff000000
101 #define FBT_B_IMM_MASK 0x00ffffff
102 #define FBT_B_INSTR 0x14000000
103
104 #define FBT_IS_ARM64_B_INSTR(x) ((x & FBT_B_MASK) == FBT_B_INSTR)
105 #define FBT_GET_ARM64_B_IMM(x) ((x & FBT_B_IMM_MASK) << 2)
106
107 #define FBT_PATCHVAL 0xe7eeee7e
108 #define FBT_AFRAMES_ENTRY 7
109 #define FBT_AFRAMES_RETURN 7
110
111 #define FBT_ENTRY "entry"
112 #define FBT_RETURN "return"
113 #define FBT_ADDR2NDX(addr) ((((uintptr_t)(addr)) >> 4) & fbt_probetab_mask)
114
115 extern dtrace_provider_id_t fbt_id;
116 extern fbt_probe_t **fbt_probetab;
117 extern int fbt_probetab_mask;
118
119 kern_return_t fbt_perfCallback(int, struct arm_saved_state *, __unused int, __unused int);
120
121 int
122 fbt_invop(uintptr_t addr, uintptr_t * stack, uintptr_t rval)
123 {
124 fbt_probe_t *fbt = fbt_probetab[FBT_ADDR2NDX(addr)];
125
126 for (; fbt != NULL; fbt = fbt->fbtp_hashnext) {
127 if ((uintptr_t) fbt->fbtp_patchpoint == addr) {
128 if (0 == CPU->cpu_dtrace_invop_underway) {
129 CPU->cpu_dtrace_invop_underway = 1; /* Race not possible on
130 * this per-cpu state */
131
132 if (fbt->fbtp_roffset == 0) {
133 /*
134 * Stack looks like this:
135 *
136 * [Higher addresses]
137 *
138 * Frame of caller
139 * Extra args for callee
140 * ------------------------
141 * Frame from traced function: <previous sp (e.g. 0x1000), return address>
142 * ------------------------
143 * arm_context_t
144 * ------------------------
145 * Frame from trap handler: <previous sp (e.g. 0x1000) , traced PC >
146 * The traced function never got to mov fp, sp,
147 * so there is no frame in the backtrace pointing
148 * to the frame on the stack containing the LR in the
149 * caller.
150 * ------------------------
151 * |
152 * |
153 * | stack grows this way
154 * |
155 * |
156 * v
157 * [Lower addresses]
158 */
159
160 arm_saved_state_t *regs = (arm_saved_state_t *)(&((arm_context_t *)stack)->ss);
161
162 /*
163 * cpu_dtrace_caller compensates for fact that the traced function never got to update its fp.
164 * When walking the stack, when we reach the frame where we extract a PC in the patched
165 * function, we put the cpu_dtrace_caller in the backtrace instead. The next frame we extract
166 * will be in the caller's caller, so we output a backtrace starting at the caller and going
167 * sequentially up the stack.
168 */
169 CPU->cpu_dtrace_caller = get_saved_state_lr(regs);
170 dtrace_probe(fbt->fbtp_id, get_saved_state_reg(regs, 0), get_saved_state_reg(regs, 1),
171 get_saved_state_reg(regs, 2), get_saved_state_reg(regs, 3), get_saved_state_reg(regs, 4));
172 CPU->cpu_dtrace_caller = 0;
173 } else {
174 /*
175 * When fbtp_roffset is non-zero, we know we are handling a return probe point.
176 *
177 *
178 * Stack looks like this, as we've already popped the frame in the traced callee, and
179 * we trap with lr set to the return address in the caller.
180 * [Higher addresses]
181 *
182 * Frame of caller
183 * Extra args for callee
184 * ------------------------
185 * arm_context_t
186 * ------------------------
187 * Frame from trap handler: <sp at time of trap, traced PC >
188 * ------------------------
189 * |
190 * |
191 * | stack grows this way
192 * |
193 * |
194 * v
195 * [Lower addresses]
196 */
197 arm_saved_state_t *regs = (arm_saved_state_t *)(&((arm_context_t *)stack)->ss);
198
199 CPU->cpu_dtrace_caller = get_saved_state_lr(regs);
200 dtrace_probe(fbt->fbtp_id, fbt->fbtp_roffset, rval, 0, 0, 0);
201 CPU->cpu_dtrace_caller = 0;
202 }
203 CPU->cpu_dtrace_invop_underway = 0;
204 }
205
206 /*
207 * On other architectures, we return a DTRACE constant to let the callback function
208 * know what was replaced. On the ARM, since the function prologue/epilogue machine code
209 * can vary, we need the actual bytes of the instruction, so return the savedval instead.
210 */
211 return fbt->fbtp_savedval;
212 }
213 }
214
215 return 0;
216 }
217
218 #define IS_USER_TRAP(regs) (PSR64_IS_USER(get_saved_state_cpsr(regs)))
219 #define T_INVALID_OPCODE EXC_BAD_INSTRUCTION
220 #define FBT_EXCEPTION_CODE T_INVALID_OPCODE
221
222 kern_return_t
223 fbt_perfCallback(
224 int trapno,
225 struct arm_saved_state * regs,
226 __unused int unused1,
227 __unused int unused2)
228 {
229 kern_return_t retval = KERN_FAILURE;
230
231 if (FBT_EXCEPTION_CODE == trapno && !IS_USER_TRAP(regs)) {
232 boolean_t oldlevel = 0;
233 machine_inst_t emul = 0;
234 uint64_t sp, pc, lr, imm;
235
236 oldlevel = ml_set_interrupts_enabled(FALSE);
237
238 __asm__ volatile (
239 "Ldtrace_invop_callsite_pre_label:\n"
240 ".data\n"
241 ".private_extern _dtrace_invop_callsite_pre\n"
242 "_dtrace_invop_callsite_pre:\n"
243 " .quad Ldtrace_invop_callsite_pre_label\n"
244 ".text\n"
245 );
246
247 emul = dtrace_invop(get_saved_state_pc(regs), (uintptr_t*) regs, get_saved_state_reg(regs, 0));
248
249 __asm__ volatile (
250 "Ldtrace_invop_callsite_post_label:\n"
251 ".data\n"
252 ".private_extern _dtrace_invop_callsite_post\n"
253 "_dtrace_invop_callsite_post:\n"
254 " .quad Ldtrace_invop_callsite_post_label\n"
255 ".text\n"
256 );
257
258 if (emul == DTRACE_INVOP_NOP) {
259 /*
260 * Skip over the patched NOP planted by sdt
261 */
262 pc = get_saved_state_pc(regs);
263 set_saved_state_pc(regs, pc + DTRACE_INVOP_NOP_SKIP);
264 retval = KERN_SUCCESS;
265 } else if (FBT_IS_ARM64_ADD_FP_SP(emul)) {
266 /* retrieve the value to add */
267 uint64_t val = (emul >> 10) & 0xfff;
268 assert(val < 4096);
269
270 /* retrieve sp */
271 sp = get_saved_state_sp(regs);
272
273 /*
274 * emulate the instruction:
275 * add fp, sp, #val
276 */
277 assert(sp < (UINT64_MAX - val));
278 set_saved_state_fp(regs, sp + val);
279
280 /* skip over the bytes of the patched instruction */
281 pc = get_saved_state_pc(regs);
282 set_saved_state_pc(regs, pc + DTRACE_INVOP_ADD_FP_SP_SKIP);
283
284 retval = KERN_SUCCESS;
285 } else if (FBT_IS_ARM64_RET(emul)) {
286 lr = get_saved_state_lr(regs);
287 #if __has_feature(ptrauth_calls)
288 lr = (user_addr_t) ptrauth_strip((void *)lr, ptrauth_key_return_address);
289 #endif
290 set_saved_state_pc(regs, lr);
291 retval = KERN_SUCCESS;
292 } else if (FBT_IS_ARM64_B_INSTR(emul)) {
293 pc = get_saved_state_pc(regs);
294 imm = FBT_GET_ARM64_B_IMM(emul);
295 set_saved_state_pc(regs, pc + imm);
296 retval = KERN_SUCCESS;
297 } else if (emul == FBT_PATCHVAL) {
298 /* Means we encountered an error but handled it, try same inst again */
299 retval = KERN_SUCCESS;
300 } else {
301 retval = KERN_FAILURE;
302 }
303
304 ml_set_interrupts_enabled(oldlevel);
305 }
306
307 return retval;
308 }
309
310 void
311 fbt_provide_probe(struct modctl *ctl, const char *modname, const char* symbolName, machine_inst_t* symbolStart, machine_inst_t *instrHigh)
312 {
313 int doenable = 0;
314 dtrace_id_t thisid;
315
316 fbt_probe_t *newfbt, *retfbt, *entryfbt;
317 machine_inst_t *instr, *pushinstr = NULL, *limit, theInstr;
318 int foundPushLR, savedRegs;
319
320 /*
321 * Guard against null and invalid symbols
322 */
323 if (!symbolStart || !instrHigh || instrHigh < symbolStart) {
324 kprintf("dtrace: %s has an invalid address\n", symbolName);
325 return;
326 }
327
328 /*
329 * Assume the compiler doesn't schedule instructions in the prologue.
330 */
331 foundPushLR = 0;
332 savedRegs = -1;
333 limit = (machine_inst_t *)instrHigh;
334
335 assert(sizeof(*instr) == 4);
336
337 for (instr = symbolStart, theInstr = 0; instr < instrHigh; instr++) {
338 /*
339 * Count the number of time we pushed something onto the stack
340 * before hitting a frame push. That will give us an estimation
341 * of how many stack pops we should expect when looking for the
342 * RET instruction.
343 */
344 theInstr = *instr;
345 if (FBT_IS_ARM64_FRAME_PUSH(theInstr)) {
346 foundPushLR = 1;
347 pushinstr = instr;
348 }
349
350 if (foundPushLR && (FBT_IS_ARM64_ADD_FP_SP(theInstr))) {
351 /* Guard against a random setting of fp from sp, we make sure we found the push first */
352 break;
353 }
354 if (FBT_IS_ARM64_RET(theInstr)) { /* We've gone too far, bail. */
355 break;
356 }
357 if (FBT_IS_ARM64_FRAME_POP(theInstr)) { /* We've gone too far, bail. */
358 break;
359 }
360 }
361
362 if (!(foundPushLR && (FBT_IS_ARM64_ADD_FP_SP(theInstr)))) {
363 return;
364 }
365
366 thisid = dtrace_probe_lookup(fbt_id, modname, symbolName, FBT_ENTRY);
367 newfbt = kmem_zalloc(sizeof(fbt_probe_t), KM_SLEEP);
368 newfbt->fbtp_next = NULL;
369 strlcpy((char *)&(newfbt->fbtp_name), symbolName, MAX_FBTP_NAME_CHARS );
370
371 if (thisid != 0) {
372 /*
373 * The dtrace_probe previously existed, so we have to hook
374 * the newfbt entry onto the end of the existing fbt's
375 * chain.
376 * If we find an fbt entry that was previously patched to
377 * fire, (as indicated by the current patched value), then
378 * we want to enable this newfbt on the spot.
379 */
380 entryfbt = dtrace_probe_arg(fbt_id, thisid);
381 ASSERT(entryfbt != NULL);
382 for (; entryfbt != NULL; entryfbt = entryfbt->fbtp_next) {
383 if (entryfbt->fbtp_currentval == entryfbt->fbtp_patchval) {
384 doenable++;
385 }
386
387 if (entryfbt->fbtp_next == NULL) {
388 entryfbt->fbtp_next = newfbt;
389 newfbt->fbtp_id = entryfbt->fbtp_id;
390 break;
391 }
392 }
393 } else {
394 /*
395 * The dtrace_probe did not previously exist, so we
396 * create it and hook in the newfbt. Since the probe is
397 * new, we obviously do not need to enable it on the spot.
398 */
399 newfbt->fbtp_id = dtrace_probe_create(fbt_id, modname, symbolName, FBT_ENTRY, FBT_AFRAMES_ENTRY, newfbt);
400 doenable = 0;
401 }
402
403 newfbt->fbtp_patchpoint = instr;
404 newfbt->fbtp_ctl = ctl;
405 newfbt->fbtp_loadcnt = ctl->mod_loadcnt;
406 newfbt->fbtp_rval = DTRACE_INVOP_PUSH_FRAME;
407 newfbt->fbtp_savedval = theInstr;
408 newfbt->fbtp_patchval = FBT_PATCHVAL;
409 newfbt->fbtp_currentval = 0;
410 newfbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
411 fbt_probetab[FBT_ADDR2NDX(instr)] = newfbt;
412
413 if (doenable) {
414 fbt_enable(NULL, newfbt->fbtp_id, newfbt);
415 }
416
417 /*
418 * The fbt entry chain is in place, one entry point per symbol.
419 * The fbt return chain can have multiple return points per
420 * symbol.
421 * Here we find the end of the fbt return chain.
422 */
423
424 doenable = 0;
425
426 thisid = dtrace_probe_lookup(fbt_id, modname, symbolName, FBT_RETURN);
427
428 if (thisid != 0) {
429 /* The dtrace_probe previously existed, so we have to
430 * find the end of the existing fbt chain. If we find
431 * an fbt return that was previously patched to fire,
432 * (as indicated by the currrent patched value), then
433 * we want to enable any new fbts on the spot.
434 */
435 retfbt = dtrace_probe_arg(fbt_id, thisid);
436 ASSERT(retfbt != NULL);
437 for (; retfbt != NULL; retfbt = retfbt->fbtp_next) {
438 if (retfbt->fbtp_currentval == retfbt->fbtp_patchval) {
439 doenable++;
440 }
441 if (retfbt->fbtp_next == NULL) {
442 break;
443 }
444 }
445 } else {
446 doenable = 0;
447 retfbt = NULL;
448 }
449
450 /*
451 * Go back to the start of the function, in case
452 * the compiler emitted pcrel data loads
453 * before FP was adjusted.
454 */
455 instr = pushinstr + 1;
456 again:
457 if (instr >= limit) {
458 return;
459 }
460
461 /* XXX FIXME ... extra jump table detection? */
462
463 /*
464 * OK, it's an instruction.
465 */
466 theInstr = *instr;
467
468 /* Walked onto the start of the next routine? If so, bail out from this function */
469 if (FBT_IS_ARM64_FRAME_PUSH(theInstr)) {
470 if (!retfbt) {
471 kprintf("dtrace: fbt: No return probe for %s, walked to next routine at 0x%016llx\n", symbolName, (uint64_t)instr);
472 }
473 return;
474 }
475
476 /* XXX fancy detection of end of function using PC-relative loads */
477
478 /*
479 * Look for:
480 * ldp fp, lr, [sp], #val
481 * ldp fp, lr, [sp, #val]
482 */
483 if (!FBT_IS_ARM64_FRAME_POP(theInstr)) {
484 instr++;
485 goto again;
486 }
487
488 /* go to the next instruction */
489 instr++;
490
491 /* Scan ahead for a ret or a branch outside the function */
492 for (; instr < limit; instr++) {
493 theInstr = *instr;
494 if (FBT_IS_ARM64_RET(theInstr)) {
495 break;
496 }
497 if (FBT_IS_ARM64_B_INSTR(theInstr)) {
498 machine_inst_t *dest = instr + FBT_GET_ARM64_B_IMM(theInstr);
499 /*
500 * Check whether the destination of the branch
501 * is outside of the function
502 */
503 if (dest >= limit || dest < symbolStart) {
504 break;
505 }
506 }
507 }
508
509 if (!FBT_IS_ARM64_RET(theInstr) && !FBT_IS_ARM64_B_INSTR(theInstr)) {
510 return;
511 }
512
513 newfbt = kmem_zalloc(sizeof(fbt_probe_t), KM_SLEEP);
514 newfbt->fbtp_next = NULL;
515 strlcpy((char *)&(newfbt->fbtp_name), symbolName, MAX_FBTP_NAME_CHARS );
516
517 if (retfbt == NULL) {
518 newfbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
519 symbolName, FBT_RETURN, FBT_AFRAMES_RETURN, newfbt);
520 } else {
521 retfbt->fbtp_next = newfbt;
522 newfbt->fbtp_id = retfbt->fbtp_id;
523 }
524
525 retfbt = newfbt;
526 newfbt->fbtp_patchpoint = instr;
527 newfbt->fbtp_ctl = ctl;
528 newfbt->fbtp_loadcnt = ctl->mod_loadcnt;
529
530 ASSERT(FBT_IS_ARM64_RET(theInstr));
531 newfbt->fbtp_rval = DTRACE_INVOP_RET;
532 newfbt->fbtp_roffset = (uintptr_t) ((uint8_t*) instr - (uint8_t *)symbolStart);
533 newfbt->fbtp_savedval = theInstr;
534 newfbt->fbtp_patchval = FBT_PATCHVAL;
535 newfbt->fbtp_currentval = 0;
536 newfbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
537 fbt_probetab[FBT_ADDR2NDX(instr)] = newfbt;
538
539 if (doenable) {
540 fbt_enable(NULL, newfbt->fbtp_id, newfbt);
541 }
542
543 instr++;
544 goto again;
545 }
mirror of XNU