]> git.saurik.com Git - apple/xnu.git/blob - bsd/dev/arm64/fbt_arm.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-4570.1.46.tar.gz
[apple/xnu.git] / bsd / dev / arm64 / fbt_arm.c
1 /*
2 * Copyright (c) 2007 Apple Inc. All rights reserved.
3 */
4 /*
5 * CDDL HEADER START
6 *
7 * The contents of this file are subject to the terms of the
8 * Common Development and Distribution License, Version 1.0 only
9 * (the "License"). You may not use this file except in compliance
10 * with the License.
11 *
12 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
13 * or http://www.opensolaris.org/os/licensing.
14 * See the License for the specific language governing permissions
15 * and limitations under the License.
16 *
17 * When distributing Covered Code, include this CDDL HEADER in each
18 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
19 * If applicable, add the following below this CDDL HEADER, with the
20 * fields enclosed by brackets "[]" replaced with your own identifying
21 * information: Portions Copyright [yyyy] [name of copyright owner]
22 *
23 * CDDL HEADER END
24 */
25 /*
26 * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
27 * Use is subject to license terms.
28 */
29
30 /* #pragma ident "@(#)fbt.c 1.15 05/09/19 SMI" */
31
32 #ifdef KERNEL
33 #ifndef _KERNEL
34 #define _KERNEL /* Solaris vs. Darwin */
35 #endif
36 #endif
37
38 #define MACH__POSIX_C_SOURCE_PRIVATE 1 /* pulls in suitable savearea from
39 * mach/ppc/thread_status.h */
40 #include <kern/thread.h>
41 #include <mach/thread_status.h>
42 #include <arm/proc_reg.h>
43 #include <arm/caches_internal.h>
44
45 #include <mach-o/loader.h>
46 #include <mach-o/nlist.h>
47 #include <libkern/kernel_mach_header.h>
48
49 #include <sys/param.h>
50 #include <sys/systm.h>
51 #include <sys/errno.h>
52 #include <sys/stat.h>
53 #include <sys/ioctl.h>
54 #include <sys/conf.h>
55 #include <sys/fcntl.h>
56 #include <miscfs/devfs/devfs.h>
57
58 #include <sys/dtrace.h>
59 #include <sys/dtrace_impl.h>
60 #include <sys/fbt.h>
61
62 #include <sys/dtrace_glue.h>
63
64 #define DTRACE_INVOP_PUSH_FRAME 11
65
66 #define DTRACE_INVOP_NOP_SKIP 4
67 #define DTRACE_INVOP_ADD_FP_SP_SKIP 4
68
69 #define DTRACE_INVOP_POP_PC_SKIP 2
70
71 /*
72 * stp fp, lr, [sp, #val]
73 * stp fp, lr, [sp, #val]!
74 */
75 #define FBT_IS_ARM64_FRAME_PUSH(x) \
76 (((x) & 0xffc07fff) == 0xa9007bfd || ((x) & 0xffc07fff) == 0xa9807bfd)
77
78 /*
79 * stp Xt1, Xt2, [sp, #val]
80 * stp Xt1, Xt2, [sp, #val]!
81 */
82 #define FBT_IS_ARM64_PUSH(x) \
83 (((x) & 0xffc003e0) == 0xa90003e0 || ((x) & 0xffc003e0) == 0xa98003e0)
84
85 /*
86 * ldp fp, lr, [sp, #val]
87 * ldp fp, lr, [sp], #val
88 */
89 #define FBT_IS_ARM64_FRAME_POP(x) \
90 (((x) & 0xffc07fff) == 0xa9407bfd || ((x) & 0xffc07fff) == 0xa8c07bfd)
91
92 #define FBT_IS_ARM64_ADD_FP_SP(x) (((x) & 0xffc003ff) == 0x910003fd) /* add fp, sp, #val (add fp, sp, #0 == mov fp, sp) */
93 #define FBT_IS_ARM64_RET(x) ((x) == 0xd65f03c0) /* ret */
94
95
96 #define FBT_B_MASK 0xff000000
97 #define FBT_B_IMM_MASK 0x00ffffff
98 #define FBT_B_INSTR 0x14000000
99
100 #define FBT_IS_ARM64_B_INSTR(x) ((x & FBT_B_MASK) == FBT_B_INSTR)
101 #define FBT_GET_ARM64_B_IMM(x) ((x & FBT_B_IMM_MASK) << 2)
102
103 #define FBT_PATCHVAL 0xe7eeee7e
104 #define FBT_AFRAMES_ENTRY 7
105 #define FBT_AFRAMES_RETURN 7
106
107 #define FBT_ENTRY "entry"
108 #define FBT_RETURN "return"
109 #define FBT_ADDR2NDX(addr) ((((uintptr_t)(addr)) >> 4) & fbt_probetab_mask)
110
111 extern dtrace_provider_id_t fbt_id;
112 extern fbt_probe_t **fbt_probetab;
113 extern int fbt_probetab_mask;
114
115 kern_return_t fbt_perfCallback(int, struct arm_saved_state *, __unused int, __unused int);
116
117 int
118 fbt_invop(uintptr_t addr, uintptr_t * stack, uintptr_t rval)
119 {
120 fbt_probe_t *fbt = fbt_probetab[FBT_ADDR2NDX(addr)];
121
122 for (; fbt != NULL; fbt = fbt->fbtp_hashnext) {
123 if ((uintptr_t) fbt->fbtp_patchpoint == addr) {
124 if (0 == CPU->cpu_dtrace_invop_underway) {
125 CPU->cpu_dtrace_invop_underway = 1; /* Race not possible on
126 * this per-cpu state */
127
128 if (fbt->fbtp_roffset == 0) {
129 /*
130 * Stack looks like this:
131 *
132 * [Higher addresses]
133 *
134 * Frame of caller
135 * Extra args for callee
136 * ------------------------
137 * Frame from traced function: <previous sp (e.g. 0x1000), return address>
138 * ------------------------
139 * arm_context_t
140 * ------------------------
141 * Frame from trap handler: <previous sp (e.g. 0x1000) , traced PC >
142 * The traced function never got to mov fp, sp,
143 * so there is no frame in the backtrace pointing
144 * to the frame on the stack containing the LR in the
145 * caller.
146 * ------------------------
147 * |
148 * |
149 * | stack grows this way
150 * |
151 * |
152 * v
153 * [Lower addresses]
154 */
155
156 arm_saved_state_t *regs = (arm_saved_state_t *)(&((arm_context_t *)stack)->ss);
157
158 /*
159 * cpu_dtrace_caller compensates for fact that the traced function never got to update its fp.
160 * When walking the stack, when we reach the frame where we extract a PC in the patched
161 * function, we put the cpu_dtrace_caller in the backtrace instead. The next frame we extract
162 * will be in the caller's caller, so we output a backtrace starting at the caller and going
163 * sequentially up the stack.
164 */
165 CPU->cpu_dtrace_caller = get_saved_state_lr(regs);
166 dtrace_probe(fbt->fbtp_id, get_saved_state_reg(regs, 0), get_saved_state_reg(regs, 1),
167 get_saved_state_reg(regs, 2), get_saved_state_reg(regs, 3),get_saved_state_reg(regs, 4));
168 CPU->cpu_dtrace_caller = 0;
169 } else {
170 /*
171 * When fbtp_roffset is non-zero, we know we are handling a return probe point.
172 *
173 *
174 * Stack looks like this, as we've already popped the frame in the traced callee, and
175 * we trap with lr set to the return address in the caller.
176 * [Higher addresses]
177 *
178 * Frame of caller
179 * Extra args for callee
180 * ------------------------
181 * arm_context_t
182 * ------------------------
183 * Frame from trap handler: <sp at time of trap, traced PC >
184 * ------------------------
185 * |
186 * |
187 * | stack grows this way
188 * |
189 * |
190 * v
191 * [Lower addresses]
192 */
193 arm_saved_state_t *regs = (arm_saved_state_t *)(&((arm_context_t *)stack)->ss);
194
195 CPU->cpu_dtrace_caller = get_saved_state_lr(regs);
196 dtrace_probe(fbt->fbtp_id, fbt->fbtp_roffset, rval, 0, 0, 0);
197 CPU->cpu_dtrace_caller = 0;
198 }
199 CPU->cpu_dtrace_invop_underway = 0;
200 }
201
202 /*
203 On other architectures, we return a DTRACE constant to let the callback function
204 know what was replaced. On the ARM, since the function prologue/epilogue machine code
205 can vary, we need the actual bytes of the instruction, so return the savedval instead.
206 */
207 return (fbt->fbtp_savedval);
208 }
209 }
210
211 return (0);
212 }
213
214 #define IS_USER_TRAP(regs) (PSR64_IS_USER(get_saved_state_cpsr(regs)))
215 #define T_INVALID_OPCODE EXC_BAD_INSTRUCTION
216 #define FBT_EXCEPTION_CODE T_INVALID_OPCODE
217
218 kern_return_t
219 fbt_perfCallback(
220 int trapno,
221 struct arm_saved_state * regs,
222 __unused int unused1,
223 __unused int unused2)
224 {
225 kern_return_t retval = KERN_FAILURE;
226
227 if (FBT_EXCEPTION_CODE == trapno && !IS_USER_TRAP(regs)) {
228 boolean_t oldlevel = 0;
229 machine_inst_t emul = 0;
230 uint64_t sp, pc, lr, imm;
231
232 oldlevel = ml_set_interrupts_enabled(FALSE);
233
234 __asm__ volatile(
235 "Ldtrace_invop_callsite_pre_label:\n"
236 ".data\n"
237 ".private_extern _dtrace_invop_callsite_pre\n"
238 "_dtrace_invop_callsite_pre:\n"
239 " .quad Ldtrace_invop_callsite_pre_label\n"
240 ".text\n"
241 );
242
243 emul = dtrace_invop(get_saved_state_pc(regs), (uintptr_t*) regs, get_saved_state_reg(regs,0));
244
245 __asm__ volatile(
246 "Ldtrace_invop_callsite_post_label:\n"
247 ".data\n"
248 ".private_extern _dtrace_invop_callsite_post\n"
249 "_dtrace_invop_callsite_post:\n"
250 " .quad Ldtrace_invop_callsite_post_label\n"
251 ".text\n"
252 );
253
254 if (emul == DTRACE_INVOP_NOP) {
255 /*
256 * Skip over the patched NOP planted by sdt
257 */
258 pc = get_saved_state_pc(regs);
259 set_saved_state_pc(regs, pc + DTRACE_INVOP_NOP_SKIP);
260 retval = KERN_SUCCESS;
261 } else if (FBT_IS_ARM64_ADD_FP_SP(emul)) {
262 /* retrieve the value to add */
263 uint64_t val = (emul >> 10) & 0xfff;
264 assert(val < 4096);
265
266 /* retrieve sp */
267 sp = get_saved_state_sp(regs);
268
269 /*
270 * emulate the instruction:
271 * add fp, sp, #val
272 */
273 assert(sp < (UINT64_MAX - val));
274 set_saved_state_fp(regs, sp + val);
275
276 /* skip over the bytes of the patched instruction */
277 pc = get_saved_state_pc(regs);
278 set_saved_state_pc(regs, pc + DTRACE_INVOP_ADD_FP_SP_SKIP);
279
280 retval = KERN_SUCCESS;
281 } else if (FBT_IS_ARM64_RET(emul)) {
282 lr = get_saved_state_lr(regs);
283 set_saved_state_pc(regs, lr);
284 retval = KERN_SUCCESS;
285 } else if (FBT_IS_ARM64_B_INSTR(emul)) {
286 pc = get_saved_state_pc(regs);
287 imm = FBT_GET_ARM64_B_IMM(emul);
288 set_saved_state_pc(regs, pc + imm);
289 retval = KERN_SUCCESS;
290 } else if (emul == FBT_PATCHVAL) {
291 /* Means we encountered an error but handled it, try same inst again */
292 retval = KERN_SUCCESS;
293 } else {
294 retval = KERN_FAILURE;
295 }
296
297 ml_set_interrupts_enabled(oldlevel);
298 }
299
300 return retval;
301 }
302
303 void
304 fbt_provide_probe(struct modctl *ctl, uintptr_t instrLow, uintptr_t instrHigh, char *modname, char* symbolName, machine_inst_t* symbolStart)
305 {
306 unsigned int j;
307 int doenable = 0;
308 dtrace_id_t thisid;
309
310 fbt_probe_t *newfbt, *retfbt, *entryfbt;
311 machine_inst_t *instr, *pushinstr = NULL, *limit, theInstr;
312 int foundPushLR, savedRegs;
313
314 /*
315 * Guard against null symbols
316 */
317 if (!symbolStart || !instrLow || !instrHigh) {
318 kprintf("dtrace: %s has an invalid address\n", symbolName);
319 return;
320 }
321
322 /*
323 * Assume the compiler doesn't schedule instructions in the prologue.
324 */
325
326 foundPushLR = 0;
327 savedRegs = -1;
328 limit = (machine_inst_t *)instrHigh;
329
330 assert(sizeof(*instr) == 4);
331
332 for (j = 0, instr = symbolStart, theInstr = 0;
333 (j < 8) && ((uintptr_t)instr >= instrLow) && (instrHigh > (uintptr_t)(instr)); j++, instr++)
334 {
335 /*
336 * Count the number of time we pushed something onto the stack
337 * before hitting a frame push. That will give us an estimation
338 * of how many stack pops we should expect when looking for the
339 * RET instruction.
340 */
341 theInstr = *instr;
342 if (FBT_IS_ARM64_FRAME_PUSH(theInstr)) {
343 foundPushLR = 1;
344 pushinstr = instr;
345 }
346
347 if (foundPushLR && (FBT_IS_ARM64_ADD_FP_SP(theInstr)))
348 /* Guard against a random setting of fp from sp, we make sure we found the push first */
349 break;
350 if (FBT_IS_ARM64_RET(theInstr)) /* We've gone too far, bail. */
351 break;
352 if (FBT_IS_ARM64_FRAME_POP(theInstr)) /* We've gone too far, bail. */
353 break;
354 }
355
356 if (!(foundPushLR && (FBT_IS_ARM64_ADD_FP_SP(theInstr)))) {
357 return;
358 }
359
360 thisid = dtrace_probe_lookup(fbt_id, modname, symbolName, FBT_ENTRY);
361 newfbt = kmem_zalloc(sizeof(fbt_probe_t), KM_SLEEP);
362 newfbt->fbtp_next = NULL;
363 strlcpy( (char *)&(newfbt->fbtp_name), symbolName, MAX_FBTP_NAME_CHARS );
364
365 if (thisid != 0) {
366 /*
367 * The dtrace_probe previously existed, so we have to hook
368 * the newfbt entry onto the end of the existing fbt's
369 * chain.
370 * If we find an fbt entry that was previously patched to
371 * fire, (as indicated by the current patched value), then
372 * we want to enable this newfbt on the spot.
373 */
374 entryfbt = dtrace_probe_arg (fbt_id, thisid);
375 ASSERT (entryfbt != NULL);
376 for(; entryfbt != NULL; entryfbt = entryfbt->fbtp_next) {
377 if (entryfbt->fbtp_currentval == entryfbt->fbtp_patchval)
378 doenable++;
379
380 if (entryfbt->fbtp_next == NULL) {
381 entryfbt->fbtp_next = newfbt;
382 newfbt->fbtp_id = entryfbt->fbtp_id;
383 break;
384 }
385 }
386 }
387 else {
388 /*
389 * The dtrace_probe did not previously exist, so we
390 * create it and hook in the newfbt. Since the probe is
391 * new, we obviously do not need to enable it on the spot.
392 */
393 newfbt->fbtp_id = dtrace_probe_create(fbt_id, modname, symbolName, FBT_ENTRY, FBT_AFRAMES_ENTRY, newfbt);
394 doenable = 0;
395 }
396
397 newfbt->fbtp_patchpoint = instr;
398 newfbt->fbtp_ctl = ctl;
399 newfbt->fbtp_loadcnt = ctl->mod_loadcnt;
400 newfbt->fbtp_rval = DTRACE_INVOP_PUSH_FRAME;
401 newfbt->fbtp_savedval = theInstr;
402 newfbt->fbtp_patchval = FBT_PATCHVAL;
403 newfbt->fbtp_currentval = 0;
404 newfbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
405 fbt_probetab[FBT_ADDR2NDX(instr)] = newfbt;
406
407 if (doenable)
408 fbt_enable(NULL, newfbt->fbtp_id, newfbt);
409
410 /*
411 * The fbt entry chain is in place, one entry point per symbol.
412 * The fbt return chain can have multiple return points per
413 * symbol.
414 * Here we find the end of the fbt return chain.
415 */
416
417 doenable=0;
418
419 thisid = dtrace_probe_lookup(fbt_id, modname, symbolName, FBT_RETURN);
420
421 if (thisid != 0) {
422 /* The dtrace_probe previously existed, so we have to
423 * find the end of the existing fbt chain. If we find
424 * an fbt return that was previously patched to fire,
425 * (as indicated by the currrent patched value), then
426 * we want to enable any new fbts on the spot.
427 */
428 retfbt = dtrace_probe_arg (fbt_id, thisid);
429 ASSERT(retfbt != NULL);
430 for (; retfbt != NULL; retfbt = retfbt->fbtp_next) {
431 if (retfbt->fbtp_currentval == retfbt->fbtp_patchval)
432 doenable++;
433 if(retfbt->fbtp_next == NULL)
434 break;
435 }
436 }
437 else {
438 doenable = 0;
439 retfbt = NULL;
440 }
441
442 /*
443 * Go back to the start of the function, in case
444 * the compiler emitted pcrel data loads
445 * before FP was adjusted.
446 */
447 instr = pushinstr + 1;
448 again:
449 if (instr >= limit)
450 return;
451
452 /* XXX FIXME ... extra jump table detection? */
453
454 /*
455 * OK, it's an instruction.
456 */
457 theInstr = *instr;
458
459 /* Walked onto the start of the next routine? If so, bail out from this function */
460 if (FBT_IS_ARM64_FRAME_PUSH(theInstr)) {
461 if (!retfbt)
462 kprintf("dtrace: fbt: No return probe for %s, walked to next routine at 0x%016llx\n",symbolName,(uint64_t)instr);
463 return;
464 }
465
466 /* XXX fancy detection of end of function using PC-relative loads */
467
468 /*
469 * Look for:
470 * ldp fp, lr, [sp], #val
471 * ldp fp, lr, [sp, #val]
472 */
473 if (!FBT_IS_ARM64_FRAME_POP(theInstr)) {
474 instr++;
475 goto again;
476 }
477
478 /* go to the next instruction */
479 instr++;
480
481 /* Scan ahead for a ret or a branch outside the function */
482 for (; instr < limit; instr++) {
483 theInstr = *instr;
484 if (FBT_IS_ARM64_RET(theInstr))
485 break;
486 if (FBT_IS_ARM64_B_INSTR(theInstr)) {
487 machine_inst_t *dest = instr + FBT_GET_ARM64_B_IMM(theInstr);
488 /*
489 * Check whether the destination of the branch
490 * is outside of the function
491 */
492 if (dest >= limit || dest < symbolStart)
493 break;
494 }
495 }
496
497 if (!FBT_IS_ARM64_RET(theInstr) && !FBT_IS_ARM64_B_INSTR(theInstr))
498 return;
499
500 newfbt = kmem_zalloc(sizeof(fbt_probe_t), KM_SLEEP);
501 newfbt->fbtp_next = NULL;
502 strlcpy( (char *)&(newfbt->fbtp_name), symbolName, MAX_FBTP_NAME_CHARS );
503
504 if (retfbt == NULL) {
505 newfbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
506 symbolName, FBT_RETURN, FBT_AFRAMES_RETURN, newfbt);
507 } else {
508 retfbt->fbtp_next = newfbt;
509 newfbt->fbtp_id = retfbt->fbtp_id;
510 }
511
512 retfbt = newfbt;
513 newfbt->fbtp_patchpoint = instr;
514 newfbt->fbtp_ctl = ctl;
515 newfbt->fbtp_loadcnt = ctl->mod_loadcnt;
516
517 ASSERT(FBT_IS_ARM64_RET(theInstr));
518 newfbt->fbtp_rval = DTRACE_INVOP_RET;
519 newfbt->fbtp_roffset = (uintptr_t) ((uint8_t*) instr - (uint8_t *)symbolStart);
520 newfbt->fbtp_savedval = theInstr;
521 newfbt->fbtp_patchval = FBT_PATCHVAL;
522 newfbt->fbtp_currentval = 0;
523 newfbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
524 fbt_probetab[FBT_ADDR2NDX(instr)] = newfbt;
525
526 if (doenable)
527 fbt_enable(NULL, newfbt->fbtp_id, newfbt);
528
529 instr++;
530 goto again;
531 }
532
533 void
534 fbt_provide_module_kernel_syms(struct modctl *ctl)
535 {
536 kernel_mach_header_t *mh;
537 struct load_command *cmd;
538 kernel_segment_command_t *orig_ts = NULL, *orig_le = NULL;
539 struct symtab_command *orig_st = NULL;
540 kernel_nlist_t *sym = NULL;
541 char *strings;
542 uintptr_t instrLow, instrHigh;
543 char *modname;
544 unsigned int i;
545
546 mh = (kernel_mach_header_t *)(ctl->mod_address);
547 modname = ctl->mod_modname;
548
549 /*
550 * Employees of dtrace and their families are ineligible. Void
551 * where prohibited.
552 */
553
554 if (mh->magic != MH_MAGIC_KERNEL)
555 return;
556
557 cmd = (struct load_command *) & mh[1];
558 for (i = 0; i < mh->ncmds; i++) {
559 if (cmd->cmd == LC_SEGMENT_KERNEL) {
560 kernel_segment_command_t *orig_sg = (kernel_segment_command_t *) cmd;
561
562 if (LIT_STRNEQL(orig_sg->segname, SEG_TEXT))
563 orig_ts = orig_sg;
564 else if (LIT_STRNEQL(orig_sg->segname, SEG_LINKEDIT))
565 orig_le = orig_sg;
566 else if (LIT_STRNEQL(orig_sg->segname, ""))
567 orig_ts = orig_sg; /* kexts have a single
568 * unnamed segment */
569 } else if (cmd->cmd == LC_SYMTAB)
570 orig_st = (struct symtab_command *) cmd;
571
572 cmd = (struct load_command *) ((caddr_t) cmd + cmd->cmdsize);
573 }
574
575 if ((orig_ts == NULL) || (orig_st == NULL) || (orig_le == NULL))
576 return;
577
578 sym = (kernel_nlist_t *)(orig_le->vmaddr + orig_st->symoff - orig_le->fileoff);
579 strings = (char *)(orig_le->vmaddr + orig_st->stroff - orig_le->fileoff);
580
581 /* Find extent of the TEXT section */
582 instrLow = (uintptr_t) orig_ts->vmaddr;
583 instrHigh = (uintptr_t) (orig_ts->vmaddr + orig_ts->vmsize);
584
585 for (i = 0; i < orig_st->nsyms; i++) {
586 uint8_t n_type = sym[i].n_type & (N_TYPE | N_EXT);
587 char *name = strings + sym[i].n_un.n_strx;
588
589 /* Check that the symbol is a global and that it has a name. */
590 if (((N_SECT | N_EXT) != n_type && (N_ABS | N_EXT) != n_type))
591 continue;
592
593 if (0 == sym[i].n_un.n_strx) /* iff a null, "", name. */
594 continue;
595
596 /* Lop off omnipresent leading underscore. */
597 if (*name == '_')
598 name += 1;
599
600 /*
601 * We're only blacklisting functions in the kernel for now.
602 */
603 if (MOD_IS_MACH_KERNEL(ctl) && fbt_excluded(name))
604 continue;
605
606 fbt_provide_probe(ctl, instrLow, instrHigh, modname, name, (machine_inst_t*)sym[i].n_value);
607 }
608 }
mirror of XNU