]>
Commit | Line | Data |
---|---|---|
5ba3f43e | 1 | /* |
cb323159 | 2 | * Copyright (c) 2007-2018 Apple Inc. All rights reserved. |
5ba3f43e A |
3 | */ |
4 | /* | |
5 | * CDDL HEADER START | |
6 | * | |
7 | * The contents of this file are subject to the terms of the | |
8 | * Common Development and Distribution License, Version 1.0 only | |
9 | * (the "License"). You may not use this file except in compliance | |
10 | * with the License. | |
11 | * | |
12 | * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE | |
13 | * or http://www.opensolaris.org/os/licensing. | |
14 | * See the License for the specific language governing permissions | |
15 | * and limitations under the License. | |
16 | * | |
17 | * When distributing Covered Code, include this CDDL HEADER in each | |
18 | * file and include the License file at usr/src/OPENSOLARIS.LICENSE. | |
19 | * If applicable, add the following below this CDDL HEADER, with the | |
20 | * fields enclosed by brackets "[]" replaced with your own identifying | |
21 | * information: Portions Copyright [yyyy] [name of copyright owner] | |
22 | * | |
23 | * CDDL HEADER END | |
24 | */ | |
25 | /* | |
26 | * Copyright 2005 Sun Microsystems, Inc. All rights reserved. | |
27 | * Use is subject to license terms. | |
28 | */ | |
29 | ||
5ba3f43e A |
30 | #include <kern/thread.h> |
31 | #include <mach/thread_status.h> | |
32 | #include <arm/proc_reg.h> | |
33 | #include <arm/caches_internal.h> | |
34 | ||
35 | #include <mach-o/loader.h> | |
36 | #include <mach-o/nlist.h> | |
37 | #include <libkern/kernel_mach_header.h> | |
38 | ||
39 | #include <sys/param.h> | |
40 | #include <sys/systm.h> | |
41 | #include <sys/errno.h> | |
42 | #include <sys/stat.h> | |
43 | #include <sys/ioctl.h> | |
44 | #include <sys/conf.h> | |
45 | #include <sys/fcntl.h> | |
46 | #include <miscfs/devfs/devfs.h> | |
47 | ||
48 | #include <sys/dtrace.h> | |
49 | #include <sys/dtrace_impl.h> | |
50 | #include <sys/fbt.h> | |
51 | ||
52 | #include <sys/dtrace_glue.h> | |
53 | ||
d9a64523 A |
54 | #if __has_include(<ptrauth.h>) |
55 | #include <ptrauth.h> | |
56 | #endif | |
57 | ||
5ba3f43e A |
58 | #define DTRACE_INVOP_PUSH_FRAME 11 |
59 | ||
0a7de745 A |
60 | #define DTRACE_INVOP_NOP_SKIP 4 |
61 | #define DTRACE_INVOP_ADD_FP_SP_SKIP 4 | |
5ba3f43e A |
62 | |
63 | #define DTRACE_INVOP_POP_PC_SKIP 2 | |
64 | ||
65 | /* | |
66 | * stp fp, lr, [sp, #val] | |
67 | * stp fp, lr, [sp, #val]! | |
68 | */ | |
0a7de745 | 69 | #define FBT_IS_ARM64_FRAME_PUSH(x) \ |
5ba3f43e A |
70 | (((x) & 0xffc07fff) == 0xa9007bfd || ((x) & 0xffc07fff) == 0xa9807bfd) |
71 | ||
72 | /* | |
73 | * stp Xt1, Xt2, [sp, #val] | |
74 | * stp Xt1, Xt2, [sp, #val]! | |
75 | */ | |
0a7de745 | 76 | #define FBT_IS_ARM64_PUSH(x) \ |
5ba3f43e A |
77 | (((x) & 0xffc003e0) == 0xa90003e0 || ((x) & 0xffc003e0) == 0xa98003e0) |
78 | ||
79 | /* | |
80 | * ldp fp, lr, [sp, #val] | |
81 | * ldp fp, lr, [sp], #val | |
82 | */ | |
0a7de745 | 83 | #define FBT_IS_ARM64_FRAME_POP(x) \ |
5ba3f43e A |
84 | (((x) & 0xffc07fff) == 0xa9407bfd || ((x) & 0xffc07fff) == 0xa8c07bfd) |
85 | ||
0a7de745 A |
86 | #define FBT_IS_ARM64_ADD_FP_SP(x) (((x) & 0xffc003ff) == 0x910003fd) /* add fp, sp, #val (add fp, sp, #0 == mov fp, sp) */ |
87 | #define FBT_IS_ARM64_RET(x) (((x) == 0xd65f03c0) || ((x) == 0xd65f0fff)) /* ret, retab */ | |
5ba3f43e A |
88 | |
89 | ||
0a7de745 A |
90 | #define FBT_B_MASK 0xff000000 |
91 | #define FBT_B_IMM_MASK 0x00ffffff | |
92 | #define FBT_B_INSTR 0x14000000 | |
5ba3f43e | 93 | |
0a7de745 A |
94 | #define FBT_IS_ARM64_B_INSTR(x) ((x & FBT_B_MASK) == FBT_B_INSTR) |
95 | #define FBT_GET_ARM64_B_IMM(x) ((x & FBT_B_IMM_MASK) << 2) | |
5ba3f43e | 96 | |
0a7de745 A |
97 | #define FBT_PATCHVAL 0xe7eeee7e |
98 | #define FBT_AFRAMES_ENTRY 7 | |
99 | #define FBT_AFRAMES_RETURN 7 | |
5ba3f43e | 100 | |
0a7de745 A |
101 | #define FBT_ENTRY "entry" |
102 | #define FBT_RETURN "return" | |
103 | #define FBT_ADDR2NDX(addr) ((((uintptr_t)(addr)) >> 4) & fbt_probetab_mask) | |
5ba3f43e | 104 | |
0a7de745 A |
105 | extern dtrace_provider_id_t fbt_id; |
106 | extern fbt_probe_t **fbt_probetab; | |
107 | extern int fbt_probetab_mask; | |
5ba3f43e A |
108 | |
109 | kern_return_t fbt_perfCallback(int, struct arm_saved_state *, __unused int, __unused int); | |
110 | ||
111 | int | |
112 | fbt_invop(uintptr_t addr, uintptr_t * stack, uintptr_t rval) | |
113 | { | |
114 | fbt_probe_t *fbt = fbt_probetab[FBT_ADDR2NDX(addr)]; | |
115 | ||
116 | for (; fbt != NULL; fbt = fbt->fbtp_hashnext) { | |
117 | if ((uintptr_t) fbt->fbtp_patchpoint == addr) { | |
118 | if (0 == CPU->cpu_dtrace_invop_underway) { | |
0a7de745 A |
119 | CPU->cpu_dtrace_invop_underway = 1; /* Race not possible on |
120 | * this per-cpu state */ | |
5ba3f43e A |
121 | |
122 | if (fbt->fbtp_roffset == 0) { | |
123 | /* | |
124 | * Stack looks like this: | |
d9a64523 | 125 | * |
5ba3f43e | 126 | * [Higher addresses] |
d9a64523 | 127 | * |
5ba3f43e A |
128 | * Frame of caller |
129 | * Extra args for callee | |
d9a64523 | 130 | * ------------------------ |
5ba3f43e A |
131 | * Frame from traced function: <previous sp (e.g. 0x1000), return address> |
132 | * ------------------------ | |
133 | * arm_context_t | |
134 | * ------------------------ | |
135 | * Frame from trap handler: <previous sp (e.g. 0x1000) , traced PC > | |
136 | * The traced function never got to mov fp, sp, | |
d9a64523 | 137 | * so there is no frame in the backtrace pointing |
5ba3f43e A |
138 | * to the frame on the stack containing the LR in the |
139 | * caller. | |
140 | * ------------------------ | |
141 | * | | |
142 | * | | |
143 | * | stack grows this way | |
144 | * | | |
145 | * | | |
146 | * v | |
147 | * [Lower addresses] | |
148 | */ | |
149 | ||
150 | arm_saved_state_t *regs = (arm_saved_state_t *)(&((arm_context_t *)stack)->ss); | |
151 | ||
d9a64523 A |
152 | /* |
153 | * cpu_dtrace_caller compensates for fact that the traced function never got to update its fp. | |
154 | * When walking the stack, when we reach the frame where we extract a PC in the patched | |
5ba3f43e | 155 | * function, we put the cpu_dtrace_caller in the backtrace instead. The next frame we extract |
d9a64523 | 156 | * will be in the caller's caller, so we output a backtrace starting at the caller and going |
5ba3f43e A |
157 | * sequentially up the stack. |
158 | */ | |
d9a64523 | 159 | CPU->cpu_dtrace_caller = get_saved_state_lr(regs); |
5ba3f43e | 160 | dtrace_probe(fbt->fbtp_id, get_saved_state_reg(regs, 0), get_saved_state_reg(regs, 1), |
0a7de745 | 161 | get_saved_state_reg(regs, 2), get_saved_state_reg(regs, 3), get_saved_state_reg(regs, 4)); |
5ba3f43e A |
162 | CPU->cpu_dtrace_caller = 0; |
163 | } else { | |
164 | /* | |
165 | * When fbtp_roffset is non-zero, we know we are handling a return probe point. | |
d9a64523 | 166 | * |
5ba3f43e A |
167 | * |
168 | * Stack looks like this, as we've already popped the frame in the traced callee, and | |
169 | * we trap with lr set to the return address in the caller. | |
170 | * [Higher addresses] | |
d9a64523 | 171 | * |
5ba3f43e A |
172 | * Frame of caller |
173 | * Extra args for callee | |
d9a64523 | 174 | * ------------------------ |
5ba3f43e A |
175 | * arm_context_t |
176 | * ------------------------ | |
177 | * Frame from trap handler: <sp at time of trap, traced PC > | |
178 | * ------------------------ | |
179 | * | | |
180 | * | | |
181 | * | stack grows this way | |
182 | * | | |
183 | * | | |
184 | * v | |
185 | * [Lower addresses] | |
186 | */ | |
187 | arm_saved_state_t *regs = (arm_saved_state_t *)(&((arm_context_t *)stack)->ss); | |
188 | ||
189 | CPU->cpu_dtrace_caller = get_saved_state_lr(regs); | |
190 | dtrace_probe(fbt->fbtp_id, fbt->fbtp_roffset, rval, 0, 0, 0); | |
191 | CPU->cpu_dtrace_caller = 0; | |
192 | } | |
193 | CPU->cpu_dtrace_invop_underway = 0; | |
194 | } | |
d9a64523 | 195 | |
5ba3f43e | 196 | /* |
0a7de745 A |
197 | * On other architectures, we return a DTRACE constant to let the callback function |
198 | * know what was replaced. On the ARM, since the function prologue/epilogue machine code | |
199 | * can vary, we need the actual bytes of the instruction, so return the savedval instead. | |
200 | */ | |
201 | return fbt->fbtp_savedval; | |
5ba3f43e A |
202 | } |
203 | } | |
204 | ||
0a7de745 | 205 | return 0; |
5ba3f43e A |
206 | } |
207 | ||
208 | #define IS_USER_TRAP(regs) (PSR64_IS_USER(get_saved_state_cpsr(regs))) | |
209 | #define T_INVALID_OPCODE EXC_BAD_INSTRUCTION | |
210 | #define FBT_EXCEPTION_CODE T_INVALID_OPCODE | |
211 | ||
212 | kern_return_t | |
213 | fbt_perfCallback( | |
0a7de745 A |
214 | int trapno, |
215 | struct arm_saved_state * regs, | |
216 | __unused int unused1, | |
217 | __unused int unused2) | |
5ba3f43e A |
218 | { |
219 | kern_return_t retval = KERN_FAILURE; | |
220 | ||
221 | if (FBT_EXCEPTION_CODE == trapno && !IS_USER_TRAP(regs)) { | |
222 | boolean_t oldlevel = 0; | |
223 | machine_inst_t emul = 0; | |
cb323159 | 224 | uint64_t sp, lr, imm; |
5ba3f43e A |
225 | |
226 | oldlevel = ml_set_interrupts_enabled(FALSE); | |
227 | ||
0a7de745 A |
228 | __asm__ volatile ( |
229 | "Ldtrace_invop_callsite_pre_label:\n" | |
230 | ".data\n" | |
231 | ".private_extern _dtrace_invop_callsite_pre\n" | |
232 | "_dtrace_invop_callsite_pre:\n" | |
233 | " .quad Ldtrace_invop_callsite_pre_label\n" | |
234 | ".text\n" | |
235 | ); | |
236 | ||
237 | emul = dtrace_invop(get_saved_state_pc(regs), (uintptr_t*) regs, get_saved_state_reg(regs, 0)); | |
238 | ||
239 | __asm__ volatile ( | |
240 | "Ldtrace_invop_callsite_post_label:\n" | |
241 | ".data\n" | |
242 | ".private_extern _dtrace_invop_callsite_post\n" | |
243 | "_dtrace_invop_callsite_post:\n" | |
244 | " .quad Ldtrace_invop_callsite_post_label\n" | |
245 | ".text\n" | |
246 | ); | |
5ba3f43e A |
247 | |
248 | if (emul == DTRACE_INVOP_NOP) { | |
249 | /* | |
250 | * Skip over the patched NOP planted by sdt | |
251 | */ | |
cb323159 | 252 | add_saved_state_pc(regs, DTRACE_INVOP_NOP_SKIP); |
5ba3f43e A |
253 | retval = KERN_SUCCESS; |
254 | } else if (FBT_IS_ARM64_ADD_FP_SP(emul)) { | |
255 | /* retrieve the value to add */ | |
256 | uint64_t val = (emul >> 10) & 0xfff; | |
257 | assert(val < 4096); | |
258 | ||
259 | /* retrieve sp */ | |
260 | sp = get_saved_state_sp(regs); | |
261 | ||
262 | /* | |
263 | * emulate the instruction: | |
0a7de745 | 264 | * add fp, sp, #val |
5ba3f43e A |
265 | */ |
266 | assert(sp < (UINT64_MAX - val)); | |
267 | set_saved_state_fp(regs, sp + val); | |
268 | ||
269 | /* skip over the bytes of the patched instruction */ | |
cb323159 | 270 | add_saved_state_pc(regs, DTRACE_INVOP_ADD_FP_SP_SKIP); |
5ba3f43e A |
271 | |
272 | retval = KERN_SUCCESS; | |
273 | } else if (FBT_IS_ARM64_RET(emul)) { | |
274 | lr = get_saved_state_lr(regs); | |
d9a64523 A |
275 | #if __has_feature(ptrauth_calls) |
276 | lr = (user_addr_t) ptrauth_strip((void *)lr, ptrauth_key_return_address); | |
277 | #endif | |
5ba3f43e | 278 | set_saved_state_pc(regs, lr); |
d9a64523 | 279 | retval = KERN_SUCCESS; |
5ba3f43e | 280 | } else if (FBT_IS_ARM64_B_INSTR(emul)) { |
5ba3f43e | 281 | imm = FBT_GET_ARM64_B_IMM(emul); |
cb323159 | 282 | add_saved_state_pc(regs, imm); |
5ba3f43e A |
283 | retval = KERN_SUCCESS; |
284 | } else if (emul == FBT_PATCHVAL) { | |
285 | /* Means we encountered an error but handled it, try same inst again */ | |
286 | retval = KERN_SUCCESS; | |
287 | } else { | |
288 | retval = KERN_FAILURE; | |
289 | } | |
290 | ||
291 | ml_set_interrupts_enabled(oldlevel); | |
292 | } | |
293 | ||
294 | return retval; | |
295 | } | |
296 | ||
297 | void | |
d9a64523 | 298 | fbt_provide_probe(struct modctl *ctl, const char *modname, const char* symbolName, machine_inst_t* symbolStart, machine_inst_t *instrHigh) |
5ba3f43e | 299 | { |
0a7de745 A |
300 | int doenable = 0; |
301 | dtrace_id_t thisid; | |
5ba3f43e | 302 | |
0a7de745 | 303 | fbt_probe_t *newfbt, *retfbt, *entryfbt; |
5ba3f43e A |
304 | machine_inst_t *instr, *pushinstr = NULL, *limit, theInstr; |
305 | int foundPushLR, savedRegs; | |
d9a64523 | 306 | |
5ba3f43e | 307 | /* |
d9a64523 | 308 | * Guard against null and invalid symbols |
5ba3f43e | 309 | */ |
d9a64523 | 310 | if (!symbolStart || !instrHigh || instrHigh < symbolStart) { |
5ba3f43e A |
311 | kprintf("dtrace: %s has an invalid address\n", symbolName); |
312 | return; | |
313 | } | |
314 | ||
315 | /* | |
316 | * Assume the compiler doesn't schedule instructions in the prologue. | |
317 | */ | |
5ba3f43e A |
318 | foundPushLR = 0; |
319 | savedRegs = -1; | |
320 | limit = (machine_inst_t *)instrHigh; | |
321 | ||
322 | assert(sizeof(*instr) == 4); | |
323 | ||
0a7de745 | 324 | for (instr = symbolStart, theInstr = 0; instr < instrHigh; instr++) { |
5ba3f43e A |
325 | /* |
326 | * Count the number of time we pushed something onto the stack | |
327 | * before hitting a frame push. That will give us an estimation | |
328 | * of how many stack pops we should expect when looking for the | |
329 | * RET instruction. | |
330 | */ | |
331 | theInstr = *instr; | |
332 | if (FBT_IS_ARM64_FRAME_PUSH(theInstr)) { | |
333 | foundPushLR = 1; | |
334 | pushinstr = instr; | |
335 | } | |
336 | ||
0a7de745 | 337 | if (foundPushLR && (FBT_IS_ARM64_ADD_FP_SP(theInstr))) { |
5ba3f43e A |
338 | /* Guard against a random setting of fp from sp, we make sure we found the push first */ |
339 | break; | |
0a7de745 A |
340 | } |
341 | if (FBT_IS_ARM64_RET(theInstr)) { /* We've gone too far, bail. */ | |
5ba3f43e | 342 | break; |
0a7de745 A |
343 | } |
344 | if (FBT_IS_ARM64_FRAME_POP(theInstr)) { /* We've gone too far, bail. */ | |
5ba3f43e | 345 | break; |
0a7de745 | 346 | } |
5ba3f43e A |
347 | } |
348 | ||
349 | if (!(foundPushLR && (FBT_IS_ARM64_ADD_FP_SP(theInstr)))) { | |
350 | return; | |
351 | } | |
352 | ||
353 | thisid = dtrace_probe_lookup(fbt_id, modname, symbolName, FBT_ENTRY); | |
354 | newfbt = kmem_zalloc(sizeof(fbt_probe_t), KM_SLEEP); | |
355 | newfbt->fbtp_next = NULL; | |
0a7de745 | 356 | strlcpy((char *)&(newfbt->fbtp_name), symbolName, MAX_FBTP_NAME_CHARS ); |
d9a64523 | 357 | |
5ba3f43e A |
358 | if (thisid != 0) { |
359 | /* | |
360 | * The dtrace_probe previously existed, so we have to hook | |
361 | * the newfbt entry onto the end of the existing fbt's | |
362 | * chain. | |
363 | * If we find an fbt entry that was previously patched to | |
364 | * fire, (as indicated by the current patched value), then | |
365 | * we want to enable this newfbt on the spot. | |
366 | */ | |
0a7de745 A |
367 | entryfbt = dtrace_probe_arg(fbt_id, thisid); |
368 | ASSERT(entryfbt != NULL); | |
369 | for (; entryfbt != NULL; entryfbt = entryfbt->fbtp_next) { | |
370 | if (entryfbt->fbtp_currentval == entryfbt->fbtp_patchval) { | |
5ba3f43e | 371 | doenable++; |
0a7de745 | 372 | } |
5ba3f43e A |
373 | |
374 | if (entryfbt->fbtp_next == NULL) { | |
375 | entryfbt->fbtp_next = newfbt; | |
376 | newfbt->fbtp_id = entryfbt->fbtp_id; | |
377 | break; | |
378 | } | |
379 | } | |
0a7de745 | 380 | } else { |
5ba3f43e A |
381 | /* |
382 | * The dtrace_probe did not previously exist, so we | |
383 | * create it and hook in the newfbt. Since the probe is | |
384 | * new, we obviously do not need to enable it on the spot. | |
385 | */ | |
386 | newfbt->fbtp_id = dtrace_probe_create(fbt_id, modname, symbolName, FBT_ENTRY, FBT_AFRAMES_ENTRY, newfbt); | |
387 | doenable = 0; | |
388 | } | |
389 | ||
390 | newfbt->fbtp_patchpoint = instr; | |
391 | newfbt->fbtp_ctl = ctl; | |
392 | newfbt->fbtp_loadcnt = ctl->mod_loadcnt; | |
393 | newfbt->fbtp_rval = DTRACE_INVOP_PUSH_FRAME; | |
394 | newfbt->fbtp_savedval = theInstr; | |
395 | newfbt->fbtp_patchval = FBT_PATCHVAL; | |
396 | newfbt->fbtp_currentval = 0; | |
397 | newfbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; | |
398 | fbt_probetab[FBT_ADDR2NDX(instr)] = newfbt; | |
399 | ||
0a7de745 | 400 | if (doenable) { |
5ba3f43e | 401 | fbt_enable(NULL, newfbt->fbtp_id, newfbt); |
0a7de745 | 402 | } |
5ba3f43e A |
403 | |
404 | /* | |
405 | * The fbt entry chain is in place, one entry point per symbol. | |
406 | * The fbt return chain can have multiple return points per | |
407 | * symbol. | |
408 | * Here we find the end of the fbt return chain. | |
409 | */ | |
410 | ||
0a7de745 | 411 | doenable = 0; |
5ba3f43e A |
412 | |
413 | thisid = dtrace_probe_lookup(fbt_id, modname, symbolName, FBT_RETURN); | |
d9a64523 | 414 | |
5ba3f43e A |
415 | if (thisid != 0) { |
416 | /* The dtrace_probe previously existed, so we have to | |
417 | * find the end of the existing fbt chain. If we find | |
418 | * an fbt return that was previously patched to fire, | |
419 | * (as indicated by the currrent patched value), then | |
420 | * we want to enable any new fbts on the spot. | |
421 | */ | |
0a7de745 | 422 | retfbt = dtrace_probe_arg(fbt_id, thisid); |
5ba3f43e | 423 | ASSERT(retfbt != NULL); |
0a7de745 A |
424 | for (; retfbt != NULL; retfbt = retfbt->fbtp_next) { |
425 | if (retfbt->fbtp_currentval == retfbt->fbtp_patchval) { | |
5ba3f43e | 426 | doenable++; |
0a7de745 A |
427 | } |
428 | if (retfbt->fbtp_next == NULL) { | |
5ba3f43e | 429 | break; |
0a7de745 | 430 | } |
5ba3f43e | 431 | } |
0a7de745 | 432 | } else { |
5ba3f43e A |
433 | doenable = 0; |
434 | retfbt = NULL; | |
435 | } | |
436 | ||
437 | /* | |
438 | * Go back to the start of the function, in case | |
439 | * the compiler emitted pcrel data loads | |
440 | * before FP was adjusted. | |
441 | */ | |
442 | instr = pushinstr + 1; | |
443 | again: | |
0a7de745 | 444 | if (instr >= limit) { |
5ba3f43e | 445 | return; |
0a7de745 | 446 | } |
5ba3f43e A |
447 | |
448 | /* XXX FIXME ... extra jump table detection? */ | |
449 | ||
450 | /* | |
451 | * OK, it's an instruction. | |
452 | */ | |
453 | theInstr = *instr; | |
d9a64523 | 454 | |
5ba3f43e A |
455 | /* Walked onto the start of the next routine? If so, bail out from this function */ |
456 | if (FBT_IS_ARM64_FRAME_PUSH(theInstr)) { | |
0a7de745 A |
457 | if (!retfbt) { |
458 | kprintf("dtrace: fbt: No return probe for %s, walked to next routine at 0x%016llx\n", symbolName, (uint64_t)instr); | |
459 | } | |
5ba3f43e A |
460 | return; |
461 | } | |
462 | ||
463 | /* XXX fancy detection of end of function using PC-relative loads */ | |
464 | ||
465 | /* | |
466 | * Look for: | |
0a7de745 A |
467 | * ldp fp, lr, [sp], #val |
468 | * ldp fp, lr, [sp, #val] | |
5ba3f43e A |
469 | */ |
470 | if (!FBT_IS_ARM64_FRAME_POP(theInstr)) { | |
471 | instr++; | |
472 | goto again; | |
473 | } | |
474 | ||
475 | /* go to the next instruction */ | |
476 | instr++; | |
477 | ||
478 | /* Scan ahead for a ret or a branch outside the function */ | |
479 | for (; instr < limit; instr++) { | |
480 | theInstr = *instr; | |
0a7de745 | 481 | if (FBT_IS_ARM64_RET(theInstr)) { |
5ba3f43e | 482 | break; |
0a7de745 | 483 | } |
5ba3f43e A |
484 | if (FBT_IS_ARM64_B_INSTR(theInstr)) { |
485 | machine_inst_t *dest = instr + FBT_GET_ARM64_B_IMM(theInstr); | |
486 | /* | |
487 | * Check whether the destination of the branch | |
488 | * is outside of the function | |
489 | */ | |
0a7de745 | 490 | if (dest >= limit || dest < symbolStart) { |
5ba3f43e | 491 | break; |
0a7de745 | 492 | } |
5ba3f43e A |
493 | } |
494 | } | |
495 | ||
0a7de745 | 496 | if (!FBT_IS_ARM64_RET(theInstr) && !FBT_IS_ARM64_B_INSTR(theInstr)) { |
5ba3f43e | 497 | return; |
0a7de745 | 498 | } |
5ba3f43e A |
499 | |
500 | newfbt = kmem_zalloc(sizeof(fbt_probe_t), KM_SLEEP); | |
d9a64523 | 501 | newfbt->fbtp_next = NULL; |
0a7de745 | 502 | strlcpy((char *)&(newfbt->fbtp_name), symbolName, MAX_FBTP_NAME_CHARS ); |
5ba3f43e A |
503 | |
504 | if (retfbt == NULL) { | |
505 | newfbt->fbtp_id = dtrace_probe_create(fbt_id, modname, | |
506 | symbolName, FBT_RETURN, FBT_AFRAMES_RETURN, newfbt); | |
507 | } else { | |
508 | retfbt->fbtp_next = newfbt; | |
509 | newfbt->fbtp_id = retfbt->fbtp_id; | |
510 | } | |
511 | ||
512 | retfbt = newfbt; | |
513 | newfbt->fbtp_patchpoint = instr; | |
514 | newfbt->fbtp_ctl = ctl; | |
515 | newfbt->fbtp_loadcnt = ctl->mod_loadcnt; | |
516 | ||
517 | ASSERT(FBT_IS_ARM64_RET(theInstr)); | |
518 | newfbt->fbtp_rval = DTRACE_INVOP_RET; | |
519 | newfbt->fbtp_roffset = (uintptr_t) ((uint8_t*) instr - (uint8_t *)symbolStart); | |
520 | newfbt->fbtp_savedval = theInstr; | |
521 | newfbt->fbtp_patchval = FBT_PATCHVAL; | |
522 | newfbt->fbtp_currentval = 0; | |
523 | newfbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; | |
524 | fbt_probetab[FBT_ADDR2NDX(instr)] = newfbt; | |
525 | ||
0a7de745 | 526 | if (doenable) { |
5ba3f43e | 527 | fbt_enable(NULL, newfbt->fbtp_id, newfbt); |
0a7de745 | 528 | } |
5ba3f43e A |
529 | |
530 | instr++; | |
531 | goto again; | |
532 | } |