bsd/dev/arm/fbt_arm.c

   1 /*
   2  * Copyright (c) 2007-2018 Apple Inc. All rights reserved.
   3  */
   4 /*
   5  * CDDL HEADER START
   6  *
   7  * The contents of this file are subject to the terms of the
   8  * Common Development and Distribution License, Version 1.0 only
   9  * (the "License").  You may not use this file except in compliance
  10  * with the License.
  11  *
  12  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  13  * or http://www.opensolaris.org/os/licensing.
  14  * See the License for the specific language governing permissions
  15  * and limitations under the License.
  16  *
  17  * When distributing Covered Code, include this CDDL HEADER in each
  18  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  19  * If applicable, add the following below this CDDL HEADER, with the
  20  * fields enclosed by brackets "[]" replaced with your own identifying
  21  * information: Portions Copyright [yyyy] [name of copyright owner]
  22  *
  23  * CDDL HEADER END
  24  */
  25 /*
  26  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
  27  * Use is subject to license terms.
  28  */
  29
  30 #include <kern/thread.h>
  31 #include <mach/thread_status.h>
  32 #include <arm/proc_reg.h>
  33 #include <arm/caches_internal.h>
  34 #include <arm/thread.h>
  35
  36 #include <mach-o/loader.h>
  37 #include <mach-o/nlist.h>
  38 #include <libkern/kernel_mach_header.h>
  39
  40 #include <sys/param.h>
  41 #include <sys/systm.h>
  42 #include <sys/errno.h>
  43 #include <sys/stat.h>
  44 #include <sys/ioctl.h>
  45 #include <sys/conf.h>
  46 #include <sys/fcntl.h>
  47 #include <miscfs/devfs/devfs.h>
  48
  49 #include <sys/dtrace.h>
  50 #include <sys/dtrace_impl.h>
  51 #include <sys/fbt.h>
  52
  53 #include <sys/dtrace_glue.h>
  54
  55 #define DTRACE_INVOP_PUSH_LR 8
  56 #define DTRACE_INVOP_BL 9
  57 #define DTRACE_INVOP_POP_PC 10
  58
  59 #define DTRACE_INVOP_THUMB_NOP_SKIP 2
  60 #define DTRACE_INVOP_POP_PC_SKIP 2
  61 #define DTRACE_INVOP_THUMB_SET_R7_SKIP 2
  62 #define DTRACE_INVOP_THUMB_MOV_SP_TO_R7_SKIP 2
  63
  64 #define FBT_IS_THUMB_PUSH_LR(x)         (((x) & 0x0000ff00) == 0x0000b500)
  65 #define FBT_IS_THUMB_POP_R7(x)          (((x) & 0x0000ff80) == 0x0000bc80)
  66 #define FBT_IS_THUMB32_POP_R7LR(x, y)    (((x) == 0x0000e8bd) && (((y) & 0x00004080) == 0x00004080))
  67 #define FBT_IS_THUMB_POP_PC(x)          (((x) & 0x0000ff00) == 0x0000bd00)
  68 #define FBT_IS_THUMB_SET_R7(x)          (((x) & 0x0000ff00) == 0x0000af00)
  69 #define FBT_IS_THUMB_MOV_SP_TO_R7(x)            (((x) & 0x0000ffff) == 0x0000466f)
  70 #define FBT_THUMB_SET_R7_OFFSET(x)      (((x) & 0x000000ff) << 2)
  71 #define FBT_IS_THUMB_LDR_PC(x)          (((x) & 0x0000f800) == 0x00004800)
  72 #define FBT_IS_THUMB32_LDR_PC(x, y)      ((x) == 0x0000f8df)                    /* Only for positive offset PC relative loads */
  73 #define FBT_THUMB_STACK_REGS(x)         ((x) & 0x00FF)
  74 #define FBT_IS_THUMB_BX_REG(x)          (((x) & 0x0000ff87) == 0x00004700)
  75
  76 #define FBT_PATCHVAL                    0xdefc
  77 #define FBT_AFRAMES_ENTRY               8
  78 #define FBT_AFRAMES_RETURN              6
  79
  80 #define FBT_ENTRY       "entry"
  81 #define FBT_RETURN      "return"
  82 #define FBT_ADDR2NDX(addr)      ((((uintptr_t)(addr)) >> 4) & fbt_probetab_mask)
  83
  84 #define VFPSAVE_ALIGN_DTRACE    16      /* This value should come from VFPSAVE_ALIGN */
  85
  86 extern dtrace_provider_id_t     fbt_id;
  87 extern fbt_probe_t               **fbt_probetab;
  88 extern int                      fbt_probetab_mask;
  89
  90 kern_return_t fbt_perfCallback(int, struct arm_saved_state *, __unused int, __unused int);
  91
  92 extern int dtrace_arm_condition_true(int cond, int cpsr);
  93
  94
  95 /* Calculate the address of the ldr. (From the ARM Architecture reference) */
  96 /* Does not check to see if it's really a load instruction, caller must do that */
  97
  98 static uint32_t
  99 thumb_ldr_pc_address(uint32_t address)
 100 {
 101         return (address & 0xFFFFFFFC) + (*(uint16_t*) address & 0xFF) * 4 + 4;
 102 }
 103
 104 static uint32_t
 105 thumb32_ldr_pc_address(uint32_t address)
 106 {
 107         return (address & 0xFFFFFFFC) + (*(uint16_t*) (address + 2) & 0xFFF) + 4;
 108 }
 109
 110 /* Extract the current ITSTATE from the CPSR */
 111 static uint32_t
 112 get_itstate(uint32_t cpsr)
 113 {
 114         return
 115                 ((cpsr & 0x06000000) >> 25) |
 116                 ((cpsr & 0x0000FC00) >> 8);
 117 }
 118
 119 static void
 120 clear_itstate(uint32_t* cpsr)
 121 {
 122         *cpsr &= ~0x0600FC00;
 123 }
 124
 125 int
 126 fbt_invop(uintptr_t addr, uintptr_t * stack, uintptr_t rval)
 127 {
 128         fbt_probe_t    *fbt = fbt_probetab[FBT_ADDR2NDX(addr)];
 129
 130         for (; fbt != NULL; fbt = fbt->fbtp_hashnext) {
 131                 if ((uintptr_t) fbt->fbtp_patchpoint == addr) {
 132                         if (0 == CPU->cpu_dtrace_invop_underway) {
 133                                 CPU->cpu_dtrace_invop_underway = 1;     /* Race not possible on
 134                                                                         * this per-cpu state */
 135
 136                                 struct arm_saved_state* regs = (struct arm_saved_state*) stack;
 137                                 uintptr_t stack4 = *((uintptr_t*) regs->sp);
 138
 139                                 if ((regs->cpsr & PSR_MODE_MASK) == PSR_FIQ_MODE) {
 140                                         /*
 141                                          * We do not handle probes firing from FIQ context. We used to
 142                                          * try to undo the patch and rerun the instruction, but
 143                                          * most of the time we can't do that successfully anyway.
 144                                          * Instead, we just panic now so we fail fast.
 145                                          */
 146                                         panic("dtrace: fbt: The probe at %08x was called from FIQ_MODE", (unsigned) addr);
 147                                 }
 148
 149                                 /*
 150                                  * If we are not outside an IT block, and are not executing the last instruction of an IT block,
 151                                  * then that is an instrumentation error or a code gen error. Either way, we panic.
 152                                  */
 153                                 uint32_t itstate = get_itstate(regs->cpsr);
 154                                 if ((itstate & 0x7) != 0) {
 155                                         panic("dtrace: fbt: Instruction stream error: Middle of IT block at %08x", (unsigned) addr);
 156                                 }
 157
 158                                 if (fbt->fbtp_roffset == 0) {
 159                                         /*
 160                                          *       We need the frames to set up the backtrace, but we won't have the frame pointers
 161                                          *       until after the instruction is emulated. So here we calculate the address of the
 162                                          *       frame pointer from the saved instruction and put it in the stack. Yes, we end up
 163                                          *       repeating this work again when we emulate the instruction.
 164                                          *
 165                                          *       This assumes that the frame area is immediately after the saved reg storage!
 166                                          */
 167                                         uint32_t offset = ((uint32_t) regs) + sizeof(struct arm_saved_state);
 168 #if __ARM_VFP__
 169                                         /* Match the stack alignment required for arm_vfpsaved_state */
 170                                         offset &= ~(VFPSAVE_ALIGN_DTRACE - 1);
 171                                         offset += VFPSAVE_ALIGN_DTRACE + sizeof(struct arm_vfpsaved_state);
 172 #endif /* __ARM_VFP__ */
 173                                         if (FBT_IS_THUMB_SET_R7(fbt->fbtp_savedval)) {
 174                                                 *((uint32_t*) offset) = regs->sp + FBT_THUMB_SET_R7_OFFSET(fbt->fbtp_savedval);
 175                                         } else {
 176                                                 *((uint32_t*) offset) = regs->sp;
 177                                         }
 178
 179                                         CPU->cpu_dtrace_caller = regs->lr;
 180                                         dtrace_probe(fbt->fbtp_id, regs->r[0], regs->r[1], regs->r[2], regs->r[3], stack4);
 181                                         CPU->cpu_dtrace_caller = 0;
 182                                 } else {
 183                                         /* Check to see if we're in the middle of an IT block. */
 184                                         if (itstate != 0) {
 185                                                 /*
 186                                                  * We've already checked previously to see how far we are in the IT block.
 187                                                  * Here we must be getting ready to execute the last instruction.
 188                                                  */
 189                                                 int condition_it = (itstate & 0xF0) >> 4;
 190
 191                                                 if (dtrace_arm_condition_true(condition_it, regs->cpsr) == 0) {
 192                                                         /* Condition wasn't true, so becomes a nop. */
 193                                                         clear_itstate(&regs->cpsr);
 194                                                         CPU->cpu_dtrace_invop_underway = 0;
 195                                                         return DTRACE_INVOP_NOP;
 196                                                 }
 197                                         }
 198
 199                                         dtrace_probe(fbt->fbtp_id, fbt->fbtp_roffset, rval, 0, 0, 0);
 200                                         CPU->cpu_dtrace_caller = 0;
 201
 202                                         /* The dtrace script may access cpsr, so make sure to clear only after probe fired. */
 203                                         clear_itstate(&regs->cpsr);
 204                                 }
 205                                 CPU->cpu_dtrace_invop_underway = 0;
 206                         }
 207
 208                         /*
 209                          *       On other architectures, we return a DTRACE constant to let the callback function
 210                          *       know what was replaced. On the ARM, since the function prologue/epilogue machine code
 211                          *       can vary, we need the actual bytes of the instruction, so return the savedval instead.
 212                          */
 213                         return fbt->fbtp_savedval;
 214                 }
 215         }
 216
 217         return 0;
 218 }
 219
 220 #define IS_USER_TRAP(regs)  (((regs)->cpsr & PSR_MODE_MASK) == PSR_USER_MODE)
 221 #define T_INVALID_OPCODE EXC_BAD_INSTRUCTION
 222 #define FBT_EXCEPTION_CODE T_INVALID_OPCODE
 223
 224 kern_return_t
 225 fbt_perfCallback(
 226         int trapno,
 227         struct arm_saved_state * regs,
 228         __unused int unused1,
 229         __unused int unused2)
 230 {
 231 #pragma unused (unused1)
 232 #pragma unused (unused2)
 233         kern_return_t   retval = KERN_FAILURE;
 234
 235         if (FBT_EXCEPTION_CODE == trapno && !IS_USER_TRAP(regs)) {
 236                 boolean_t oldlevel = 0;
 237                 machine_inst_t emul = 0;
 238
 239                 oldlevel = ml_set_interrupts_enabled(FALSE);
 240
 241                 __asm__ volatile (
 242                          "Ldtrace_invop_callsite_pre_label:\n"
 243                          ".data\n"
 244                          ".private_extern _dtrace_invop_callsite_pre\n"
 245                          "_dtrace_invop_callsite_pre:\n"
 246                          "  .long Ldtrace_invop_callsite_pre_label\n"
 247                          ".text\n"
 248                 );
 249
 250                 emul = dtrace_invop(regs->pc, (uintptr_t*) regs, regs->r[0]);
 251
 252                 __asm__ volatile (
 253                          "Ldtrace_invop_callsite_post_label:\n"
 254                          ".data\n"
 255                          ".private_extern _dtrace_invop_callsite_post\n"
 256                          "_dtrace_invop_callsite_post:\n"
 257                          "  .long Ldtrace_invop_callsite_post_label\n"
 258                          ".text\n"
 259                 );
 260
 261                 /*
 262                  * The following emulation code does not execute properly if we are in the middle of
 263                  * an IT block. IT blocks need to be handled in the dtrace_invop function. If we do
 264                  * manage to get here and we are inside an IT block, then we missed a case somewhere
 265                  * prior to this point.
 266                  */
 267                 uint32_t itstate = get_itstate(regs->cpsr);
 268                 if (itstate != 0) {
 269                         panic("dtrace: fbt: Not emulated: Middle of IT block at %08x", (unsigned) regs->pc);
 270                 }
 271
 272                 if (emul == DTRACE_INVOP_NOP) {
 273                         regs->pc += DTRACE_INVOP_THUMB_NOP_SKIP;
 274                         retval = KERN_SUCCESS;
 275                 } else if (FBT_IS_THUMB_SET_R7(emul)) {
 276                         regs->r[7] = regs->sp + FBT_THUMB_SET_R7_OFFSET(emul);
 277                         regs->pc += DTRACE_INVOP_THUMB_SET_R7_SKIP;
 278                         retval = KERN_SUCCESS;
 279                 } else if (FBT_IS_THUMB_MOV_SP_TO_R7(emul)) {
 280                         regs->r[7] = regs->sp;
 281                         regs->pc += DTRACE_INVOP_THUMB_MOV_SP_TO_R7_SKIP;
 282                         retval = KERN_SUCCESS;
 283                 } else if (FBT_IS_THUMB_POP_PC(emul)) {
 284                         uintptr_t* sp = (uintptr_t*) regs->sp;
 285
 286                         machine_inst_t mask = 0x0001;
 287                         int regnum = 0;
 288                         while (mask & 0x00ff) {
 289                                 if (emul & mask) {
 290                                         /* Pop this register */
 291                                         regs->r[regnum] = *sp++;
 292                                 }
 293                                 mask <<= 1;
 294                                 regnum++;
 295                         }
 296
 297                         regs->pc = *sp++;
 298                         regs->sp = (uintptr_t) sp;
 299                         if (regs->pc & 1) {
 300                                 regs->cpsr |= PSR_TF;
 301                         } else {
 302                                 regs->cpsr &= ~PSR_TF;
 303                         }
 304
 305                         retval = KERN_SUCCESS;
 306                 } else if (FBT_IS_THUMB_BX_REG(emul)) {
 307                         regs->pc = regs->r[(emul >> 3) & 0xF];
 308
 309                         if (regs->pc & 1) {
 310                                 regs->cpsr |= PSR_TF;
 311                         } else {
 312                                 regs->cpsr &= ~PSR_TF;
 313                         }
 314
 315                         retval = KERN_SUCCESS;
 316                 } else if (emul == FBT_PATCHVAL) {
 317                         /* Means we encountered an error but handled it, try same inst again */
 318                         retval = KERN_SUCCESS;
 319                 } else {
 320                         retval = KERN_FAILURE;
 321                 }
 322
 323                 ml_set_interrupts_enabled(oldlevel);
 324         }
 325
 326         return retval;
 327 }
 328
 329 void
 330 fbt_provide_probe(struct modctl *ctl, const char *modname, const char* symbolName, machine_inst_t* symbolStart, machine_inst_t *instrHigh)
 331 {
 332         unsigned int    j;
 333         int             doenable = 0;
 334         dtrace_id_t     thisid;
 335
 336         fbt_probe_t     *newfbt, *retfbt, *entryfbt;
 337         machine_inst_t *instr, *pushinstr = NULL, *limit, theInstr;
 338         int             foundPushLR, savedRegs;
 339
 340         /*
 341          * Guard against null symbols
 342          */
 343         if (!symbolStart || !instrHigh || instrHigh < symbolStart) {
 344                 kprintf("dtrace: %s has an invalid address\n", symbolName);
 345                 return;
 346         }
 347
 348         /*
 349          * Assume the compiler doesn't schedule instructions in the prologue.
 350          */
 351         foundPushLR = 0;
 352         savedRegs = -1;
 353         limit = (machine_inst_t *)instrHigh;
 354         for (j = 0, instr = symbolStart, theInstr = 0;
 355             (j < 8) && instr < instrHigh; j++, instr++) {
 356                 theInstr = *instr;
 357                 if (FBT_IS_THUMB_PUSH_LR(theInstr)) {
 358                         foundPushLR = 1;
 359                         /* Keep track of what registers we pushed. Compare this against the pop later. */
 360                         savedRegs = FBT_THUMB_STACK_REGS(theInstr);
 361                         pushinstr = instr;
 362                 }
 363                 if (foundPushLR && (FBT_IS_THUMB_SET_R7(theInstr) || FBT_IS_THUMB_MOV_SP_TO_R7(theInstr))) {
 364                         /* Guard against a random setting of r7 from sp, we make sure we found the push first */
 365                         break;
 366                 }
 367                 if (FBT_IS_THUMB_BX_REG(theInstr)) { /* We've gone too far, bail. */
 368                         break;
 369                 }
 370                 if (FBT_IS_THUMB_POP_PC(theInstr)) { /* We've gone too far, bail. */
 371                         break;
 372                 }
 373
 374                 /* Check for 4 byte thumb instruction */
 375                 if (dtrace_instr_size(theInstr, 1) == 4) {
 376                         instr++;
 377                 }
 378         }
 379
 380         if (!(foundPushLR && (FBT_IS_THUMB_SET_R7(theInstr) || FBT_IS_THUMB_MOV_SP_TO_R7(theInstr)))) {
 381                 return;
 382         }
 383
 384         thisid = dtrace_probe_lookup(fbt_id, modname, symbolName, FBT_ENTRY);
 385         newfbt = kmem_zalloc(sizeof(fbt_probe_t), KM_SLEEP);
 386         newfbt->fbtp_next = NULL;
 387         strlcpy((char *)&(newfbt->fbtp_name), symbolName, MAX_FBTP_NAME_CHARS );
 388
 389         if (thisid != 0) {
 390                 /*
 391                  * The dtrace_probe previously existed, so we have to hook
 392                  * the newfbt entry onto the end of the existing fbt's
 393                  * chain.
 394                  * If we find an fbt entry that was previously patched to
 395                  * fire, (as indicated by the current patched value), then
 396                  * we want to enable this newfbt on the spot.
 397                  */
 398                 entryfbt = dtrace_probe_arg(fbt_id, thisid);
 399                 ASSERT(entryfbt != NULL);
 400                 for (; entryfbt != NULL; entryfbt = entryfbt->fbtp_next) {
 401                         if (entryfbt->fbtp_currentval == entryfbt->fbtp_patchval) {
 402                                 doenable++;
 403                         }
 404
 405                         if (entryfbt->fbtp_next == NULL) {
 406                                 entryfbt->fbtp_next = newfbt;
 407                                 newfbt->fbtp_id = entryfbt->fbtp_id;
 408                                 break;
 409                         }
 410                 }
 411         } else {
 412                 /*
 413                  * The dtrace_probe did not previously exist, so we
 414                  * create it and hook in the newfbt.  Since the probe is
 415                  * new, we obviously do not need to enable it on the spot.
 416                  */
 417                 newfbt->fbtp_id = dtrace_probe_create(fbt_id, modname, symbolName, FBT_ENTRY, FBT_AFRAMES_ENTRY, newfbt);
 418                 doenable = 0;
 419         }
 420
 421         newfbt->fbtp_patchpoint = instr;
 422         newfbt->fbtp_ctl = ctl;
 423         newfbt->fbtp_loadcnt = ctl->mod_loadcnt;
 424         newfbt->fbtp_rval = DTRACE_INVOP_PUSH_LR;
 425         newfbt->fbtp_savedval = theInstr;
 426         newfbt->fbtp_patchval = FBT_PATCHVAL;
 427         newfbt->fbtp_currentval = 0;
 428         newfbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
 429         fbt_probetab[FBT_ADDR2NDX(instr)] = newfbt;
 430
 431         if (doenable) {
 432                 fbt_enable(NULL, newfbt->fbtp_id, newfbt);
 433         }
 434
 435         /*
 436          * The fbt entry chain is in place, one entry point per symbol.
 437          * The fbt return chain can have multiple return points per
 438          * symbol.
 439          * Here we find the end of the fbt return chain.
 440          */
 441
 442         doenable = 0;
 443
 444         thisid = dtrace_probe_lookup(fbt_id, modname, symbolName, FBT_RETURN);
 445
 446         if (thisid != 0) {
 447                 /* The dtrace_probe previously existed, so we have to
 448                  * find the end of the existing fbt chain.  If we find
 449                  * an fbt return that was previously patched to fire,
 450                  * (as indicated by the currrent patched value), then
 451                  * we want to enable any new fbts on the spot.
 452                  */
 453                 retfbt = dtrace_probe_arg(fbt_id, thisid);
 454                 ASSERT(retfbt != NULL);
 455                 for (; retfbt != NULL; retfbt =  retfbt->fbtp_next) {
 456                         if (retfbt->fbtp_currentval == retfbt->fbtp_patchval) {
 457                                 doenable++;
 458                         }
 459                         if (retfbt->fbtp_next == NULL) {
 460                                 break;
 461                         }
 462                 }
 463         } else {
 464                 doenable = 0;
 465                 retfbt = NULL;
 466         }
 467
 468         /*
 469          * Go back to the start of the function, in case
 470          * the compiler emitted pcrel data loads
 471          * before R7 was adjusted.
 472          */
 473         instr = pushinstr + 1;
 474 again:
 475         if (instr >= limit) {
 476                 return;
 477         }
 478
 479         /*
 480          * We (desperately) want to avoid erroneously instrumenting a
 481          * jump table. To determine if we're looking at a true instruction
 482          * or an inline jump table that happens to contain the same
 483          * byte sequences, we resort to some heuristic sleeze:  we
 484          * treat this instruction as being contained within a pointer,
 485          * and see if that pointer points to within the body of the
 486          * function.  If it does, we refuse to instrument it.
 487          */
 488         if (((uintptr_t)instr & 0x3) == 0) {
 489                 machine_inst_t *ptr = *(machine_inst_t **)(void *)instr;
 490
 491                 if (ptr >= (machine_inst_t *)symbolStart && ptr < limit) {
 492                         /* kprintf("dtrace: fbt: Found jump table in %s, at %08x\n",symbolName,(unsigned)instr); */
 493                         instr++;
 494                         goto again;
 495                 }
 496         }
 497
 498         /*
 499          * OK, it's an instruction.
 500          */
 501         theInstr = *instr;
 502
 503         /* Walked onto the start of the next routine? If so, bail out from this function */
 504         if (FBT_IS_THUMB_PUSH_LR(theInstr)) {
 505                 if (!retfbt) {
 506                         kprintf("dtrace: fbt: No return probe for %s, walked to next routine at %08x\n", symbolName, (unsigned)instr);
 507                 }
 508                 return;
 509         }
 510
 511         /* The PC relative data should be stored after the end of the function. If
 512          * we see a PC relative load, assume the address to load from is the new end
 513          * of the function. */
 514         if (FBT_IS_THUMB_LDR_PC(theInstr)) {
 515                 uint32_t newlimit = thumb_ldr_pc_address((uint32_t) instr);
 516                 if (newlimit < (uint32_t) limit) {
 517                         limit = (machine_inst_t*) newlimit;
 518                 }
 519         }
 520         if ((instr + 1) < limit && FBT_IS_THUMB32_LDR_PC(*instr, *(instr + 1))) {
 521                 uint32_t newlimit = thumb32_ldr_pc_address((uint32_t) instr);
 522                 if (newlimit < (uint32_t) limit) {
 523                         limit = (machine_inst_t*) newlimit;
 524                 }
 525         }
 526
 527         /* Look for the 1. pop { ..., pc } or 2. pop { ..., r7 } ... bx reg or 3. ldmia.w sp!, { ..., r7, lr } ... bx reg */
 528         if (!FBT_IS_THUMB_POP_PC(theInstr) &&
 529             !FBT_IS_THUMB_POP_R7(theInstr) &&
 530             !FBT_IS_THUMB32_POP_R7LR(theInstr, *(instr + 1))) {
 531                 instr++;
 532                 if (dtrace_instr_size(theInstr, 1) == 4) {
 533                         instr++;
 534                 }
 535                 goto again;
 536         }
 537
 538         if (FBT_IS_THUMB_POP_PC(theInstr)) {
 539                 if (savedRegs != FBT_THUMB_STACK_REGS(theInstr)) {
 540                         /* What we're popping doesn't match what we're pushing, assume that we've
 541                          * gone too far in the function. Bail.
 542                          */
 543                         kprintf("dtrace: fbt: No return probe for %s, popped regs don't match at %08x\n", symbolName, (unsigned)instr);
 544                         return;
 545                 }
 546         } else {
 547                 /* Scan ahead for the bx */
 548                 for (j = 0; (j < 4) && (instr < limit); j++, instr++) {
 549                         theInstr = *instr;
 550                         if (FBT_IS_THUMB_BX_REG(theInstr)) {
 551                                 break;
 552                         }
 553                         if (dtrace_instr_size(theInstr, 1) == 4) {
 554                                 instr++;
 555                         }
 556                 }
 557
 558                 if (!FBT_IS_THUMB_BX_REG(theInstr)) {
 559                         return;
 560                 }
 561         }
 562
 563         /*
 564          * pop { ..., pc}, bx reg -- We have a winner!
 565          */
 566
 567         newfbt = kmem_zalloc(sizeof(fbt_probe_t), KM_SLEEP);
 568         newfbt->fbtp_next = NULL;
 569         strlcpy((char *)&(newfbt->fbtp_name), symbolName, MAX_FBTP_NAME_CHARS );
 570
 571         if (retfbt == NULL) {
 572                 newfbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
 573                     symbolName, FBT_RETURN, FBT_AFRAMES_RETURN, newfbt);
 574         } else {
 575                 retfbt->fbtp_next = newfbt;
 576                 newfbt->fbtp_id = retfbt->fbtp_id;
 577         }
 578
 579         retfbt = newfbt;
 580         newfbt->fbtp_patchpoint = instr;
 581         newfbt->fbtp_ctl = ctl;
 582         newfbt->fbtp_loadcnt = ctl->mod_loadcnt;
 583
 584         ASSERT(FBT_IS_THUMB_POP_PC(theInstr) || FBT_IS_THUMB_BX_REG(theInstr));
 585         newfbt->fbtp_rval = DTRACE_INVOP_POP_PC;
 586         newfbt->fbtp_roffset =
 587             (uintptr_t) ((uint8_t*) instr - (uint8_t *)symbolStart);
 588         newfbt->fbtp_savedval = theInstr;
 589         newfbt->fbtp_patchval = FBT_PATCHVAL;
 590         newfbt->fbtp_currentval = 0;
 591         newfbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
 592         fbt_probetab[FBT_ADDR2NDX(instr)] = newfbt;
 593
 594         if (doenable) {
 595                 fbt_enable(NULL, newfbt->fbtp_id, newfbt);
 596         }
 597
 598         instr++;
 599         goto again;
 600 }