bsd/dev/arm64/fbt_arm.c

   1 /*
   2  * Copyright (c) 2007-2018 Apple Inc. All rights reserved.
   3  */
   4 /*
   5  * CDDL HEADER START
   6  *
   7  * The contents of this file are subject to the terms of the
   8  * Common Development and Distribution License, Version 1.0 only
   9  * (the "License").  You may not use this file except in compliance
  10  * with the License.
  11  *
  12  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  13  * or http://www.opensolaris.org/os/licensing.
  14  * See the License for the specific language governing permissions
  15  * and limitations under the License.
  16  *
  17  * When distributing Covered Code, include this CDDL HEADER in each
  18  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  19  * If applicable, add the following below this CDDL HEADER, with the
  20  * fields enclosed by brackets "[]" replaced with your own identifying
  21  * information: Portions Copyright [yyyy] [name of copyright owner]
  22  *
  23  * CDDL HEADER END
  24  */
  25 /*
  26  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
  27  * Use is subject to license terms.
  28  */
  29
  30 #include <kern/thread.h>
  31 #include <mach/thread_status.h>
  32 #include <arm/proc_reg.h>
  33 #include <arm/caches_internal.h>
  34
  35 #include <mach-o/loader.h>
  36 #include <mach-o/nlist.h>
  37 #include <libkern/kernel_mach_header.h>
  38
  39 #include <sys/param.h>
  40 #include <sys/systm.h>
  41 #include <sys/errno.h>
  42 #include <sys/stat.h>
  43 #include <sys/ioctl.h>
  44 #include <sys/conf.h>
  45 #include <sys/fcntl.h>
  46 #include <miscfs/devfs/devfs.h>
  47
  48 #include <sys/dtrace.h>
  49 #include <sys/dtrace_impl.h>
  50 #include <sys/fbt.h>
  51
  52 #include <sys/dtrace_glue.h>
  53
  54 #if __has_include(<ptrauth.h>)
  55 #include <ptrauth.h>
  56 #endif
  57
  58 #define DTRACE_INVOP_PUSH_FRAME 11
  59
  60 #define DTRACE_INVOP_NOP_SKIP           4
  61 #define DTRACE_INVOP_ADD_FP_SP_SKIP     4
  62
  63 #define DTRACE_INVOP_POP_PC_SKIP 2
  64
  65 /*
  66  * stp  fp, lr, [sp, #val]
  67  * stp  fp, lr, [sp, #val]!
  68  */
  69 #define FBT_IS_ARM64_FRAME_PUSH(x)      \
  70         (((x) & 0xffc07fff) == 0xa9007bfd || ((x) & 0xffc07fff) == 0xa9807bfd)
  71
  72 /*
  73  * stp  Xt1, Xt2, [sp, #val]
  74  * stp  Xt1, Xt2, [sp, #val]!
  75  */
  76 #define FBT_IS_ARM64_PUSH(x)            \
  77         (((x) & 0xffc003e0) == 0xa90003e0 || ((x) & 0xffc003e0) == 0xa98003e0)
  78
  79 /*
  80  * ldp  fp, lr, [sp,  #val]
  81  * ldp  fp, lr, [sp], #val
  82  */
  83 #define FBT_IS_ARM64_FRAME_POP(x)       \
  84         (((x) & 0xffc07fff) == 0xa9407bfd || ((x) & 0xffc07fff) == 0xa8c07bfd)
  85
  86 #define FBT_IS_ARM64_ADD_FP_SP(x)       (((x) & 0xffc003ff) == 0x910003fd)      /* add fp, sp, #val  (add fp, sp, #0 == mov fp, sp) */
  87 #define FBT_IS_ARM64_RET(x)             (((x) == 0xd65f03c0) || ((x) == 0xd65f0fff))                    /* ret, retab */
  88
  89
  90 #define FBT_B_MASK                      0xff000000
  91 #define FBT_B_IMM_MASK                  0x00ffffff
  92 #define FBT_B_INSTR                     0x14000000
  93
  94 #define FBT_IS_ARM64_B_INSTR(x)         ((x & FBT_B_MASK) == FBT_B_INSTR)
  95 #define FBT_GET_ARM64_B_IMM(x)          ((x & FBT_B_IMM_MASK) << 2)
  96
  97 #define FBT_PATCHVAL                    0xe7eeee7e
  98 #define FBT_AFRAMES_ENTRY               7
  99 #define FBT_AFRAMES_RETURN              7
 100
 101 #define FBT_ENTRY       "entry"
 102 #define FBT_RETURN      "return"
 103 #define FBT_ADDR2NDX(addr)      ((((uintptr_t)(addr)) >> 4) & fbt_probetab_mask)
 104
 105 extern dtrace_provider_id_t     fbt_id;
 106 extern fbt_probe_t               **fbt_probetab;
 107 extern int                      fbt_probetab_mask;
 108
 109 kern_return_t fbt_perfCallback(int, struct arm_saved_state *, __unused int, __unused int);
 110
 111 int
 112 fbt_invop(uintptr_t addr, uintptr_t * stack, uintptr_t rval)
 113 {
 114         fbt_probe_t    *fbt = fbt_probetab[FBT_ADDR2NDX(addr)];
 115
 116         for (; fbt != NULL; fbt = fbt->fbtp_hashnext) {
 117                 if ((uintptr_t) fbt->fbtp_patchpoint == addr) {
 118                         if (0 == CPU->cpu_dtrace_invop_underway) {
 119                                 CPU->cpu_dtrace_invop_underway = 1;     /* Race not possible on
 120                                                                         * this per-cpu state */
 121
 122                                 if (fbt->fbtp_roffset == 0) {
 123                                         /*
 124                                          * Stack looks like this:
 125                                          *
 126                                          *      [Higher addresses]
 127                                          *
 128                                          *      Frame of caller
 129                                          *      Extra args for callee
 130                                          *      ------------------------
 131                                          *      Frame from traced function: <previous sp (e.g. 0x1000), return address>
 132                                          *      ------------------------
 133                                          *      arm_context_t
 134                                          *      ------------------------
 135                                          *      Frame from trap handler:  <previous sp (e.g. 0x1000) , traced PC >
 136                                          *                              The traced function never got to mov fp, sp,
 137                                          *                              so there is no frame in the backtrace pointing
 138                                          *                              to the frame on the stack containing the LR in the
 139                                          *                              caller.
 140                                          *      ------------------------
 141                                          *           |
 142                                          *           |
 143                                          *           |  stack grows this way
 144                                          *           |
 145                                          *           |
 146                                          *           v
 147                                          *      [Lower addresses]
 148                                          */
 149
 150                                         arm_saved_state_t *regs = (arm_saved_state_t *)(&((arm_context_t *)stack)->ss);
 151
 152                                         /*
 153                                          * cpu_dtrace_caller compensates for fact that the traced function never got to update its fp.
 154                                          * When walking the stack, when we reach the frame where we extract a PC in the patched
 155                                          * function, we put the cpu_dtrace_caller in the backtrace instead.  The next frame we extract
 156                                          * will be in the caller's caller, so we output a backtrace starting at the caller and going
 157                                          * sequentially up the stack.
 158                                          */
 159                                         CPU->cpu_dtrace_caller = get_saved_state_lr(regs);
 160                                         dtrace_probe(fbt->fbtp_id, get_saved_state_reg(regs, 0), get_saved_state_reg(regs, 1),
 161                                             get_saved_state_reg(regs, 2), get_saved_state_reg(regs, 3), get_saved_state_reg(regs, 4));
 162                                         CPU->cpu_dtrace_caller = 0;
 163                                 } else {
 164                                         /*
 165                                          * When fbtp_roffset is non-zero, we know we are handling a return probe point.
 166                                          *
 167                                          *
 168                                          * Stack looks like this, as we've already popped the frame in the traced callee, and
 169                                          * we trap with lr set to the return address in the caller.
 170                                          *      [Higher addresses]
 171                                          *
 172                                          *      Frame of caller
 173                                          *      Extra args for callee
 174                                          *      ------------------------
 175                                          *      arm_context_t
 176                                          *      ------------------------
 177                                          *      Frame from trap handler:  <sp at time of trap, traced PC >
 178                                          *      ------------------------
 179                                          *           |
 180                                          *           |
 181                                          *           |  stack grows this way
 182                                          *           |
 183                                          *           |
 184                                          *           v
 185                                          *      [Lower addresses]
 186                                          */
 187                                         arm_saved_state_t *regs = (arm_saved_state_t *)(&((arm_context_t *)stack)->ss);
 188
 189                                         CPU->cpu_dtrace_caller = get_saved_state_lr(regs);
 190                                         dtrace_probe(fbt->fbtp_id, fbt->fbtp_roffset, rval, 0, 0, 0);
 191                                         CPU->cpu_dtrace_caller = 0;
 192                                 }
 193                                 CPU->cpu_dtrace_invop_underway = 0;
 194                         }
 195
 196                         /*
 197                          *       On other architectures, we return a DTRACE constant to let the callback function
 198                          *       know what was replaced. On the ARM, since the function prologue/epilogue machine code
 199                          *       can vary, we need the actual bytes of the instruction, so return the savedval instead.
 200                          */
 201                         return fbt->fbtp_savedval;
 202                 }
 203         }
 204
 205         return 0;
 206 }
 207
 208 #define IS_USER_TRAP(regs)   (PSR64_IS_USER(get_saved_state_cpsr(regs)))
 209 #define T_INVALID_OPCODE EXC_BAD_INSTRUCTION
 210 #define FBT_EXCEPTION_CODE T_INVALID_OPCODE
 211
 212 kern_return_t
 213 fbt_perfCallback(
 214         int trapno,
 215         struct arm_saved_state * regs,
 216         __unused int unused1,
 217         __unused int unused2)
 218 {
 219         kern_return_t   retval = KERN_FAILURE;
 220
 221         if (FBT_EXCEPTION_CODE == trapno && !IS_USER_TRAP(regs)) {
 222                 boolean_t oldlevel = 0;
 223                 machine_inst_t emul = 0;
 224                 uint64_t sp, lr, imm;
 225
 226                 oldlevel = ml_set_interrupts_enabled(FALSE);
 227
 228                 __asm__ volatile (
 229                          "Ldtrace_invop_callsite_pre_label:\n"
 230                          ".data\n"
 231                          ".private_extern _dtrace_invop_callsite_pre\n"
 232                          "_dtrace_invop_callsite_pre:\n"
 233                          "  .quad Ldtrace_invop_callsite_pre_label\n"
 234                          ".text\n"
 235                 );
 236
 237                 emul = dtrace_invop(get_saved_state_pc(regs), (uintptr_t*) regs, get_saved_state_reg(regs, 0));
 238
 239                 __asm__ volatile (
 240                          "Ldtrace_invop_callsite_post_label:\n"
 241                          ".data\n"
 242                          ".private_extern _dtrace_invop_callsite_post\n"
 243                          "_dtrace_invop_callsite_post:\n"
 244                          "  .quad Ldtrace_invop_callsite_post_label\n"
 245                          ".text\n"
 246                 );
 247
 248                 if (emul == DTRACE_INVOP_NOP) {
 249                         /*
 250                          * Skip over the patched NOP planted by sdt
 251                          */
 252                         add_saved_state_pc(regs, DTRACE_INVOP_NOP_SKIP);
 253                         retval = KERN_SUCCESS;
 254                 } else if (FBT_IS_ARM64_ADD_FP_SP(emul)) {
 255                         /* retrieve the value to add */
 256                         uint64_t val = (emul >> 10) & 0xfff;
 257                         assert(val < 4096);
 258
 259                         /* retrieve sp */
 260                         sp = get_saved_state_sp(regs);
 261
 262                         /*
 263                          * emulate the instruction:
 264                          *      add     fp, sp, #val
 265                          */
 266                         assert(sp < (UINT64_MAX - val));
 267                         set_saved_state_fp(regs, sp + val);
 268
 269                         /* skip over the bytes of the patched instruction */
 270                         add_saved_state_pc(regs, DTRACE_INVOP_ADD_FP_SP_SKIP);
 271
 272                         retval = KERN_SUCCESS;
 273                 } else if (FBT_IS_ARM64_RET(emul)) {
 274                         lr = get_saved_state_lr(regs);
 275 #if __has_feature(ptrauth_calls)
 276                         lr = (user_addr_t) ptrauth_strip((void *)lr, ptrauth_key_return_address);
 277 #endif
 278                         set_saved_state_pc(regs, lr);
 279                         retval = KERN_SUCCESS;
 280                 } else if (FBT_IS_ARM64_B_INSTR(emul)) {
 281                         imm = FBT_GET_ARM64_B_IMM(emul);
 282                         add_saved_state_pc(regs, imm);
 283                         retval = KERN_SUCCESS;
 284                 } else if (emul == FBT_PATCHVAL) {
 285                         /* Means we encountered an error but handled it, try same inst again */
 286                         retval = KERN_SUCCESS;
 287                 } else {
 288                         retval = KERN_FAILURE;
 289                 }
 290
 291                 ml_set_interrupts_enabled(oldlevel);
 292         }
 293
 294         return retval;
 295 }
 296
 297 void
 298 fbt_provide_probe(struct modctl *ctl, const char *modname, const char* symbolName, machine_inst_t* symbolStart, machine_inst_t *instrHigh)
 299 {
 300         int             doenable = 0;
 301         dtrace_id_t     thisid;
 302
 303         fbt_probe_t     *newfbt, *retfbt, *entryfbt;
 304         machine_inst_t *instr, *pushinstr = NULL, *limit, theInstr;
 305         int             foundPushLR, savedRegs;
 306
 307         /*
 308          * Guard against null and invalid symbols
 309          */
 310         if (!symbolStart || !instrHigh || instrHigh < symbolStart) {
 311                 kprintf("dtrace: %s has an invalid address\n", symbolName);
 312                 return;
 313         }
 314
 315         /*
 316          * Assume the compiler doesn't schedule instructions in the prologue.
 317          */
 318         foundPushLR = 0;
 319         savedRegs = -1;
 320         limit = (machine_inst_t *)instrHigh;
 321
 322         assert(sizeof(*instr) == 4);
 323
 324         for (instr = symbolStart, theInstr = 0; instr < instrHigh; instr++) {
 325                 /*
 326                  * Count the number of time we pushed something onto the stack
 327                  * before hitting a frame push. That will give us an estimation
 328                  * of how many stack pops we should expect when looking for the
 329                  * RET instruction.
 330                  */
 331                 theInstr = *instr;
 332                 if (FBT_IS_ARM64_FRAME_PUSH(theInstr)) {
 333                         foundPushLR = 1;
 334                         pushinstr = instr;
 335                 }
 336
 337                 if (foundPushLR && (FBT_IS_ARM64_ADD_FP_SP(theInstr))) {
 338                         /* Guard against a random setting of fp from sp, we make sure we found the push first */
 339                         break;
 340                 }
 341                 if (FBT_IS_ARM64_RET(theInstr)) { /* We've gone too far, bail. */
 342                         break;
 343                 }
 344                 if (FBT_IS_ARM64_FRAME_POP(theInstr)) { /* We've gone too far, bail. */
 345                         break;
 346                 }
 347         }
 348
 349         if (!(foundPushLR && (FBT_IS_ARM64_ADD_FP_SP(theInstr)))) {
 350                 return;
 351         }
 352
 353         thisid = dtrace_probe_lookup(fbt_id, modname, symbolName, FBT_ENTRY);
 354         newfbt = kmem_zalloc(sizeof(fbt_probe_t), KM_SLEEP);
 355         newfbt->fbtp_next = NULL;
 356         strlcpy((char *)&(newfbt->fbtp_name), symbolName, MAX_FBTP_NAME_CHARS );
 357
 358         if (thisid != 0) {
 359                 /*
 360                  * The dtrace_probe previously existed, so we have to hook
 361                  * the newfbt entry onto the end of the existing fbt's
 362                  * chain.
 363                  * If we find an fbt entry that was previously patched to
 364                  * fire, (as indicated by the current patched value), then
 365                  * we want to enable this newfbt on the spot.
 366                  */
 367                 entryfbt = dtrace_probe_arg(fbt_id, thisid);
 368                 ASSERT(entryfbt != NULL);
 369                 for (; entryfbt != NULL; entryfbt = entryfbt->fbtp_next) {
 370                         if (entryfbt->fbtp_currentval == entryfbt->fbtp_patchval) {
 371                                 doenable++;
 372                         }
 373
 374                         if (entryfbt->fbtp_next == NULL) {
 375                                 entryfbt->fbtp_next = newfbt;
 376                                 newfbt->fbtp_id = entryfbt->fbtp_id;
 377                                 break;
 378                         }
 379                 }
 380         } else {
 381                 /*
 382                  * The dtrace_probe did not previously exist, so we
 383                  * create it and hook in the newfbt.  Since the probe is
 384                  * new, we obviously do not need to enable it on the spot.
 385                  */
 386                 newfbt->fbtp_id = dtrace_probe_create(fbt_id, modname, symbolName, FBT_ENTRY, FBT_AFRAMES_ENTRY, newfbt);
 387                 doenable = 0;
 388         }
 389
 390         newfbt->fbtp_patchpoint = instr;
 391         newfbt->fbtp_ctl = ctl;
 392         newfbt->fbtp_loadcnt = ctl->mod_loadcnt;
 393         newfbt->fbtp_rval = DTRACE_INVOP_PUSH_FRAME;
 394         newfbt->fbtp_savedval = theInstr;
 395         newfbt->fbtp_patchval = FBT_PATCHVAL;
 396         newfbt->fbtp_currentval = 0;
 397         newfbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
 398         fbt_probetab[FBT_ADDR2NDX(instr)] = newfbt;
 399
 400         if (doenable) {
 401                 fbt_enable(NULL, newfbt->fbtp_id, newfbt);
 402         }
 403
 404         /*
 405          * The fbt entry chain is in place, one entry point per symbol.
 406          * The fbt return chain can have multiple return points per
 407          * symbol.
 408          * Here we find the end of the fbt return chain.
 409          */
 410
 411         doenable = 0;
 412
 413         thisid = dtrace_probe_lookup(fbt_id, modname, symbolName, FBT_RETURN);
 414
 415         if (thisid != 0) {
 416                 /* The dtrace_probe previously existed, so we have to
 417                  * find the end of the existing fbt chain.  If we find
 418                  * an fbt return that was previously patched to fire,
 419                  * (as indicated by the currrent patched value), then
 420                  * we want to enable any new fbts on the spot.
 421                  */
 422                 retfbt = dtrace_probe_arg(fbt_id, thisid);
 423                 ASSERT(retfbt != NULL);
 424                 for (; retfbt != NULL; retfbt =  retfbt->fbtp_next) {
 425                         if (retfbt->fbtp_currentval == retfbt->fbtp_patchval) {
 426                                 doenable++;
 427                         }
 428                         if (retfbt->fbtp_next == NULL) {
 429                                 break;
 430                         }
 431                 }
 432         } else {
 433                 doenable = 0;
 434                 retfbt = NULL;
 435         }
 436
 437         /*
 438          * Go back to the start of the function, in case
 439          * the compiler emitted pcrel data loads
 440          * before FP was adjusted.
 441          */
 442         instr = pushinstr + 1;
 443 again:
 444         if (instr >= limit) {
 445                 return;
 446         }
 447
 448         /* XXX FIXME ... extra jump table detection? */
 449
 450         /*
 451          * OK, it's an instruction.
 452          */
 453         theInstr = *instr;
 454
 455         /* Walked onto the start of the next routine? If so, bail out from this function */
 456         if (FBT_IS_ARM64_FRAME_PUSH(theInstr)) {
 457                 if (!retfbt) {
 458                         kprintf("dtrace: fbt: No return probe for %s, walked to next routine at 0x%016llx\n", symbolName, (uint64_t)instr);
 459                 }
 460                 return;
 461         }
 462
 463         /* XXX fancy detection of end of function using PC-relative loads */
 464
 465         /*
 466          * Look for:
 467          *      ldp fp, lr, [sp], #val
 468          *      ldp fp, lr, [sp,  #val]
 469          */
 470         if (!FBT_IS_ARM64_FRAME_POP(theInstr)) {
 471                 instr++;
 472                 goto again;
 473         }
 474
 475         /* go to the next instruction */
 476         instr++;
 477
 478         /* Scan ahead for a ret or a branch outside the function */
 479         for (; instr < limit; instr++) {
 480                 theInstr = *instr;
 481                 if (FBT_IS_ARM64_RET(theInstr)) {
 482                         break;
 483                 }
 484                 if (FBT_IS_ARM64_B_INSTR(theInstr)) {
 485                         machine_inst_t *dest = instr + FBT_GET_ARM64_B_IMM(theInstr);
 486                         /*
 487                          * Check whether the destination of the branch
 488                          * is outside of the function
 489                          */
 490                         if (dest >= limit || dest < symbolStart) {
 491                                 break;
 492                         }
 493                 }
 494         }
 495
 496         if (!FBT_IS_ARM64_RET(theInstr) && !FBT_IS_ARM64_B_INSTR(theInstr)) {
 497                 return;
 498         }
 499
 500         newfbt = kmem_zalloc(sizeof(fbt_probe_t), KM_SLEEP);
 501         newfbt->fbtp_next = NULL;
 502         strlcpy((char *)&(newfbt->fbtp_name), symbolName, MAX_FBTP_NAME_CHARS );
 503
 504         if (retfbt == NULL) {
 505                 newfbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
 506                     symbolName, FBT_RETURN, FBT_AFRAMES_RETURN, newfbt);
 507         } else {
 508                 retfbt->fbtp_next = newfbt;
 509                 newfbt->fbtp_id = retfbt->fbtp_id;
 510         }
 511
 512         retfbt = newfbt;
 513         newfbt->fbtp_patchpoint = instr;
 514         newfbt->fbtp_ctl = ctl;
 515         newfbt->fbtp_loadcnt = ctl->mod_loadcnt;
 516
 517         ASSERT(FBT_IS_ARM64_RET(theInstr));
 518         newfbt->fbtp_rval = DTRACE_INVOP_RET;
 519         newfbt->fbtp_roffset = (uintptr_t) ((uint8_t*) instr - (uint8_t *)symbolStart);
 520         newfbt->fbtp_savedval = theInstr;
 521         newfbt->fbtp_patchval = FBT_PATCHVAL;
 522         newfbt->fbtp_currentval = 0;
 523         newfbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
 524         fbt_probetab[FBT_ADDR2NDX(instr)] = newfbt;
 525
 526         if (doenable) {
 527                 fbt_enable(NULL, newfbt->fbtp_id, newfbt);
 528         }
 529
 530         instr++;
 531         goto again;
 532 }