bsd/dev/arm/dtrace_isa.c

   1 /*
   2  * Copyright (c) 2005-2018 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <arm/caches_internal.h>
  30 #include <arm/proc_reg.h>
  31
  32 #include <kern/thread.h>
  33 #include <mach/thread_status.h>
  34
  35 #include <stdarg.h>
  36 #include <string.h>
  37 #include <sys/malloc.h>
  38 #include <sys/time.h>
  39 #include <sys/systm.h>
  40 #include <sys/proc.h>
  41 #include <sys/proc_internal.h>
  42 #include <sys/kauth.h>
  43 #include <sys/dtrace.h>
  44 #include <sys/dtrace_impl.h>
  45 #include <machine/atomic.h>
  46 #include <kern/simple_lock.h>
  47 #include <kern/sched_prim.h>            /* for thread_wakeup() */
  48 #include <kern/thread_call.h>
  49 #include <kern/task.h>
  50 #include <miscfs/devfs/devfs.h>
  51 #include <mach/vm_param.h>
  52
  53 extern struct arm_saved_state *find_kern_regs(thread_t);
  54
  55 extern dtrace_id_t      dtrace_probeid_error;   /* special ERROR probe */
  56 typedef arm_saved_state_t savearea_t;
  57
  58 extern lck_attr_t       *dtrace_lck_attr;
  59 extern lck_grp_t        *dtrace_lck_grp;
  60
  61 int dtrace_arm_condition_true(int condition, int cpsr);
  62
  63 /*
  64  * Atomicity and synchronization
  65  */
  66 inline void
  67 dtrace_membar_producer(void)
  68 {
  69         __asm__ volatile ("dmb ish" : : : "memory");
  70 }
  71
  72 inline void
  73 dtrace_membar_consumer(void)
  74 {
  75         __asm__ volatile ("dmb ish" : : : "memory");
  76 }
  77
  78 /*
  79  * Interrupt manipulation
  80  * XXX dtrace_getipl() can be called from probe context.
  81  */
  82 int
  83 dtrace_getipl(void)
  84 {
  85         /*
  86          * XXX Drat, get_interrupt_level is MACH_KERNEL_PRIVATE
  87          * in osfmk/kern/cpu_data.h
  88          */
  89         /* return get_interrupt_level(); */
  90         return ml_at_interrupt_context() ? 1 : 0;
  91 }
  92
  93 /*
  94  * MP coordination
  95  */
  96
  97 decl_lck_mtx_data(static, dt_xc_lock);
  98 static uint32_t dt_xc_sync;
  99
 100 typedef struct xcArg {
 101         processorid_t   cpu;
 102         dtrace_xcall_t  f;
 103         void           *arg;
 104 } xcArg_t;
 105
 106 static void
 107 xcRemote(void *foo)
 108 {
 109         xcArg_t *pArg = (xcArg_t *) foo;
 110
 111         if (pArg->cpu == CPU->cpu_id || pArg->cpu == DTRACE_CPUALL) {
 112                 (pArg->f)(pArg->arg);
 113         }
 114
 115         if (os_atomic_dec(&dt_xc_sync, relaxed) == 0) {
 116                 thread_wakeup((event_t) &dt_xc_sync);
 117         }
 118 }
 119
 120 /*
 121  * dtrace_xcall() is not called from probe context.
 122  */
 123 void
 124 dtrace_xcall(processorid_t cpu, dtrace_xcall_t f, void *arg)
 125 {
 126         /* Only one dtrace_xcall in flight allowed */
 127         lck_mtx_lock(&dt_xc_lock);
 128
 129         xcArg_t xcArg;
 130
 131         xcArg.cpu = cpu;
 132         xcArg.f = f;
 133         xcArg.arg = arg;
 134
 135         cpu_broadcast_xcall(&dt_xc_sync, TRUE, xcRemote, (void*) &xcArg);
 136
 137         lck_mtx_unlock(&dt_xc_lock);
 138         return;
 139 }
 140
 141 /*
 142  * Initialization
 143  */
 144 void
 145 dtrace_isa_init(void)
 146 {
 147         lck_mtx_init(&dt_xc_lock, dtrace_lck_grp, dtrace_lck_attr);
 148         return;
 149 }
 150
 151 /*
 152  * Runtime and ABI
 153  */
 154 uint64_t
 155 dtrace_getreg(struct regs * savearea, uint_t reg)
 156 {
 157         struct arm_saved_state *regs = (struct arm_saved_state *) savearea;
 158         if (regs == NULL) {
 159                 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
 160                 return 0;
 161         }
 162         /* beyond register limit? */
 163         if (reg > ARM_SAVED_STATE32_COUNT - 1) {
 164                 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
 165                 return 0;
 166         }
 167
 168         return (uint64_t) ((unsigned int *) (&(regs->r)))[reg];
 169 }
 170
 171 uint64_t
 172 dtrace_getvmreg(uint_t ndx)
 173 {
 174 #pragma unused(ndx)
 175         DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
 176         return 0;
 177 }
 178
 179 #define RETURN_OFFSET 4
 180
 181 static int
 182 dtrace_getustack_common(uint64_t * pcstack, int pcstack_limit, user_addr_t pc,
 183     user_addr_t sp)
 184 {
 185         volatile uint16_t *flags = (volatile uint16_t *) &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
 186         int ret = 0;
 187
 188         ASSERT(pcstack == NULL || pcstack_limit > 0);
 189
 190         while (pc != 0) {
 191                 ret++;
 192                 if (pcstack != NULL) {
 193                         *pcstack++ = (uint64_t) pc;
 194                         pcstack_limit--;
 195                         if (pcstack_limit <= 0) {
 196                                 break;
 197                         }
 198                 }
 199
 200                 if (sp == 0) {
 201                         break;
 202                 }
 203
 204                 pc = dtrace_fuword32((sp + RETURN_OFFSET));
 205                 sp = dtrace_fuword32(sp);
 206
 207                 /* Truncate ustack if the iterator causes fault. */
 208                 if (*flags & CPU_DTRACE_FAULT) {
 209                         *flags &= ~CPU_DTRACE_FAULT;
 210                         break;
 211                 }
 212         }
 213
 214         return ret;
 215 }
 216
 217 void
 218 dtrace_getupcstack(uint64_t * pcstack, int pcstack_limit)
 219 {
 220         thread_t        thread = current_thread();
 221         savearea_t     *regs;
 222         user_addr_t     pc, sp;
 223         volatile uint16_t *flags = (volatile uint16_t *) &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
 224         int n;
 225
 226         if (*flags & CPU_DTRACE_FAULT) {
 227                 return;
 228         }
 229
 230         if (pcstack_limit <= 0) {
 231                 return;
 232         }
 233
 234         /*
 235          * If there's no user context we still need to zero the stack.
 236          */
 237         if (thread == NULL) {
 238                 goto zero;
 239         }
 240
 241         regs = (savearea_t *) find_user_regs(thread);
 242         if (regs == NULL) {
 243                 goto zero;
 244         }
 245
 246         *pcstack++ = (uint64_t)dtrace_proc_selfpid();
 247         pcstack_limit--;
 248
 249         if (pcstack_limit <= 0) {
 250                 return;
 251         }
 252
 253         pc = regs->pc;
 254         sp = regs->sp;
 255
 256         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
 257                 *pcstack++ = (uint64_t) pc;
 258                 pcstack_limit--;
 259                 if (pcstack_limit <= 0) {
 260                         return;
 261                 }
 262
 263                 pc = regs->lr;
 264         }
 265
 266         n = dtrace_getustack_common(pcstack, pcstack_limit, pc, regs->r[7]);
 267
 268         ASSERT(n >= 0);
 269         ASSERT(n <= pcstack_limit);
 270
 271         pcstack += n;
 272         pcstack_limit -= n;
 273
 274 zero:
 275         while (pcstack_limit-- > 0) {
 276                 *pcstack++ = 0ULL;
 277         }
 278 }
 279
 280 int
 281 dtrace_getustackdepth(void)
 282 {
 283         thread_t        thread = current_thread();
 284         savearea_t     *regs;
 285         user_addr_t     pc, sp;
 286         int             n = 0;
 287
 288         if (thread == NULL) {
 289                 return 0;
 290         }
 291
 292         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT)) {
 293                 return -1;
 294         }
 295
 296         regs = (savearea_t *) find_user_regs(thread);
 297         if (regs == NULL) {
 298                 return 0;
 299         }
 300
 301         pc = regs->pc;
 302         sp = regs->sp;
 303
 304         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
 305                 n++;
 306                 pc = regs->lr;
 307         }
 308
 309         /*
 310          * Note that unlike ppc, the arm code does not use
 311          * CPU_DTRACE_USTACK_FP. This is because arm always
 312          * traces from the sp, even in syscall/profile/fbt
 313          * providers.
 314          */
 315
 316         n += dtrace_getustack_common(NULL, 0, pc, regs->r[7]);
 317
 318         return n;
 319 }
 320
 321 void
 322 dtrace_getufpstack(uint64_t * pcstack, uint64_t * fpstack, int pcstack_limit)
 323 {
 324         /* XXX ARMTODO 64vs32 */
 325         thread_t        thread = current_thread();
 326         savearea_t      *regs;
 327         user_addr_t     pc, sp;
 328
 329         volatile        uint16_t  *flags = (volatile uint16_t *) &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
 330
 331 #if 0
 332         uintptr_t oldcontext;
 333         size_t          s1, s2;
 334 #endif
 335
 336         if (*flags & CPU_DTRACE_FAULT) {
 337                 return;
 338         }
 339
 340         if (pcstack_limit <= 0) {
 341                 return;
 342         }
 343
 344         /*
 345          * If there's no user context we still need to zero the stack.
 346          */
 347         if (thread == NULL) {
 348                 goto zero;
 349         }
 350
 351         regs = (savearea_t *) find_user_regs(thread);
 352         if (regs == NULL) {
 353                 goto zero;
 354         }
 355
 356         *pcstack++ = (uint64_t)dtrace_proc_selfpid();
 357         pcstack_limit--;
 358
 359         if (pcstack_limit <= 0) {
 360                 return;
 361         }
 362
 363         pc = regs->pc;
 364         sp = regs->sp;
 365
 366 #if 0                           /* XXX signal stack crawl */
 367         oldcontext = lwp->lwp_oldcontext;
 368
 369         if (p->p_model == DATAMODEL_NATIVE) {
 370                 s1 = sizeof(struct frame) + 2 * sizeof(long);
 371                 s2 = s1 + sizeof(siginfo_t);
 372         } else {
 373                 s1 = sizeof(struct frame32) + 3 * sizeof(int);
 374                 s2 = s1 + sizeof(siginfo32_t);
 375         }
 376 #endif
 377
 378         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
 379                 *pcstack++ = (uint64_t) pc;
 380                 *fpstack++ = 0;
 381                 pcstack_limit--;
 382                 if (pcstack_limit <= 0) {
 383                         return;
 384                 }
 385
 386                 pc = dtrace_fuword32(sp);
 387         }
 388         while (pc != 0 && sp != 0) {
 389                 *pcstack++ = (uint64_t) pc;
 390                 *fpstack++ = sp;
 391                 pcstack_limit--;
 392                 if (pcstack_limit <= 0) {
 393                         break;
 394                 }
 395
 396 #if 0                           /* XXX signal stack crawl */
 397                 if (oldcontext == sp + s1 || oldcontext == sp + s2) {
 398                         if (p->p_model == DATAMODEL_NATIVE) {
 399                                 ucontext_t     *ucp = (ucontext_t *) oldcontext;
 400                                 greg_t         *gregs = ucp->uc_mcontext.gregs;
 401
 402                                 sp = dtrace_fulword(&gregs[REG_FP]);
 403                                 pc = dtrace_fulword(&gregs[REG_PC]);
 404
 405                                 oldcontext = dtrace_fulword(&ucp->uc_link);
 406                         } else {
 407                                 ucontext_t     *ucp = (ucontext_t *) oldcontext;
 408                                 greg_t         *gregs = ucp->uc_mcontext.gregs;
 409
 410                                 sp = dtrace_fuword32(&gregs[EBP]);
 411                                 pc = dtrace_fuword32(&gregs[EIP]);
 412
 413                                 oldcontext = dtrace_fuword32(&ucp->uc_link);
 414                         }
 415                 } else
 416 #endif
 417                 {
 418                         pc = dtrace_fuword32((sp + RETURN_OFFSET));
 419                         sp = dtrace_fuword32(sp);
 420                 }
 421
 422                 /* Truncate ustack if the iterator causes fault. */
 423                 if (*flags & CPU_DTRACE_FAULT) {
 424                         *flags &= ~CPU_DTRACE_FAULT;
 425                         break;
 426                 }
 427         }
 428
 429 zero:
 430         while (pcstack_limit-- > 0) {
 431                 *pcstack++ = 0ULL;
 432         }
 433 }
 434
 435 void
 436 dtrace_getpcstack(pc_t * pcstack, int pcstack_limit, int aframes,
 437     uint32_t * intrpc)
 438 {
 439         struct frame   *fp = (struct frame *) __builtin_frame_address(0);
 440         struct frame   *nextfp, *minfp, *stacktop;
 441         int             depth = 0;
 442         int             on_intr;
 443         int             last = 0;
 444         uintptr_t       pc;
 445         uintptr_t       caller = CPU->cpu_dtrace_caller;
 446
 447         if ((on_intr = CPU_ON_INTR(CPU)) != 0) {
 448                 stacktop = (struct frame *) dtrace_get_cpu_int_stack_top();
 449         } else {
 450                 stacktop = (struct frame *) (dtrace_get_kernel_stack(current_thread()) + kernel_stack_size);
 451         }
 452
 453         minfp = fp;
 454
 455         aframes++;
 456
 457         if (intrpc != NULL && depth < pcstack_limit) {
 458                 pcstack[depth++] = (pc_t) intrpc;
 459         }
 460
 461         while (depth < pcstack_limit) {
 462                 nextfp = *(struct frame **) fp;
 463                 pc = *(uintptr_t *) (((uint32_t) fp) + RETURN_OFFSET);
 464
 465                 if (nextfp <= minfp || nextfp >= stacktop) {
 466                         if (on_intr) {
 467                                 /*
 468                                  * Hop from interrupt stack to thread stack.
 469                                  */
 470                                 arm_saved_state_t *arm_kern_regs = (arm_saved_state_t *) find_kern_regs(current_thread());
 471                                 if (arm_kern_regs) {
 472                                         nextfp = (struct frame *)arm_kern_regs->r[7];
 473
 474                                         vm_offset_t kstack_base = dtrace_get_kernel_stack(current_thread());
 475
 476                                         minfp = (struct frame *)kstack_base;
 477                                         stacktop = (struct frame *)(kstack_base + kernel_stack_size);
 478
 479                                         on_intr = 0;
 480
 481                                         if (nextfp <= minfp || nextfp >= stacktop) {
 482                                                 last = 1;
 483                                         }
 484                                 } else {
 485                                         /*
 486                                          * If this thread was on the interrupt stack, but did not
 487                                          * take an interrupt (i.e, the idle thread), there is no
 488                                          * explicit saved state for us to use.
 489                                          */
 490                                         last = 1;
 491                                 }
 492                         } else {
 493                                 /*
 494                                  * This is the last frame we can process; indicate
 495                                  * that we should return after processing this frame.
 496                                  */
 497                                 last = 1;
 498                         }
 499                 }
 500                 if (aframes > 0) {
 501                         if (--aframes == 0 && caller != (uintptr_t)NULL) {
 502                                 /*
 503                                  * We've just run out of artificial frames,
 504                                  * and we have a valid caller -- fill it in
 505                                  * now.
 506                                  */
 507                                 ASSERT(depth < pcstack_limit);
 508                                 pcstack[depth++] = (pc_t) caller;
 509                                 caller = (uintptr_t)NULL;
 510                         }
 511                 } else {
 512                         if (depth < pcstack_limit) {
 513                                 pcstack[depth++] = (pc_t) pc;
 514                         }
 515                 }
 516
 517                 if (last) {
 518                         while (depth < pcstack_limit) {
 519                                 pcstack[depth++] = (pc_t) NULL;
 520                         }
 521                         return;
 522                 }
 523                 fp = nextfp;
 524                 minfp = fp;
 525         }
 526 }
 527
 528 int
 529 dtrace_instr_size(uint32_t instr, int thumb_mode)
 530 {
 531         if (thumb_mode) {
 532                 uint16_t instr16 = *(uint16_t*) &instr;
 533                 if (((instr16 >> 11) & 0x1F) > 0x1C) {
 534                         return 4;
 535                 } else {
 536                         return 2;
 537                 }
 538         } else {
 539                 return 4;
 540         }
 541 }
 542
 543 uint64_t
 544 dtrace_getarg(int arg, int aframes, dtrace_mstate_t *mstate, dtrace_vstate_t *vstate)
 545 {
 546 #pragma unused(arg, aframes, mstate, vstate)
 547 #if 0
 548         /* XXX ARMTODO */
 549         uint64_t val;
 550         uintptr_t *fp = (uintptr_t *)__builtin_frame_address(0);
 551         uintptr_t *stack;
 552         uintptr_t pc;
 553         int i;
 554
 555         for (i = 1; i <= aframes; i++) {
 556                 fp = fp[0];
 557                 pc = fp[1];
 558
 559                 if (dtrace_invop_callsite_pre != NULL
 560                     && pc > (uintptr_t)dtrace_invop_callsite_pre
 561                     && pc <= (uintptr_t)dtrace_invop_callsite_post) {
 562                         /*
 563                          * If we pass through the invalid op handler, we will
 564                          * use the pointer that it passed to the stack as the
 565                          * second argument to dtrace_invop() as the pointer to
 566                          * the frame we're hunting for.
 567                          */
 568
 569                         stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */
 570                         fp = (struct frame *)stack[1]; /* Grab *second* argument */
 571                         stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */
 572                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
 573                         val = (uint64_t)(stack[arg]);
 574                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
 575                         return val;
 576                 }
 577         }
 578
 579         /*
 580          * Arrive here when provider has called dtrace_probe directly.
 581          */
 582         stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */
 583         stack++; /* Advance past probeID */
 584
 585         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
 586         val = *(((uint64_t *)stack) + arg); /* dtrace_probe arguments arg0 .. arg4 are 64bits wide */
 587         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
 588         return val;
 589 #endif
 590         return 0xfeedfacedeafbeadLL;
 591 }
 592
 593 void
 594 dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which,
 595     int fltoffs, int fault, uint64_t illval)
 596 {
 597         /* XXX ARMTODO */
 598         /*
 599          * For the case of the error probe firing lets
 600          * stash away "illval" here, and special-case retrieving it in DIF_VARIABLE_ARG.
 601          */
 602         state->dts_arg_error_illval = illval;
 603         dtrace_probe( dtrace_probeid_error, (uint64_t)(uintptr_t)state, epid, which, fltoffs, fault );
 604 }
 605
 606 void
 607 dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit))
 608 {
 609         /* XXX ARMTODO check copied from ppc/x86*/
 610         /*
 611          * "base" is the smallest toxic address in the range, "limit" is the first
 612          * VALID address greater than "base".
 613          */
 614         func(0x0, VM_MIN_KERNEL_ADDRESS);
 615         if (VM_MAX_KERNEL_ADDRESS < ~(uintptr_t)0) {
 616                 func(VM_MAX_KERNEL_ADDRESS + 1, ~(uintptr_t)0);
 617         }
 618 }
 619
 620 int
 621 dtrace_arm_condition_true(int cond, int cpsr)
 622 {
 623         int taken = 0;
 624         int zf = (cpsr & PSR_ZF) ? 1 : 0,
 625             nf = (cpsr & PSR_NF) ? 1 : 0,
 626             cf = (cpsr & PSR_CF) ? 1 : 0,
 627             vf = (cpsr & PSR_VF) ? 1 : 0;
 628
 629         switch (cond) {
 630         case 0: taken = zf; break;
 631         case 1: taken = !zf; break;
 632         case 2: taken = cf; break;
 633         case 3: taken = !cf; break;
 634         case 4: taken = nf; break;
 635         case 5: taken = !nf; break;
 636         case 6: taken = vf; break;
 637         case 7: taken = !vf; break;
 638         case 8: taken = (cf && !zf); break;
 639         case 9: taken = (!cf || zf); break;
 640         case 10: taken = (nf == vf); break;
 641         case 11: taken = (nf != vf); break;
 642         case 12: taken = (!zf && (nf == vf)); break;
 643         case 13: taken = (zf || (nf != vf)); break;
 644         case 14: taken = 1; break;
 645         case 15: taken = 1; break;         /* always "true" for ARM, unpredictable for THUMB. */
 646         }
 647
 648         return taken;
 649 }
 650
 651 void
 652 dtrace_flush_caches(void)
 653 {
 654         /* TODO There were some problems with flushing just the cache line that had been modified.
 655          * For now, we'll flush the entire cache, until we figure out how to flush just the patched block.
 656          */
 657         FlushPoU_Dcache();
 658         InvalidatePoU_Icache();
 659 }