bsd/dev/arm/dtrace_isa.c

   1 /*
   2  * Copyright (c) 2005-2018 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <arm/caches_internal.h>
  30 #include <arm/proc_reg.h>
  31
  32 #include <kern/thread.h>
  33 #include <mach/thread_status.h>
  34
  35 #include <stdarg.h>
  36 #include <string.h>
  37 #include <sys/malloc.h>
  38 #include <sys/time.h>
  39 #include <sys/systm.h>
  40 #include <sys/proc.h>
  41 #include <sys/proc_internal.h>
  42 #include <sys/kauth.h>
  43 #include <sys/dtrace.h>
  44 #include <sys/dtrace_impl.h>
  45 #include <machine/atomic.h>
  46 #include <kern/simple_lock.h>
  47 #include <kern/sched_prim.h>            /* for thread_wakeup() */
  48 #include <kern/thread_call.h>
  49 #include <kern/task.h>
  50 #include <miscfs/devfs/devfs.h>
  51 #include <mach/vm_param.h>
  52
  53 extern struct arm_saved_state *find_kern_regs(thread_t);
  54
  55 extern dtrace_id_t      dtrace_probeid_error;   /* special ERROR probe */
  56 typedef arm_saved_state_t savearea_t;
  57
  58 extern lck_attr_t       *dtrace_lck_attr;
  59 extern lck_grp_t        *dtrace_lck_grp;
  60
  61 int dtrace_arm_condition_true(int condition, int cpsr);
  62
  63 /*
  64  * Atomicity and synchronization
  65  */
  66 inline void
  67 dtrace_membar_producer(void)
  68 {
  69 #if __ARM_SMP__
  70         __asm__ volatile ("dmb ish" : : : "memory");
  71 #else
  72         __asm__ volatile ("nop" : : : "memory");
  73 #endif
  74 }
  75
  76 inline void
  77 dtrace_membar_consumer(void)
  78 {
  79 #if __ARM_SMP__
  80         __asm__ volatile ("dmb ish" : : : "memory");
  81 #else
  82         __asm__ volatile ("nop" : : : "memory");
  83 #endif
  84 }
  85
  86 /*
  87  * Interrupt manipulation
  88  * XXX dtrace_getipl() can be called from probe context.
  89  */
  90 int
  91 dtrace_getipl(void)
  92 {
  93         /*
  94          * XXX Drat, get_interrupt_level is MACH_KERNEL_PRIVATE
  95          * in osfmk/kern/cpu_data.h
  96          */
  97         /* return get_interrupt_level(); */
  98         return ml_at_interrupt_context() ? 1 : 0;
  99 }
 100
 101 #if __ARM_SMP__
 102 /*
 103  * MP coordination
 104  */
 105
 106 decl_lck_mtx_data(static, dt_xc_lock);
 107 static uint32_t dt_xc_sync;
 108
 109 typedef struct xcArg {
 110         processorid_t   cpu;
 111         dtrace_xcall_t  f;
 112         void           *arg;
 113 } xcArg_t;
 114
 115 static void
 116 xcRemote(void *foo)
 117 {
 118         xcArg_t *pArg = (xcArg_t *) foo;
 119
 120         if (pArg->cpu == CPU->cpu_id || pArg->cpu == DTRACE_CPUALL) {
 121                 (pArg->f)(pArg->arg);
 122         }
 123
 124         if (os_atomic_dec(&dt_xc_sync, relaxed) == 0) {
 125                 thread_wakeup((event_t) &dt_xc_sync);
 126         }
 127 }
 128 #endif
 129
 130 /*
 131  * dtrace_xcall() is not called from probe context.
 132  */
 133 void
 134 dtrace_xcall(processorid_t cpu, dtrace_xcall_t f, void *arg)
 135 {
 136 #if __ARM_SMP__
 137         /* Only one dtrace_xcall in flight allowed */
 138         lck_mtx_lock(&dt_xc_lock);
 139
 140         xcArg_t xcArg;
 141
 142         xcArg.cpu = cpu;
 143         xcArg.f = f;
 144         xcArg.arg = arg;
 145
 146         cpu_broadcast_xcall(&dt_xc_sync, TRUE, xcRemote, (void*) &xcArg);
 147
 148         lck_mtx_unlock(&dt_xc_lock);
 149         return;
 150 #else
 151 #pragma unused(cpu)
 152         /* On uniprocessor systems, the cpu should always be either ourselves or all */
 153         ASSERT(cpu == CPU->cpu_id || cpu == DTRACE_CPUALL);
 154
 155         (*f)(arg);
 156         return;
 157 #endif
 158 }
 159
 160 /*
 161  * Initialization
 162  */
 163 void
 164 dtrace_isa_init(void)
 165 {
 166 #if __ARM_SMP__
 167         lck_mtx_init(&dt_xc_lock, dtrace_lck_grp, dtrace_lck_attr);
 168 #endif
 169         return;
 170 }
 171
 172 /*
 173  * Runtime and ABI
 174  */
 175 uint64_t
 176 dtrace_getreg(struct regs * savearea, uint_t reg)
 177 {
 178         struct arm_saved_state *regs = (struct arm_saved_state *) savearea;
 179         if (regs == NULL) {
 180                 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
 181                 return 0;
 182         }
 183         /* beyond register limit? */
 184         if (reg > ARM_SAVED_STATE32_COUNT - 1) {
 185                 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
 186                 return 0;
 187         }
 188
 189         return (uint64_t) ((unsigned int *) (&(regs->r)))[reg];
 190 }
 191
 192 #define RETURN_OFFSET 4
 193
 194 static int
 195 dtrace_getustack_common(uint64_t * pcstack, int pcstack_limit, user_addr_t pc,
 196     user_addr_t sp)
 197 {
 198         int ret = 0;
 199
 200         ASSERT(pcstack == NULL || pcstack_limit > 0);
 201
 202         while (pc != 0) {
 203                 ret++;
 204                 if (pcstack != NULL) {
 205                         *pcstack++ = (uint64_t) pc;
 206                         pcstack_limit--;
 207                         if (pcstack_limit <= 0) {
 208                                 break;
 209                         }
 210                 }
 211
 212                 if (sp == 0) {
 213                         break;
 214                 }
 215
 216                 pc = dtrace_fuword32((sp + RETURN_OFFSET));
 217                 sp = dtrace_fuword32(sp);
 218         }
 219
 220         return ret;
 221 }
 222
 223 void
 224 dtrace_getupcstack(uint64_t * pcstack, int pcstack_limit)
 225 {
 226         thread_t        thread = current_thread();
 227         savearea_t     *regs;
 228         user_addr_t     pc, sp;
 229         volatile uint16_t *flags = (volatile uint16_t *) &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
 230         int n;
 231
 232         if (*flags & CPU_DTRACE_FAULT) {
 233                 return;
 234         }
 235
 236         if (pcstack_limit <= 0) {
 237                 return;
 238         }
 239
 240         /*
 241          * If there's no user context we still need to zero the stack.
 242          */
 243         if (thread == NULL) {
 244                 goto zero;
 245         }
 246
 247         regs = (savearea_t *) find_user_regs(thread);
 248         if (regs == NULL) {
 249                 goto zero;
 250         }
 251
 252         *pcstack++ = (uint64_t)dtrace_proc_selfpid();
 253         pcstack_limit--;
 254
 255         if (pcstack_limit <= 0) {
 256                 return;
 257         }
 258
 259         pc = regs->pc;
 260         sp = regs->sp;
 261
 262         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
 263                 *pcstack++ = (uint64_t) pc;
 264                 pcstack_limit--;
 265                 if (pcstack_limit <= 0) {
 266                         return;
 267                 }
 268
 269                 pc = regs->lr;
 270         }
 271
 272         n = dtrace_getustack_common(pcstack, pcstack_limit, pc, regs->r[7]);
 273
 274         ASSERT(n >= 0);
 275         ASSERT(n <= pcstack_limit);
 276
 277         pcstack += n;
 278         pcstack_limit -= n;
 279
 280 zero:
 281         while (pcstack_limit-- > 0) {
 282                 *pcstack++ = 0ULL;
 283         }
 284 }
 285
 286 int
 287 dtrace_getustackdepth(void)
 288 {
 289         thread_t        thread = current_thread();
 290         savearea_t     *regs;
 291         user_addr_t     pc, sp;
 292         int             n = 0;
 293
 294         if (thread == NULL) {
 295                 return 0;
 296         }
 297
 298         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT)) {
 299                 return -1;
 300         }
 301
 302         regs = (savearea_t *) find_user_regs(thread);
 303         if (regs == NULL) {
 304                 return 0;
 305         }
 306
 307         pc = regs->pc;
 308         sp = regs->sp;
 309
 310         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
 311                 n++;
 312                 pc = regs->lr;
 313         }
 314
 315         /*
 316          * Note that unlike ppc, the arm code does not use
 317          * CPU_DTRACE_USTACK_FP. This is because arm always
 318          * traces from the sp, even in syscall/profile/fbt
 319          * providers.
 320          */
 321
 322         n += dtrace_getustack_common(NULL, 0, pc, regs->r[7]);
 323
 324         return n;
 325 }
 326
 327 void
 328 dtrace_getufpstack(uint64_t * pcstack, uint64_t * fpstack, int pcstack_limit)
 329 {
 330         /* XXX ARMTODO 64vs32 */
 331         thread_t        thread = current_thread();
 332         savearea_t      *regs;
 333         user_addr_t     pc, sp;
 334
 335         volatile        uint16_t  *flags = (volatile uint16_t *) &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
 336
 337 #if 0
 338         uintptr_t oldcontext;
 339         size_t          s1, s2;
 340 #endif
 341
 342         if (*flags & CPU_DTRACE_FAULT) {
 343                 return;
 344         }
 345
 346         if (pcstack_limit <= 0) {
 347                 return;
 348         }
 349
 350         /*
 351          * If there's no user context we still need to zero the stack.
 352          */
 353         if (thread == NULL) {
 354                 goto zero;
 355         }
 356
 357         regs = (savearea_t *) find_user_regs(thread);
 358         if (regs == NULL) {
 359                 goto zero;
 360         }
 361
 362         *pcstack++ = (uint64_t)dtrace_proc_selfpid();
 363         pcstack_limit--;
 364
 365         if (pcstack_limit <= 0) {
 366                 return;
 367         }
 368
 369         pc = regs->pc;
 370         sp = regs->sp;
 371
 372 #if 0                           /* XXX signal stack crawl */
 373         oldcontext = lwp->lwp_oldcontext;
 374
 375         if (p->p_model == DATAMODEL_NATIVE) {
 376                 s1 = sizeof(struct frame) + 2 * sizeof(long);
 377                 s2 = s1 + sizeof(siginfo_t);
 378         } else {
 379                 s1 = sizeof(struct frame32) + 3 * sizeof(int);
 380                 s2 = s1 + sizeof(siginfo32_t);
 381         }
 382 #endif
 383
 384         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
 385                 *pcstack++ = (uint64_t) pc;
 386                 *fpstack++ = 0;
 387                 pcstack_limit--;
 388                 if (pcstack_limit <= 0) {
 389                         return;
 390                 }
 391
 392                 pc = dtrace_fuword32(sp);
 393         }
 394         while (pc != 0 && sp != 0) {
 395                 *pcstack++ = (uint64_t) pc;
 396                 *fpstack++ = sp;
 397                 pcstack_limit--;
 398                 if (pcstack_limit <= 0) {
 399                         break;
 400                 }
 401
 402 #if 0                           /* XXX signal stack crawl */
 403                 if (oldcontext == sp + s1 || oldcontext == sp + s2) {
 404                         if (p->p_model == DATAMODEL_NATIVE) {
 405                                 ucontext_t     *ucp = (ucontext_t *) oldcontext;
 406                                 greg_t         *gregs = ucp->uc_mcontext.gregs;
 407
 408                                 sp = dtrace_fulword(&gregs[REG_FP]);
 409                                 pc = dtrace_fulword(&gregs[REG_PC]);
 410
 411                                 oldcontext = dtrace_fulword(&ucp->uc_link);
 412                         } else {
 413                                 ucontext_t     *ucp = (ucontext_t *) oldcontext;
 414                                 greg_t         *gregs = ucp->uc_mcontext.gregs;
 415
 416                                 sp = dtrace_fuword32(&gregs[EBP]);
 417                                 pc = dtrace_fuword32(&gregs[EIP]);
 418
 419                                 oldcontext = dtrace_fuword32(&ucp->uc_link);
 420                         }
 421                 } else
 422 #endif
 423                 {
 424                         pc = dtrace_fuword32((sp + RETURN_OFFSET));
 425                         sp = dtrace_fuword32(sp);
 426                 }
 427
 428 #if 0
 429                 /* XXX ARMTODO*/
 430                 /*
 431                  * This is totally bogus:  if we faulted, we're going to clear
 432                  * the fault and break.  This is to deal with the apparently
 433                  * broken Java stacks on x86.
 434                  */
 435                 if (*flags & CPU_DTRACE_FAULT) {
 436                         *flags &= ~CPU_DTRACE_FAULT;
 437                         break;
 438                 }
 439 #endif
 440         }
 441
 442 zero:
 443         while (pcstack_limit-- > 0) {
 444                 *pcstack++ = 0ULL;
 445         }
 446 }
 447
 448 void
 449 dtrace_getpcstack(pc_t * pcstack, int pcstack_limit, int aframes,
 450     uint32_t * intrpc)
 451 {
 452         struct frame   *fp = (struct frame *) __builtin_frame_address(0);
 453         struct frame   *nextfp, *minfp, *stacktop;
 454         int             depth = 0;
 455         int             on_intr;
 456         int             last = 0;
 457         uintptr_t       pc;
 458         uintptr_t       caller = CPU->cpu_dtrace_caller;
 459
 460         if ((on_intr = CPU_ON_INTR(CPU)) != 0) {
 461                 stacktop = (struct frame *) dtrace_get_cpu_int_stack_top();
 462         } else {
 463                 stacktop = (struct frame *) (dtrace_get_kernel_stack(current_thread()) + kernel_stack_size);
 464         }
 465
 466         minfp = fp;
 467
 468         aframes++;
 469
 470         if (intrpc != NULL && depth < pcstack_limit) {
 471                 pcstack[depth++] = (pc_t) intrpc;
 472         }
 473
 474         while (depth < pcstack_limit) {
 475                 nextfp = *(struct frame **) fp;
 476                 pc = *(uintptr_t *) (((uint32_t) fp) + RETURN_OFFSET);
 477
 478                 if (nextfp <= minfp || nextfp >= stacktop) {
 479                         if (on_intr) {
 480                                 /*
 481                                  * Hop from interrupt stack to thread stack.
 482                                  */
 483                                 arm_saved_state_t *arm_kern_regs = (arm_saved_state_t *) find_kern_regs(current_thread());
 484                                 if (arm_kern_regs) {
 485                                         nextfp = (struct frame *)arm_kern_regs->r[7];
 486
 487                                         vm_offset_t kstack_base = dtrace_get_kernel_stack(current_thread());
 488
 489                                         minfp = (struct frame *)kstack_base;
 490                                         stacktop = (struct frame *)(kstack_base + kernel_stack_size);
 491
 492                                         on_intr = 0;
 493
 494                                         if (nextfp <= minfp || nextfp >= stacktop) {
 495                                                 last = 1;
 496                                         }
 497                                 } else {
 498                                         /*
 499                                          * If this thread was on the interrupt stack, but did not
 500                                          * take an interrupt (i.e, the idle thread), there is no
 501                                          * explicit saved state for us to use.
 502                                          */
 503                                         last = 1;
 504                                 }
 505                         } else {
 506                                 /*
 507                                  * This is the last frame we can process; indicate
 508                                  * that we should return after processing this frame.
 509                                  */
 510                                 last = 1;
 511                         }
 512                 }
 513                 if (aframes > 0) {
 514                         if (--aframes == 0 && caller != (uintptr_t)NULL) {
 515                                 /*
 516                                  * We've just run out of artificial frames,
 517                                  * and we have a valid caller -- fill it in
 518                                  * now.
 519                                  */
 520                                 ASSERT(depth < pcstack_limit);
 521                                 pcstack[depth++] = (pc_t) caller;
 522                                 caller = (uintptr_t)NULL;
 523                         }
 524                 } else {
 525                         if (depth < pcstack_limit) {
 526                                 pcstack[depth++] = (pc_t) pc;
 527                         }
 528                 }
 529
 530                 if (last) {
 531                         while (depth < pcstack_limit) {
 532                                 pcstack[depth++] = (pc_t) NULL;
 533                         }
 534                         return;
 535                 }
 536                 fp = nextfp;
 537                 minfp = fp;
 538         }
 539 }
 540
 541 int
 542 dtrace_instr_size(uint32_t instr, int thumb_mode)
 543 {
 544         if (thumb_mode) {
 545                 uint16_t instr16 = *(uint16_t*) &instr;
 546                 if (((instr16 >> 11) & 0x1F) > 0x1C) {
 547                         return 4;
 548                 } else {
 549                         return 2;
 550                 }
 551         } else {
 552                 return 4;
 553         }
 554 }
 555
 556 uint64_t
 557 dtrace_getarg(int arg, int aframes, dtrace_mstate_t *mstate, dtrace_vstate_t *vstate)
 558 {
 559 #pragma unused(arg, aframes, mstate, vstate)
 560 #if 0
 561         /* XXX ARMTODO */
 562         uint64_t val;
 563         uintptr_t *fp = (uintptr_t *)__builtin_frame_address(0);
 564         uintptr_t *stack;
 565         uintptr_t pc;
 566         int i;
 567
 568         for (i = 1; i <= aframes; i++) {
 569                 fp = fp[0];
 570                 pc = fp[1];
 571
 572                 if (dtrace_invop_callsite_pre != NULL
 573                     && pc > (uintptr_t)dtrace_invop_callsite_pre
 574                     && pc <= (uintptr_t)dtrace_invop_callsite_post) {
 575                         /*
 576                          * If we pass through the invalid op handler, we will
 577                          * use the pointer that it passed to the stack as the
 578                          * second argument to dtrace_invop() as the pointer to
 579                          * the frame we're hunting for.
 580                          */
 581
 582                         stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */
 583                         fp = (struct frame *)stack[1]; /* Grab *second* argument */
 584                         stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */
 585                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
 586                         val = (uint64_t)(stack[arg]);
 587                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
 588                         return val;
 589                 }
 590         }
 591
 592         /*
 593          * Arrive here when provider has called dtrace_probe directly.
 594          */
 595         stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */
 596         stack++; /* Advance past probeID */
 597
 598         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
 599         val = *(((uint64_t *)stack) + arg); /* dtrace_probe arguments arg0 .. arg4 are 64bits wide */
 600         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
 601         return val;
 602 #endif
 603         return 0xfeedfacedeafbeadLL;
 604 }
 605
 606 void
 607 dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which,
 608     int fltoffs, int fault, uint64_t illval)
 609 {
 610         /* XXX ARMTODO */
 611         /*
 612          * For the case of the error probe firing lets
 613          * stash away "illval" here, and special-case retrieving it in DIF_VARIABLE_ARG.
 614          */
 615         state->dts_arg_error_illval = illval;
 616         dtrace_probe( dtrace_probeid_error, (uint64_t)(uintptr_t)state, epid, which, fltoffs, fault );
 617 }
 618
 619 void
 620 dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit))
 621 {
 622         /* XXX ARMTODO check copied from ppc/x86*/
 623         /*
 624          * "base" is the smallest toxic address in the range, "limit" is the first
 625          * VALID address greater than "base".
 626          */
 627         func(0x0, VM_MIN_KERNEL_ADDRESS);
 628         if (VM_MAX_KERNEL_ADDRESS < ~(uintptr_t)0) {
 629                 func(VM_MAX_KERNEL_ADDRESS + 1, ~(uintptr_t)0);
 630         }
 631 }
 632
 633 int
 634 dtrace_arm_condition_true(int cond, int cpsr)
 635 {
 636         int taken = 0;
 637         int zf = (cpsr & PSR_ZF) ? 1 : 0,
 638             nf = (cpsr & PSR_NF) ? 1 : 0,
 639             cf = (cpsr & PSR_CF) ? 1 : 0,
 640             vf = (cpsr & PSR_VF) ? 1 : 0;
 641
 642         switch (cond) {
 643         case 0: taken = zf; break;
 644         case 1: taken = !zf; break;
 645         case 2: taken = cf; break;
 646         case 3: taken = !cf; break;
 647         case 4: taken = nf; break;
 648         case 5: taken = !nf; break;
 649         case 6: taken = vf; break;
 650         case 7: taken = !vf; break;
 651         case 8: taken = (cf && !zf); break;
 652         case 9: taken = (!cf || zf); break;
 653         case 10: taken = (nf == vf); break;
 654         case 11: taken = (nf != vf); break;
 655         case 12: taken = (!zf && (nf == vf)); break;
 656         case 13: taken = (zf || (nf != vf)); break;
 657         case 14: taken = 1; break;
 658         case 15: taken = 1; break;         /* always "true" for ARM, unpredictable for THUMB. */
 659         }
 660
 661         return taken;
 662 }
 663
 664 void
 665 dtrace_flush_caches(void)
 666 {
 667         /* TODO There were some problems with flushing just the cache line that had been modified.
 668          * For now, we'll flush the entire cache, until we figure out how to flush just the patched block.
 669          */
 670         FlushPoU_Dcache();
 671         InvalidatePoU_Icache();
 672 }