bsd/dev/arm/dtrace_isa.c

   1 /*
   2  * Copyright (c) 2005-2018 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <arm/caches_internal.h>
  30 #include <arm/proc_reg.h>
  31
  32 #include <kern/thread.h>
  33 #include <mach/thread_status.h>
  34
  35 #include <stdarg.h>
  36 #include <string.h>
  37 #include <sys/malloc.h>
  38 #include <sys/time.h>
  39 #include <sys/systm.h>
  40 #include <sys/proc.h>
  41 #include <sys/proc_internal.h>
  42 #include <sys/kauth.h>
  43 #include <sys/dtrace.h>
  44 #include <sys/dtrace_impl.h>
  45 #include <machine/atomic.h>
  46 #include <kern/simple_lock.h>
  47 #include <kern/sched_prim.h>            /* for thread_wakeup() */
  48 #include <kern/thread_call.h>
  49 #include <kern/task.h>
  50 #include <miscfs/devfs/devfs.h>
  51 #include <mach/vm_param.h>
  52
  53 extern struct arm_saved_state *find_kern_regs(thread_t);
  54
  55 extern dtrace_id_t      dtrace_probeid_error;   /* special ERROR probe */
  56 typedef arm_saved_state_t savearea_t;
  57
  58 int dtrace_arm_condition_true(int condition, int cpsr);
  59
  60 /*
  61  * Atomicity and synchronization
  62  */
  63 inline void
  64 dtrace_membar_producer(void)
  65 {
  66         __asm__ volatile ("dmb ish" : : : "memory");
  67 }
  68
  69 inline void
  70 dtrace_membar_consumer(void)
  71 {
  72         __asm__ volatile ("dmb ish" : : : "memory");
  73 }
  74
  75 /*
  76  * Interrupt manipulation
  77  * XXX dtrace_getipl() can be called from probe context.
  78  */
  79 int
  80 dtrace_getipl(void)
  81 {
  82         /*
  83          * XXX Drat, get_interrupt_level is MACH_KERNEL_PRIVATE
  84          * in osfmk/kern/cpu_data.h
  85          */
  86         /* return get_interrupt_level(); */
  87         return ml_at_interrupt_context() ? 1 : 0;
  88 }
  89
  90 /*
  91  * MP coordination
  92  */
  93
  94 static LCK_MTX_DECLARE_ATTR(dt_xc_lock, &dtrace_lck_grp, &dtrace_lck_attr);
  95 static uint32_t dt_xc_sync;
  96
  97 typedef struct xcArg {
  98         processorid_t   cpu;
  99         dtrace_xcall_t  f;
 100         void           *arg;
 101 } xcArg_t;
 102
 103 static void
 104 xcRemote(void *foo)
 105 {
 106         xcArg_t *pArg = (xcArg_t *) foo;
 107
 108         if (pArg->cpu == CPU->cpu_id || pArg->cpu == DTRACE_CPUALL) {
 109                 (pArg->f)(pArg->arg);
 110         }
 111
 112         if (os_atomic_dec(&dt_xc_sync, relaxed) == 0) {
 113                 thread_wakeup((event_t) &dt_xc_sync);
 114         }
 115 }
 116
 117 /*
 118  * dtrace_xcall() is not called from probe context.
 119  */
 120 void
 121 dtrace_xcall(processorid_t cpu, dtrace_xcall_t f, void *arg)
 122 {
 123         /* Only one dtrace_xcall in flight allowed */
 124         lck_mtx_lock(&dt_xc_lock);
 125
 126         xcArg_t xcArg;
 127
 128         xcArg.cpu = cpu;
 129         xcArg.f = f;
 130         xcArg.arg = arg;
 131
 132         cpu_broadcast_xcall(&dt_xc_sync, TRUE, xcRemote, (void*) &xcArg);
 133
 134         lck_mtx_unlock(&dt_xc_lock);
 135         return;
 136 }
 137
 138 /*
 139  * Runtime and ABI
 140  */
 141 uint64_t
 142 dtrace_getreg(struct regs * savearea, uint_t reg)
 143 {
 144         struct arm_saved_state *regs = (struct arm_saved_state *) savearea;
 145         if (regs == NULL) {
 146                 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
 147                 return 0;
 148         }
 149         /* beyond register limit? */
 150         if (reg > ARM_SAVED_STATE32_COUNT - 1) {
 151                 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
 152                 return 0;
 153         }
 154
 155         return (uint64_t) ((unsigned int *) (&(regs->r)))[reg];
 156 }
 157
 158 uint64_t
 159 dtrace_getvmreg(uint_t ndx)
 160 {
 161 #pragma unused(ndx)
 162         DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
 163         return 0;
 164 }
 165
 166 #define RETURN_OFFSET 4
 167
 168 static int
 169 dtrace_getustack_common(uint64_t * pcstack, int pcstack_limit, user_addr_t pc,
 170     user_addr_t sp)
 171 {
 172         volatile uint16_t *flags = (volatile uint16_t *) &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
 173         int ret = 0;
 174
 175         ASSERT(pcstack == NULL || pcstack_limit > 0);
 176
 177         while (pc != 0) {
 178                 ret++;
 179                 if (pcstack != NULL) {
 180                         *pcstack++ = (uint64_t) pc;
 181                         pcstack_limit--;
 182                         if (pcstack_limit <= 0) {
 183                                 break;
 184                         }
 185                 }
 186
 187                 if (sp == 0) {
 188                         break;
 189                 }
 190
 191                 pc = dtrace_fuword32((sp + RETURN_OFFSET));
 192                 sp = dtrace_fuword32(sp);
 193
 194                 /* Truncate ustack if the iterator causes fault. */
 195                 if (*flags & CPU_DTRACE_FAULT) {
 196                         *flags &= ~CPU_DTRACE_FAULT;
 197                         break;
 198                 }
 199         }
 200
 201         return ret;
 202 }
 203
 204 void
 205 dtrace_getupcstack(uint64_t * pcstack, int pcstack_limit)
 206 {
 207         thread_t        thread = current_thread();
 208         savearea_t     *regs;
 209         user_addr_t     pc, sp;
 210         volatile uint16_t *flags = (volatile uint16_t *) &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
 211         int n;
 212
 213         if (*flags & CPU_DTRACE_FAULT) {
 214                 return;
 215         }
 216
 217         if (pcstack_limit <= 0) {
 218                 return;
 219         }
 220
 221         /*
 222          * If there's no user context we still need to zero the stack.
 223          */
 224         if (thread == NULL) {
 225                 goto zero;
 226         }
 227
 228         regs = (savearea_t *) find_user_regs(thread);
 229         if (regs == NULL) {
 230                 goto zero;
 231         }
 232
 233         *pcstack++ = (uint64_t)dtrace_proc_selfpid();
 234         pcstack_limit--;
 235
 236         if (pcstack_limit <= 0) {
 237                 return;
 238         }
 239
 240         pc = regs->pc;
 241         sp = regs->sp;
 242
 243         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
 244                 *pcstack++ = (uint64_t) pc;
 245                 pcstack_limit--;
 246                 if (pcstack_limit <= 0) {
 247                         return;
 248                 }
 249
 250                 pc = regs->lr;
 251         }
 252
 253         n = dtrace_getustack_common(pcstack, pcstack_limit, pc, regs->r[7]);
 254
 255         ASSERT(n >= 0);
 256         ASSERT(n <= pcstack_limit);
 257
 258         pcstack += n;
 259         pcstack_limit -= n;
 260
 261 zero:
 262         while (pcstack_limit-- > 0) {
 263                 *pcstack++ = 0ULL;
 264         }
 265 }
 266
 267 int
 268 dtrace_getustackdepth(void)
 269 {
 270         thread_t        thread = current_thread();
 271         savearea_t     *regs;
 272         user_addr_t     pc, sp;
 273         int             n = 0;
 274
 275         if (thread == NULL) {
 276                 return 0;
 277         }
 278
 279         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT)) {
 280                 return -1;
 281         }
 282
 283         regs = (savearea_t *) find_user_regs(thread);
 284         if (regs == NULL) {
 285                 return 0;
 286         }
 287
 288         pc = regs->pc;
 289         sp = regs->sp;
 290
 291         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
 292                 n++;
 293                 pc = regs->lr;
 294         }
 295
 296         /*
 297          * Note that unlike ppc, the arm code does not use
 298          * CPU_DTRACE_USTACK_FP. This is because arm always
 299          * traces from the sp, even in syscall/profile/fbt
 300          * providers.
 301          */
 302
 303         n += dtrace_getustack_common(NULL, 0, pc, regs->r[7]);
 304
 305         return n;
 306 }
 307
 308 void
 309 dtrace_getufpstack(uint64_t * pcstack, uint64_t * fpstack, int pcstack_limit)
 310 {
 311         /* XXX ARMTODO 64vs32 */
 312         thread_t        thread = current_thread();
 313         savearea_t      *regs;
 314         user_addr_t     pc, sp;
 315
 316         volatile        uint16_t  *flags = (volatile uint16_t *) &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
 317
 318 #if 0
 319         uintptr_t oldcontext;
 320         size_t          s1, s2;
 321 #endif
 322
 323         if (*flags & CPU_DTRACE_FAULT) {
 324                 return;
 325         }
 326
 327         if (pcstack_limit <= 0) {
 328                 return;
 329         }
 330
 331         /*
 332          * If there's no user context we still need to zero the stack.
 333          */
 334         if (thread == NULL) {
 335                 goto zero;
 336         }
 337
 338         regs = (savearea_t *) find_user_regs(thread);
 339         if (regs == NULL) {
 340                 goto zero;
 341         }
 342
 343         *pcstack++ = (uint64_t)dtrace_proc_selfpid();
 344         pcstack_limit--;
 345
 346         if (pcstack_limit <= 0) {
 347                 return;
 348         }
 349
 350         pc = regs->pc;
 351         sp = regs->sp;
 352
 353 #if 0                           /* XXX signal stack crawl */
 354         oldcontext = lwp->lwp_oldcontext;
 355
 356         if (p->p_model == DATAMODEL_NATIVE) {
 357                 s1 = sizeof(struct frame) + 2 * sizeof(long);
 358                 s2 = s1 + sizeof(siginfo_t);
 359         } else {
 360                 s1 = sizeof(struct frame32) + 3 * sizeof(int);
 361                 s2 = s1 + sizeof(siginfo32_t);
 362         }
 363 #endif
 364
 365         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
 366                 *pcstack++ = (uint64_t) pc;
 367                 *fpstack++ = 0;
 368                 pcstack_limit--;
 369                 if (pcstack_limit <= 0) {
 370                         return;
 371                 }
 372
 373                 pc = dtrace_fuword32(sp);
 374         }
 375         while (pc != 0 && sp != 0) {
 376                 *pcstack++ = (uint64_t) pc;
 377                 *fpstack++ = sp;
 378                 pcstack_limit--;
 379                 if (pcstack_limit <= 0) {
 380                         break;
 381                 }
 382
 383 #if 0                           /* XXX signal stack crawl */
 384                 if (oldcontext == sp + s1 || oldcontext == sp + s2) {
 385                         if (p->p_model == DATAMODEL_NATIVE) {
 386                                 ucontext_t     *ucp = (ucontext_t *) oldcontext;
 387                                 greg_t         *gregs = ucp->uc_mcontext.gregs;
 388
 389                                 sp = dtrace_fulword(&gregs[REG_FP]);
 390                                 pc = dtrace_fulword(&gregs[REG_PC]);
 391
 392                                 oldcontext = dtrace_fulword(&ucp->uc_link);
 393                         } else {
 394                                 ucontext_t     *ucp = (ucontext_t *) oldcontext;
 395                                 greg_t         *gregs = ucp->uc_mcontext.gregs;
 396
 397                                 sp = dtrace_fuword32(&gregs[EBP]);
 398                                 pc = dtrace_fuword32(&gregs[EIP]);
 399
 400                                 oldcontext = dtrace_fuword32(&ucp->uc_link);
 401                         }
 402                 } else
 403 #endif
 404                 {
 405                         pc = dtrace_fuword32((sp + RETURN_OFFSET));
 406                         sp = dtrace_fuword32(sp);
 407                 }
 408
 409                 /* Truncate ustack if the iterator causes fault. */
 410                 if (*flags & CPU_DTRACE_FAULT) {
 411                         *flags &= ~CPU_DTRACE_FAULT;
 412                         break;
 413                 }
 414         }
 415
 416 zero:
 417         while (pcstack_limit-- > 0) {
 418                 *pcstack++ = 0ULL;
 419         }
 420 }
 421
 422 void
 423 dtrace_getpcstack(pc_t * pcstack, int pcstack_limit, int aframes,
 424     uint32_t * intrpc)
 425 {
 426         struct frame   *fp = (struct frame *) __builtin_frame_address(0);
 427         struct frame   *nextfp, *minfp, *stacktop;
 428         int             depth = 0;
 429         int             on_intr;
 430         int             last = 0;
 431         uintptr_t       pc;
 432         uintptr_t       caller = CPU->cpu_dtrace_caller;
 433
 434         if ((on_intr = CPU_ON_INTR(CPU)) != 0) {
 435                 stacktop = (struct frame *) dtrace_get_cpu_int_stack_top();
 436         } else {
 437                 stacktop = (struct frame *) (dtrace_get_kernel_stack(current_thread()) + kernel_stack_size);
 438         }
 439
 440         minfp = fp;
 441
 442         aframes++;
 443
 444         if (intrpc != NULL && depth < pcstack_limit) {
 445                 pcstack[depth++] = (pc_t) intrpc;
 446         }
 447
 448         while (depth < pcstack_limit) {
 449                 nextfp = *(struct frame **) fp;
 450                 pc = *(uintptr_t *) (((uint32_t) fp) + RETURN_OFFSET);
 451
 452                 if (nextfp <= minfp || nextfp >= stacktop) {
 453                         if (on_intr) {
 454                                 /*
 455                                  * Hop from interrupt stack to thread stack.
 456                                  */
 457                                 arm_saved_state_t *arm_kern_regs = (arm_saved_state_t *) find_kern_regs(current_thread());
 458                                 if (arm_kern_regs) {
 459                                         nextfp = (struct frame *)arm_kern_regs->r[7];
 460
 461                                         vm_offset_t kstack_base = dtrace_get_kernel_stack(current_thread());
 462
 463                                         minfp = (struct frame *)kstack_base;
 464                                         stacktop = (struct frame *)(kstack_base + kernel_stack_size);
 465
 466                                         on_intr = 0;
 467
 468                                         if (nextfp <= minfp || nextfp >= stacktop) {
 469                                                 last = 1;
 470                                         }
 471                                 } else {
 472                                         /*
 473                                          * If this thread was on the interrupt stack, but did not
 474                                          * take an interrupt (i.e, the idle thread), there is no
 475                                          * explicit saved state for us to use.
 476                                          */
 477                                         last = 1;
 478                                 }
 479                         } else {
 480                                 /*
 481                                  * This is the last frame we can process; indicate
 482                                  * that we should return after processing this frame.
 483                                  */
 484                                 last = 1;
 485                         }
 486                 }
 487                 if (aframes > 0) {
 488                         if (--aframes == 0 && caller != (uintptr_t)NULL) {
 489                                 /*
 490                                  * We've just run out of artificial frames,
 491                                  * and we have a valid caller -- fill it in
 492                                  * now.
 493                                  */
 494                                 ASSERT(depth < pcstack_limit);
 495                                 pcstack[depth++] = (pc_t) caller;
 496                                 caller = (uintptr_t)NULL;
 497                         }
 498                 } else {
 499                         if (depth < pcstack_limit) {
 500                                 pcstack[depth++] = (pc_t) pc;
 501                         }
 502                 }
 503
 504                 if (last) {
 505                         while (depth < pcstack_limit) {
 506                                 pcstack[depth++] = (pc_t) NULL;
 507                         }
 508                         return;
 509                 }
 510                 fp = nextfp;
 511                 minfp = fp;
 512         }
 513 }
 514
 515 int
 516 dtrace_instr_size(uint32_t instr, int thumb_mode)
 517 {
 518         if (thumb_mode) {
 519                 uint16_t instr16 = *(uint16_t*) &instr;
 520                 if (((instr16 >> 11) & 0x1F) > 0x1C) {
 521                         return 4;
 522                 } else {
 523                         return 2;
 524                 }
 525         } else {
 526                 return 4;
 527         }
 528 }
 529
 530 uint64_t
 531 dtrace_getarg(int arg, int aframes, dtrace_mstate_t *mstate, dtrace_vstate_t *vstate)
 532 {
 533 #pragma unused(arg, aframes, mstate, vstate)
 534 #if 0
 535         /* XXX ARMTODO */
 536         uint64_t val;
 537         uintptr_t *fp = (uintptr_t *)__builtin_frame_address(0);
 538         uintptr_t *stack;
 539         uintptr_t pc;
 540         int i;
 541
 542         for (i = 1; i <= aframes; i++) {
 543                 fp = fp[0];
 544                 pc = fp[1];
 545
 546                 if (dtrace_invop_callsite_pre != NULL
 547                     && pc > (uintptr_t)dtrace_invop_callsite_pre
 548                     && pc <= (uintptr_t)dtrace_invop_callsite_post) {
 549                         /*
 550                          * If we pass through the invalid op handler, we will
 551                          * use the pointer that it passed to the stack as the
 552                          * second argument to dtrace_invop() as the pointer to
 553                          * the frame we're hunting for.
 554                          */
 555
 556                         stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */
 557                         fp = (struct frame *)stack[1]; /* Grab *second* argument */
 558                         stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */
 559                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
 560                         val = (uint64_t)(stack[arg]);
 561                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
 562                         return val;
 563                 }
 564         }
 565
 566         /*
 567          * Arrive here when provider has called dtrace_probe directly.
 568          */
 569         stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */
 570         stack++; /* Advance past probeID */
 571
 572         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
 573         val = *(((uint64_t *)stack) + arg); /* dtrace_probe arguments arg0 .. arg4 are 64bits wide */
 574         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
 575         return val;
 576 #endif
 577         return 0xfeedfacedeafbeadLL;
 578 }
 579
 580 void
 581 dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which,
 582     int fltoffs, int fault, uint64_t illval)
 583 {
 584         /* XXX ARMTODO */
 585         /*
 586          * For the case of the error probe firing lets
 587          * stash away "illval" here, and special-case retrieving it in DIF_VARIABLE_ARG.
 588          */
 589         state->dts_arg_error_illval = illval;
 590         dtrace_probe( dtrace_probeid_error, (uint64_t)(uintptr_t)state, epid, which, fltoffs, fault );
 591 }
 592
 593 void
 594 dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit))
 595 {
 596         /* XXX ARMTODO check copied from ppc/x86*/
 597         /*
 598          * "base" is the smallest toxic address in the range, "limit" is the first
 599          * VALID address greater than "base".
 600          */
 601         func(0x0, VM_MIN_KERNEL_ADDRESS);
 602         if (VM_MAX_KERNEL_ADDRESS < ~(uintptr_t)0) {
 603                 func(VM_MAX_KERNEL_ADDRESS + 1, ~(uintptr_t)0);
 604         }
 605 }
 606
 607 int
 608 dtrace_arm_condition_true(int cond, int cpsr)
 609 {
 610         int taken = 0;
 611         int zf = (cpsr & PSR_ZF) ? 1 : 0,
 612             nf = (cpsr & PSR_NF) ? 1 : 0,
 613             cf = (cpsr & PSR_CF) ? 1 : 0,
 614             vf = (cpsr & PSR_VF) ? 1 : 0;
 615
 616         switch (cond) {
 617         case 0: taken = zf; break;
 618         case 1: taken = !zf; break;
 619         case 2: taken = cf; break;
 620         case 3: taken = !cf; break;
 621         case 4: taken = nf; break;
 622         case 5: taken = !nf; break;
 623         case 6: taken = vf; break;
 624         case 7: taken = !vf; break;
 625         case 8: taken = (cf && !zf); break;
 626         case 9: taken = (!cf || zf); break;
 627         case 10: taken = (nf == vf); break;
 628         case 11: taken = (nf != vf); break;
 629         case 12: taken = (!zf && (nf == vf)); break;
 630         case 13: taken = (zf || (nf != vf)); break;
 631         case 14: taken = 1; break;
 632         case 15: taken = 1; break;         /* always "true" for ARM, unpredictable for THUMB. */
 633         }
 634
 635         return taken;
 636 }
 637
 638 void
 639 dtrace_flush_caches(void)
 640 {
 641         /* TODO There were some problems with flushing just the cache line that had been modified.
 642          * For now, we'll flush the entire cache, until we figure out how to flush just the patched block.
 643          */
 644         FlushPoU_Dcache();
 645         InvalidatePoU_Icache();
 646 }