bsd/dev/arm/dtrace_isa.c

   1 /*
   2  * Copyright (c) 2005-2008 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #define MACH__POSIX_C_SOURCE_PRIVATE 1  /* pulls in suitable savearea from
  30                                          * mach/ppc/thread_status.h */
  31 #include <arm/caches_internal.h>
  32 #include <arm/proc_reg.h>
  33
  34 #include <kern/thread.h>
  35 #include <mach/thread_status.h>
  36
  37 #include <stdarg.h>
  38 #include <string.h>
  39 #include <sys/malloc.h>
  40 #include <sys/time.h>
  41 #include <sys/systm.h>
  42 #include <sys/proc.h>
  43 #include <sys/proc_internal.h>
  44 #include <sys/kauth.h>
  45 #include <sys/dtrace.h>
  46 #include <sys/dtrace_impl.h>
  47 #include <libkern/OSAtomic.h>
  48 #include <kern/simple_lock.h>
  49 #include <kern/sched_prim.h>            /* for thread_wakeup() */
  50 #include <kern/thread_call.h>
  51 #include <kern/task.h>
  52 #include <miscfs/devfs/devfs.h>
  53 #include <mach/vm_param.h>
  54
  55 extern struct arm_saved_state *find_kern_regs(thread_t);
  56
  57 extern dtrace_id_t      dtrace_probeid_error;   /* special ERROR probe */
  58 typedef arm_saved_state_t savearea_t;
  59
  60 extern lck_attr_t       *dtrace_lck_attr;
  61 extern lck_grp_t        *dtrace_lck_grp;
  62
  63 int dtrace_arm_condition_true(int condition, int cpsr);
  64
  65 /*
  66  * Atomicity and synchronization
  67  */
  68 inline void
  69 dtrace_membar_producer(void)
  70 {
  71 #if __ARM_SMP__
  72         __asm__ volatile ("dmb ish" : : : "memory");
  73 #else
  74         __asm__ volatile ("nop" : : : "memory");
  75 #endif
  76 }
  77
  78 inline void
  79 dtrace_membar_consumer(void)
  80 {
  81 #if __ARM_SMP__
  82         __asm__ volatile ("dmb ish" : : : "memory");
  83 #else
  84         __asm__ volatile ("nop" : : : "memory");
  85 #endif
  86 }
  87
  88 /*
  89  * Interrupt manipulation
  90  * XXX dtrace_getipl() can be called from probe context.
  91  */
  92 int
  93 dtrace_getipl(void)
  94 {
  95         /*
  96          * XXX Drat, get_interrupt_level is MACH_KERNEL_PRIVATE
  97          * in osfmk/kern/cpu_data.h
  98          */
  99         /* return get_interrupt_level(); */
 100         return ml_at_interrupt_context() ? 1 : 0;
 101 }
 102
 103 #if __ARM_SMP__
 104 /*
 105  * MP coordination
 106  */
 107
 108 decl_lck_mtx_data(static, dt_xc_lock);
 109 static uint32_t dt_xc_sync;
 110
 111 typedef struct xcArg {
 112         processorid_t   cpu;
 113         dtrace_xcall_t  f;
 114         void           *arg;
 115 } xcArg_t;
 116
 117 static void
 118 xcRemote(void *foo)
 119 {
 120         xcArg_t *pArg = (xcArg_t *) foo;
 121
 122         if (pArg->cpu == CPU->cpu_id || pArg->cpu == DTRACE_CPUALL) {
 123                 (pArg->f)(pArg->arg);
 124         }
 125
 126         if (hw_atomic_sub(&dt_xc_sync, 1) == 0) {
 127                 thread_wakeup((event_t) &dt_xc_sync);
 128         }
 129 }
 130 #endif
 131
 132 /*
 133  * dtrace_xcall() is not called from probe context.
 134  */
 135 void
 136 dtrace_xcall(processorid_t cpu, dtrace_xcall_t f, void *arg)
 137 {
 138 #if __ARM_SMP__
 139         /* Only one dtrace_xcall in flight allowed */
 140         lck_mtx_lock(&dt_xc_lock);
 141
 142         xcArg_t xcArg;
 143
 144         xcArg.cpu = cpu;
 145         xcArg.f = f;
 146         xcArg.arg = arg;
 147
 148         cpu_broadcast_xcall(&dt_xc_sync, TRUE, xcRemote, (void*) &xcArg);
 149
 150         lck_mtx_unlock(&dt_xc_lock);
 151         return;
 152 #else
 153 #pragma unused(cpu)
 154         /* On uniprocessor systems, the cpu should always be either ourselves or all */
 155         ASSERT(cpu == CPU->cpu_id || cpu == DTRACE_CPUALL);
 156
 157         (*f)(arg);
 158         return;
 159 #endif
 160 }
 161
 162 /*
 163  * Initialization
 164  */
 165 void
 166 dtrace_isa_init(void)
 167 {
 168 #if __ARM_SMP__
 169         lck_mtx_init(&dt_xc_lock, dtrace_lck_grp, dtrace_lck_attr);
 170 #endif
 171         return;
 172 }
 173
 174 /*
 175  * Runtime and ABI
 176  */
 177 uint64_t
 178 dtrace_getreg(struct regs * savearea, uint_t reg)
 179 {
 180         struct arm_saved_state *regs = (struct arm_saved_state *) savearea;
 181         if (regs == NULL) {
 182                 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
 183                 return 0;
 184         }
 185         /* beyond register limit? */
 186         if (reg > ARM_SAVED_STATE32_COUNT - 1) {
 187                 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
 188                 return 0;
 189         }
 190
 191         return (uint64_t) ((unsigned int *) (&(regs->r)))[reg];
 192 }
 193
 194 #define RETURN_OFFSET 4
 195
 196 static int
 197 dtrace_getustack_common(uint64_t * pcstack, int pcstack_limit, user_addr_t pc,
 198     user_addr_t sp)
 199 {
 200         int ret = 0;
 201
 202         ASSERT(pcstack == NULL || pcstack_limit > 0);
 203
 204         while (pc != 0) {
 205                 ret++;
 206                 if (pcstack != NULL) {
 207                         *pcstack++ = (uint64_t) pc;
 208                         pcstack_limit--;
 209                         if (pcstack_limit <= 0) {
 210                                 break;
 211                         }
 212                 }
 213
 214                 if (sp == 0) {
 215                         break;
 216                 }
 217
 218                 pc = dtrace_fuword32((sp + RETURN_OFFSET));
 219                 sp = dtrace_fuword32(sp);
 220         }
 221
 222         return ret;
 223 }
 224
 225 void
 226 dtrace_getupcstack(uint64_t * pcstack, int pcstack_limit)
 227 {
 228         thread_t        thread = current_thread();
 229         savearea_t     *regs;
 230         user_addr_t     pc, sp;
 231         volatile uint16_t *flags = (volatile uint16_t *) &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
 232         int n;
 233
 234         if (*flags & CPU_DTRACE_FAULT) {
 235                 return;
 236         }
 237
 238         if (pcstack_limit <= 0) {
 239                 return;
 240         }
 241
 242         /*
 243          * If there's no user context we still need to zero the stack.
 244          */
 245         if (thread == NULL) {
 246                 goto zero;
 247         }
 248
 249         regs = (savearea_t *) find_user_regs(thread);
 250         if (regs == NULL) {
 251                 goto zero;
 252         }
 253
 254         *pcstack++ = (uint64_t)dtrace_proc_selfpid();
 255         pcstack_limit--;
 256
 257         if (pcstack_limit <= 0) {
 258                 return;
 259         }
 260
 261         pc = regs->pc;
 262         sp = regs->sp;
 263
 264         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
 265                 *pcstack++ = (uint64_t) pc;
 266                 pcstack_limit--;
 267                 if (pcstack_limit <= 0) {
 268                         return;
 269                 }
 270
 271                 pc = regs->lr;
 272         }
 273
 274         n = dtrace_getustack_common(pcstack, pcstack_limit, pc, regs->r[7]);
 275
 276         ASSERT(n >= 0);
 277         ASSERT(n <= pcstack_limit);
 278
 279         pcstack += n;
 280         pcstack_limit -= n;
 281
 282 zero:
 283         while (pcstack_limit-- > 0) {
 284                 *pcstack++ = 0ULL;
 285         }
 286 }
 287
 288 int
 289 dtrace_getustackdepth(void)
 290 {
 291         thread_t        thread = current_thread();
 292         savearea_t     *regs;
 293         user_addr_t     pc, sp;
 294         int             n = 0;
 295
 296         if (thread == NULL) {
 297                 return 0;
 298         }
 299
 300         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT)) {
 301                 return -1;
 302         }
 303
 304         regs = (savearea_t *) find_user_regs(thread);
 305         if (regs == NULL) {
 306                 return 0;
 307         }
 308
 309         pc = regs->pc;
 310         sp = regs->sp;
 311
 312         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
 313                 n++;
 314                 pc = regs->lr;
 315         }
 316
 317         /*
 318          * Note that unlike ppc, the arm code does not use
 319          * CPU_DTRACE_USTACK_FP. This is because arm always
 320          * traces from the sp, even in syscall/profile/fbt
 321          * providers.
 322          */
 323
 324         n += dtrace_getustack_common(NULL, 0, pc, regs->r[7]);
 325
 326         return n;
 327 }
 328
 329 void
 330 dtrace_getufpstack(uint64_t * pcstack, uint64_t * fpstack, int pcstack_limit)
 331 {
 332         /* XXX ARMTODO 64vs32 */
 333         thread_t        thread = current_thread();
 334         savearea_t      *regs;
 335         user_addr_t     pc, sp;
 336
 337         volatile        uint16_t  *flags = (volatile uint16_t *) &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
 338
 339 #if 0
 340         uintptr_t oldcontext;
 341         size_t          s1, s2;
 342 #endif
 343
 344         if (*flags & CPU_DTRACE_FAULT) {
 345                 return;
 346         }
 347
 348         if (pcstack_limit <= 0) {
 349                 return;
 350         }
 351
 352         /*
 353          * If there's no user context we still need to zero the stack.
 354          */
 355         if (thread == NULL) {
 356                 goto zero;
 357         }
 358
 359         regs = (savearea_t *) find_user_regs(thread);
 360         if (regs == NULL) {
 361                 goto zero;
 362         }
 363
 364         *pcstack++ = (uint64_t)dtrace_proc_selfpid();
 365         pcstack_limit--;
 366
 367         if (pcstack_limit <= 0) {
 368                 return;
 369         }
 370
 371         pc = regs->pc;
 372         sp = regs->sp;
 373
 374 #if 0                           /* XXX signal stack crawl */
 375         oldcontext = lwp->lwp_oldcontext;
 376
 377         if (p->p_model == DATAMODEL_NATIVE) {
 378                 s1 = sizeof(struct frame) + 2 * sizeof(long);
 379                 s2 = s1 + sizeof(siginfo_t);
 380         } else {
 381                 s1 = sizeof(struct frame32) + 3 * sizeof(int);
 382                 s2 = s1 + sizeof(siginfo32_t);
 383         }
 384 #endif
 385
 386         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
 387                 *pcstack++ = (uint64_t) pc;
 388                 *fpstack++ = 0;
 389                 pcstack_limit--;
 390                 if (pcstack_limit <= 0) {
 391                         return;
 392                 }
 393
 394                 pc = dtrace_fuword32(sp);
 395         }
 396         while (pc != 0 && sp != 0) {
 397                 *pcstack++ = (uint64_t) pc;
 398                 *fpstack++ = sp;
 399                 pcstack_limit--;
 400                 if (pcstack_limit <= 0) {
 401                         break;
 402                 }
 403
 404 #if 0                           /* XXX signal stack crawl */
 405                 if (oldcontext == sp + s1 || oldcontext == sp + s2) {
 406                         if (p->p_model == DATAMODEL_NATIVE) {
 407                                 ucontext_t     *ucp = (ucontext_t *) oldcontext;
 408                                 greg_t         *gregs = ucp->uc_mcontext.gregs;
 409
 410                                 sp = dtrace_fulword(&gregs[REG_FP]);
 411                                 pc = dtrace_fulword(&gregs[REG_PC]);
 412
 413                                 oldcontext = dtrace_fulword(&ucp->uc_link);
 414                         } else {
 415                                 ucontext_t     *ucp = (ucontext_t *) oldcontext;
 416                                 greg_t         *gregs = ucp->uc_mcontext.gregs;
 417
 418                                 sp = dtrace_fuword32(&gregs[EBP]);
 419                                 pc = dtrace_fuword32(&gregs[EIP]);
 420
 421                                 oldcontext = dtrace_fuword32(&ucp->uc_link);
 422                         }
 423                 } else
 424 #endif
 425                 {
 426                         pc = dtrace_fuword32((sp + RETURN_OFFSET));
 427                         sp = dtrace_fuword32(sp);
 428                 }
 429
 430 #if 0
 431                 /* XXX ARMTODO*/
 432                 /*
 433                  * This is totally bogus:  if we faulted, we're going to clear
 434                  * the fault and break.  This is to deal with the apparently
 435                  * broken Java stacks on x86.
 436                  */
 437                 if (*flags & CPU_DTRACE_FAULT) {
 438                         *flags &= ~CPU_DTRACE_FAULT;
 439                         break;
 440                 }
 441 #endif
 442         }
 443
 444 zero:
 445         while (pcstack_limit-- > 0) {
 446                 *pcstack++ = 0ULL;
 447         }
 448 }
 449
 450 void
 451 dtrace_getpcstack(pc_t * pcstack, int pcstack_limit, int aframes,
 452     uint32_t * intrpc)
 453 {
 454         struct frame   *fp = (struct frame *) __builtin_frame_address(0);
 455         struct frame   *nextfp, *minfp, *stacktop;
 456         int             depth = 0;
 457         int             on_intr;
 458         int             last = 0;
 459         uintptr_t       pc;
 460         uintptr_t       caller = CPU->cpu_dtrace_caller;
 461
 462         if ((on_intr = CPU_ON_INTR(CPU)) != 0) {
 463                 stacktop = (struct frame *) dtrace_get_cpu_int_stack_top();
 464         } else {
 465                 stacktop = (struct frame *) (dtrace_get_kernel_stack(current_thread()) + kernel_stack_size);
 466         }
 467
 468         minfp = fp;
 469
 470         aframes++;
 471
 472         if (intrpc != NULL && depth < pcstack_limit) {
 473                 pcstack[depth++] = (pc_t) intrpc;
 474         }
 475
 476         while (depth < pcstack_limit) {
 477                 nextfp = *(struct frame **) fp;
 478                 pc = *(uintptr_t *) (((uint32_t) fp) + RETURN_OFFSET);
 479
 480                 if (nextfp <= minfp || nextfp >= stacktop) {
 481                         if (on_intr) {
 482                                 /*
 483                                  * Hop from interrupt stack to thread stack.
 484                                  */
 485                                 arm_saved_state_t *arm_kern_regs = (arm_saved_state_t *) find_kern_regs(current_thread());
 486                                 if (arm_kern_regs) {
 487                                         nextfp = (struct frame *)arm_kern_regs->r[7];
 488
 489                                         vm_offset_t kstack_base = dtrace_get_kernel_stack(current_thread());
 490
 491                                         minfp = (struct frame *)kstack_base;
 492                                         stacktop = (struct frame *)(kstack_base + kernel_stack_size);
 493
 494                                         on_intr = 0;
 495
 496                                         if (nextfp <= minfp || nextfp >= stacktop) {
 497                                                 last = 1;
 498                                         }
 499                                 } else {
 500                                         /*
 501                                          * If this thread was on the interrupt stack, but did not
 502                                          * take an interrupt (i.e, the idle thread), there is no
 503                                          * explicit saved state for us to use.
 504                                          */
 505                                         last = 1;
 506                                 }
 507                         } else {
 508                                 /*
 509                                  * This is the last frame we can process; indicate
 510                                  * that we should return after processing this frame.
 511                                  */
 512                                 last = 1;
 513                         }
 514                 }
 515                 if (aframes > 0) {
 516                         if (--aframes == 0 && caller != (uintptr_t)NULL) {
 517                                 /*
 518                                  * We've just run out of artificial frames,
 519                                  * and we have a valid caller -- fill it in
 520                                  * now.
 521                                  */
 522                                 ASSERT(depth < pcstack_limit);
 523                                 pcstack[depth++] = (pc_t) caller;
 524                                 caller = (uintptr_t)NULL;
 525                         }
 526                 } else {
 527                         if (depth < pcstack_limit) {
 528                                 pcstack[depth++] = (pc_t) pc;
 529                         }
 530                 }
 531
 532                 if (last) {
 533                         while (depth < pcstack_limit) {
 534                                 pcstack[depth++] = (pc_t) NULL;
 535                         }
 536                         return;
 537                 }
 538                 fp = nextfp;
 539                 minfp = fp;
 540         }
 541 }
 542
 543 int
 544 dtrace_instr_size(uint32_t instr, int thumb_mode)
 545 {
 546         if (thumb_mode) {
 547                 uint16_t instr16 = *(uint16_t*) &instr;
 548                 if (((instr16 >> 11) & 0x1F) > 0x1C) {
 549                         return 4;
 550                 } else {
 551                         return 2;
 552                 }
 553         } else {
 554                 return 4;
 555         }
 556 }
 557
 558 uint64_t
 559 dtrace_getarg(int arg, int aframes, dtrace_mstate_t *mstate, dtrace_vstate_t *vstate)
 560 {
 561 #pragma unused(arg, aframes, mstate, vstate)
 562 #if 0
 563         /* XXX ARMTODO */
 564         uint64_t val;
 565         uintptr_t *fp = (uintptr_t *)__builtin_frame_address(0);
 566         uintptr_t *stack;
 567         uintptr_t pc;
 568         int i;
 569
 570         for (i = 1; i <= aframes; i++) {
 571                 fp = fp[0];
 572                 pc = fp[1];
 573
 574                 if (dtrace_invop_callsite_pre != NULL
 575                     && pc > (uintptr_t)dtrace_invop_callsite_pre
 576                     && pc <= (uintptr_t)dtrace_invop_callsite_post) {
 577                         /*
 578                          * If we pass through the invalid op handler, we will
 579                          * use the pointer that it passed to the stack as the
 580                          * second argument to dtrace_invop() as the pointer to
 581                          * the frame we're hunting for.
 582                          */
 583
 584                         stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */
 585                         fp = (struct frame *)stack[1]; /* Grab *second* argument */
 586                         stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */
 587                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
 588                         val = (uint64_t)(stack[arg]);
 589                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
 590                         return val;
 591                 }
 592         }
 593
 594         /*
 595          * Arrive here when provider has called dtrace_probe directly.
 596          */
 597         stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */
 598         stack++; /* Advance past probeID */
 599
 600         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
 601         val = *(((uint64_t *)stack) + arg); /* dtrace_probe arguments arg0 .. arg4 are 64bits wide */
 602         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
 603         return val;
 604 #endif
 605         return 0xfeedfacedeafbeadLL;
 606 }
 607
 608 void
 609 dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which,
 610     int fltoffs, int fault, uint64_t illval)
 611 {
 612         /* XXX ARMTODO */
 613         /*
 614          * For the case of the error probe firing lets
 615          * stash away "illval" here, and special-case retrieving it in DIF_VARIABLE_ARG.
 616          */
 617         state->dts_arg_error_illval = illval;
 618         dtrace_probe( dtrace_probeid_error, (uint64_t)(uintptr_t)state, epid, which, fltoffs, fault );
 619 }
 620
 621 void
 622 dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit))
 623 {
 624         /* XXX ARMTODO check copied from ppc/x86*/
 625         /*
 626          * "base" is the smallest toxic address in the range, "limit" is the first
 627          * VALID address greater than "base".
 628          */
 629         func(0x0, VM_MIN_KERNEL_ADDRESS);
 630         if (VM_MAX_KERNEL_ADDRESS < ~(uintptr_t)0) {
 631                 func(VM_MAX_KERNEL_ADDRESS + 1, ~(uintptr_t)0);
 632         }
 633 }
 634
 635 int
 636 dtrace_arm_condition_true(int cond, int cpsr)
 637 {
 638         int taken = 0;
 639         int zf = (cpsr & PSR_ZF) ? 1 : 0,
 640             nf = (cpsr & PSR_NF) ? 1 : 0,
 641             cf = (cpsr & PSR_CF) ? 1 : 0,
 642             vf = (cpsr & PSR_VF) ? 1 : 0;
 643
 644         switch (cond) {
 645         case 0: taken = zf; break;
 646         case 1: taken = !zf; break;
 647         case 2: taken = cf; break;
 648         case 3: taken = !cf; break;
 649         case 4: taken = nf; break;
 650         case 5: taken = !nf; break;
 651         case 6: taken = vf; break;
 652         case 7: taken = !vf; break;
 653         case 8: taken = (cf && !zf); break;
 654         case 9: taken = (!cf || zf); break;
 655         case 10: taken = (nf == vf); break;
 656         case 11: taken = (nf != vf); break;
 657         case 12: taken = (!zf && (nf == vf)); break;
 658         case 13: taken = (zf || (nf != vf)); break;
 659         case 14: taken = 1; break;
 660         case 15: taken = 1; break;         /* always "true" for ARM, unpredictable for THUMB. */
 661         }
 662
 663         return taken;
 664 }
 665
 666 void
 667 dtrace_flush_caches(void)
 668 {
 669         /* TODO There were some problems with flushing just the cache line that had been modified.
 670          * For now, we'll flush the entire cache, until we figure out how to flush just the patched block.
 671          */
 672         FlushPoU_Dcache();
 673         InvalidatePoU_Icache();
 674 }