bsd/dev/arm/dtrace_isa.c

   1 /*
   2  * Copyright (c) 2005-2008 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #define MACH__POSIX_C_SOURCE_PRIVATE 1  /* pulls in suitable savearea from
  30                                          * mach/ppc/thread_status.h */
  31 #include <arm/proc_reg.h>
  32
  33 #include <kern/thread.h>
  34 #include <mach/thread_status.h>
  35
  36 #include <stdarg.h>
  37 #include <string.h>
  38 #include <sys/malloc.h>
  39 #include <sys/time.h>
  40 #include <sys/systm.h>
  41 #include <sys/proc.h>
  42 #include <sys/proc_internal.h>
  43 #include <sys/kauth.h>
  44 #include <sys/dtrace.h>
  45 #include <sys/dtrace_impl.h>
  46 #include <libkern/OSAtomic.h>
  47 #include <kern/simple_lock.h>
  48 #include <kern/sched_prim.h>            /* for thread_wakeup() */
  49 #include <kern/thread_call.h>
  50 #include <kern/task.h>
  51 #include <miscfs/devfs/devfs.h>
  52 #include <mach/vm_param.h>
  53
  54 extern struct arm_saved_state *find_kern_regs(thread_t);
  55
  56 extern dtrace_id_t      dtrace_probeid_error;   /* special ERROR probe */
  57 typedef arm_saved_state_t savearea_t;
  58
  59 extern lck_attr_t       *dtrace_lck_attr;
  60 extern lck_grp_t        *dtrace_lck_grp;
  61
  62 int dtrace_arm_condition_true(int condition, int cpsr);
  63
  64 /*
  65  * Atomicity and synchronization
  66  */
  67 inline void
  68 dtrace_membar_producer(void)
  69 {
  70 #if __ARM_SMP__
  71         __asm__ volatile("dmb ish" : : : "memory");
  72 #else
  73         __asm__ volatile("nop" : : : "memory");
  74 #endif
  75 }
  76
  77 inline void
  78 dtrace_membar_consumer(void)
  79 {
  80 #if __ARM_SMP__
  81         __asm__ volatile("dmb ish" : : : "memory");
  82 #else
  83         __asm__ volatile("nop" : : : "memory");
  84 #endif
  85 }
  86
  87 /*
  88  * Interrupt manipulation
  89  * XXX dtrace_getipl() can be called from probe context.
  90  */
  91 int
  92 dtrace_getipl(void)
  93 {
  94         /*
  95          * XXX Drat, get_interrupt_level is MACH_KERNEL_PRIVATE
  96          * in osfmk/kern/cpu_data.h
  97          */
  98         /* return get_interrupt_level(); */
  99         return (ml_at_interrupt_context() ? 1 : 0);
 100 }
 101
 102 #if __ARM_SMP__
 103 /*
 104  * MP coordination
 105  */
 106
 107 decl_lck_mtx_data(static, dt_xc_lock);
 108 static uint32_t dt_xc_sync;
 109
 110 typedef struct xcArg {
 111         processorid_t   cpu;
 112         dtrace_xcall_t  f;
 113         void           *arg;
 114 } xcArg_t;
 115
 116 static void
 117 xcRemote(void *foo)
 118 {
 119         xcArg_t *pArg = (xcArg_t *) foo;
 120
 121         if (pArg->cpu == CPU->cpu_id || pArg->cpu == DTRACE_CPUALL)
 122                 (pArg->f) (pArg->arg);
 123
 124         if (hw_atomic_sub(&dt_xc_sync, 1) == 0)
 125                 thread_wakeup((event_t) &dt_xc_sync);
 126 }
 127 #endif
 128
 129 /*
 130  * dtrace_xcall() is not called from probe context.
 131  */
 132 void
 133 dtrace_xcall(processorid_t cpu, dtrace_xcall_t f, void *arg)
 134 {
 135 #if __ARM_SMP__
 136         /* Only one dtrace_xcall in flight allowed */
 137         lck_mtx_lock(&dt_xc_lock);
 138
 139         xcArg_t xcArg;
 140
 141         xcArg.cpu = cpu;
 142         xcArg.f = f;
 143         xcArg.arg = arg;
 144
 145         cpu_broadcast_xcall(&dt_xc_sync, TRUE, xcRemote, (void*) &xcArg);
 146
 147         lck_mtx_unlock(&dt_xc_lock);
 148         return;
 149 #else
 150 #pragma unused(cpu)
 151         /* On uniprocessor systems, the cpu should always be either ourselves or all */
 152         ASSERT(cpu == CPU->cpu_id || cpu == DTRACE_CPUALL);
 153
 154         (*f)(arg);
 155         return;
 156 #endif
 157 }
 158
 159 /*
 160  * Initialization
 161  */
 162 void
 163 dtrace_isa_init(void)
 164 {
 165 #if __ARM_SMP__
 166         lck_mtx_init(&dt_xc_lock, dtrace_lck_grp, dtrace_lck_attr);
 167 #endif
 168         return;
 169 }
 170
 171 /*
 172  * Runtime and ABI
 173  */
 174 uint64_t
 175 dtrace_getreg(struct regs * savearea, uint_t reg)
 176 {
 177         struct arm_saved_state *regs = (struct arm_saved_state *) savearea;
 178
 179         /* beyond register limit? */
 180         if (reg > ARM_SAVED_STATE32_COUNT - 1) {
 181                 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
 182                 return (0);
 183         }
 184         return (uint64_t) ((unsigned int *) (&(regs->r)))[reg];
 185 }
 186
 187 #define RETURN_OFFSET 4
 188
 189 static int
 190 dtrace_getustack_common(uint64_t * pcstack, int pcstack_limit, user_addr_t pc,
 191                         user_addr_t sp)
 192 {
 193         int ret = 0;
 194
 195         ASSERT(pcstack == NULL || pcstack_limit > 0);
 196
 197         while (pc != 0) {
 198                 ret++;
 199                 if (pcstack != NULL) {
 200                         *pcstack++ = (uint64_t) pc;
 201                         pcstack_limit--;
 202                         if (pcstack_limit <= 0)
 203                                 break;
 204                 }
 205
 206                 if (sp == 0)
 207                         break;
 208
 209                 pc = dtrace_fuword32((sp + RETURN_OFFSET));
 210                 sp = dtrace_fuword32(sp);
 211         }
 212
 213         return (ret);
 214 }
 215
 216 void
 217 dtrace_getupcstack(uint64_t * pcstack, int pcstack_limit)
 218 {
 219         thread_t        thread = current_thread();
 220         savearea_t     *regs;
 221         user_addr_t     pc, sp;
 222         volatile uint16_t *flags = (volatile uint16_t *) & cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
 223         int n;
 224
 225         if (*flags & CPU_DTRACE_FAULT)
 226                 return;
 227
 228         if (pcstack_limit <= 0)
 229                 return;
 230
 231         /*
 232          * If there's no user context we still need to zero the stack.
 233          */
 234         if (thread == NULL)
 235                 goto zero;
 236
 237         regs = (savearea_t *) find_user_regs(thread);
 238         if (regs == NULL)
 239                 goto zero;
 240
 241         *pcstack++ = (uint64_t)dtrace_proc_selfpid();
 242         pcstack_limit--;
 243
 244         if (pcstack_limit <= 0)
 245                 return;
 246
 247         pc = regs->pc;
 248         sp = regs->sp;
 249
 250         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
 251                 *pcstack++ = (uint64_t) pc;
 252                 pcstack_limit--;
 253                 if (pcstack_limit <= 0)
 254                         return;
 255
 256                 pc = regs->lr;
 257         }
 258
 259         n = dtrace_getustack_common(pcstack, pcstack_limit, pc, regs->r[7]);
 260
 261         ASSERT(n >= 0);
 262         ASSERT(n <= pcstack_limit);
 263
 264         pcstack += n;
 265         pcstack_limit -= n;
 266
 267 zero:
 268         while (pcstack_limit-- > 0)
 269                 *pcstack++ = 0ULL;
 270 }
 271
 272 int
 273 dtrace_getustackdepth(void)
 274 {
 275         thread_t        thread = current_thread();
 276         savearea_t     *regs;
 277         user_addr_t     pc, sp;
 278         int             n = 0;
 279
 280         if (thread == NULL)
 281                 return 0;
 282
 283         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT))
 284                 return (-1);
 285
 286         regs = (savearea_t *) find_user_regs(thread);
 287         if (regs == NULL)
 288                 return 0;
 289
 290         pc = regs->pc;
 291         sp = regs->sp;
 292
 293         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
 294                 n++;
 295                 pc = regs->lr;
 296         }
 297
 298         /*
 299          * Note that unlike ppc, the arm code does not use
 300          * CPU_DTRACE_USTACK_FP. This is because arm always
 301          * traces from the sp, even in syscall/profile/fbt
 302          * providers.
 303          */
 304
 305         n += dtrace_getustack_common(NULL, 0, pc, regs->r[7]);
 306
 307         return (n);
 308 }
 309
 310 void
 311 dtrace_getufpstack(uint64_t * pcstack, uint64_t * fpstack, int pcstack_limit)
 312 {
 313         /* XXX ARMTODO 64vs32 */
 314         thread_t        thread = current_thread();
 315         savearea_t      *regs;
 316         user_addr_t     pc, sp;
 317
 318         volatile        uint16_t  *flags = (volatile uint16_t *) & cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
 319
 320 #if 0
 321         uintptr_t oldcontext;
 322         size_t          s1, s2;
 323 #endif
 324
 325         if (*flags & CPU_DTRACE_FAULT)
 326                 return;
 327
 328         if (pcstack_limit <= 0)
 329                 return;
 330
 331         /*
 332          * If there's no user context we still need to zero the stack.
 333          */
 334         if (thread == NULL)
 335                 goto zero;
 336
 337         regs = (savearea_t *) find_user_regs(thread);
 338         if (regs == NULL)
 339                 goto zero;
 340
 341         *pcstack++ = (uint64_t)dtrace_proc_selfpid();
 342         pcstack_limit--;
 343
 344         if (pcstack_limit <= 0)
 345                 return;
 346
 347         pc = regs->pc;
 348         sp = regs->sp;
 349
 350 #if 0                           /* XXX signal stack crawl */
 351         oldcontext = lwp->lwp_oldcontext;
 352
 353         if (p->p_model == DATAMODEL_NATIVE) {
 354                 s1 = sizeof(struct frame) + 2 * sizeof(long);
 355                 s2 = s1 + sizeof(siginfo_t);
 356         } else {
 357                 s1 = sizeof(struct frame32) + 3 * sizeof(int);
 358                 s2 = s1 + sizeof(siginfo32_t);
 359         }
 360 #endif
 361
 362         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
 363                 *pcstack++ = (uint64_t) pc;
 364                 *fpstack++ = 0;
 365                 pcstack_limit--;
 366                 if (pcstack_limit <= 0)
 367                         return;
 368
 369                 pc = dtrace_fuword32(sp);
 370         }
 371         while (pc != 0 && sp != 0) {
 372                 *pcstack++ = (uint64_t) pc;
 373                 *fpstack++ = sp;
 374                 pcstack_limit--;
 375                 if (pcstack_limit <= 0)
 376                         break;
 377
 378 #if 0                           /* XXX signal stack crawl */
 379                 if (oldcontext == sp + s1 || oldcontext == sp + s2) {
 380                         if (p->p_model == DATAMODEL_NATIVE) {
 381                                 ucontext_t     *ucp = (ucontext_t *) oldcontext;
 382                                 greg_t         *gregs = ucp->uc_mcontext.gregs;
 383
 384                                 sp = dtrace_fulword(&gregs[REG_FP]);
 385                                 pc = dtrace_fulword(&gregs[REG_PC]);
 386
 387                                 oldcontext = dtrace_fulword(&ucp->uc_link);
 388                         } else {
 389                                 ucontext_t     *ucp = (ucontext_t *) oldcontext;
 390                                 greg_t         *gregs = ucp->uc_mcontext.gregs;
 391
 392                                 sp = dtrace_fuword32(&gregs[EBP]);
 393                                 pc = dtrace_fuword32(&gregs[EIP]);
 394
 395                                 oldcontext = dtrace_fuword32(&ucp->uc_link);
 396                         }
 397                 } else
 398 #endif
 399                 {
 400                         pc = dtrace_fuword32((sp + RETURN_OFFSET));
 401                         sp = dtrace_fuword32(sp);
 402                 }
 403
 404 #if 0
 405                 /* XXX ARMTODO*/
 406                 /*
 407                  * This is totally bogus:  if we faulted, we're going to clear
 408                  * the fault and break.  This is to deal with the apparently
 409                  * broken Java stacks on x86.
 410                  */
 411                 if (*flags & CPU_DTRACE_FAULT) {
 412                         *flags &= ~CPU_DTRACE_FAULT;
 413                         break;
 414                 }
 415 #endif
 416         }
 417
 418 zero:
 419         while (pcstack_limit-- > 0)
 420                 *pcstack++ = 0ULL;
 421 }
 422
 423 void
 424 dtrace_getpcstack(pc_t * pcstack, int pcstack_limit, int aframes,
 425                   uint32_t * intrpc)
 426 {
 427         struct frame   *fp = (struct frame *) __builtin_frame_address(0);
 428         struct frame   *nextfp, *minfp, *stacktop;
 429         int             depth = 0;
 430         int             on_intr;
 431         int             last = 0;
 432         uintptr_t       pc;
 433         uintptr_t       caller = CPU->cpu_dtrace_caller;
 434
 435         if ((on_intr = CPU_ON_INTR(CPU)) != 0)
 436                 stacktop = (struct frame *) dtrace_get_cpu_int_stack_top();
 437         else
 438                 stacktop = (struct frame *) (dtrace_get_kernel_stack(current_thread()) + kernel_stack_size);
 439
 440         minfp = fp;
 441
 442         aframes++;
 443
 444         if (intrpc != NULL && depth < pcstack_limit)
 445                 pcstack[depth++] = (pc_t) intrpc;
 446
 447         while (depth < pcstack_limit) {
 448                 nextfp = *(struct frame **) fp;
 449                 pc = *(uintptr_t *) (((uint32_t) fp) + RETURN_OFFSET);
 450
 451                 if (nextfp <= minfp || nextfp >= stacktop) {
 452                         if (on_intr) {
 453                                 /*
 454                                  * Hop from interrupt stack to thread stack.
 455                                  */
 456                                 arm_saved_state_t *arm_kern_regs = (arm_saved_state_t *) find_kern_regs(current_thread());
 457                                 if (arm_kern_regs) {
 458                                         nextfp = (struct frame *)arm_kern_regs->r[7];
 459
 460                                         vm_offset_t kstack_base = dtrace_get_kernel_stack(current_thread());
 461
 462                                         minfp = (struct frame *)kstack_base;
 463                                         stacktop = (struct frame *)(kstack_base + kernel_stack_size);
 464
 465                                         on_intr = 0;
 466
 467                                         if (nextfp <= minfp || nextfp >= stacktop) {
 468                                                 last = 1;
 469                                         }
 470                                 } else {
 471                                         /*
 472                                          * If this thread was on the interrupt stack, but did not
 473                                          * take an interrupt (i.e, the idle thread), there is no
 474                                          * explicit saved state for us to use.
 475                                          */
 476                                         last = 1;
 477                                 }
 478                         } else {
 479                                 /*
 480                                  * This is the last frame we can process; indicate
 481                                  * that we should return after processing this frame.
 482                                  */
 483                                 last = 1;
 484                         }
 485                 }
 486                 if (aframes > 0) {
 487                         if (--aframes == 0 && caller != (uintptr_t)NULL) {
 488                                 /*
 489                                  * We've just run out of artificial frames,
 490                                  * and we have a valid caller -- fill it in
 491                                  * now.
 492                                  */
 493                                 ASSERT(depth < pcstack_limit);
 494                                 pcstack[depth++] = (pc_t) caller;
 495                                 caller = (uintptr_t)NULL;
 496                         }
 497                 } else {
 498                         if (depth < pcstack_limit)
 499                                 pcstack[depth++] = (pc_t) pc;
 500                 }
 501
 502                 if (last) {
 503                         while (depth < pcstack_limit)
 504                                 pcstack[depth++] = (pc_t) NULL;
 505                         return;
 506                 }
 507                 fp = nextfp;
 508                 minfp = fp;
 509         }
 510 }
 511
 512 int
 513 dtrace_instr_size(uint32_t instr, int thumb_mode)
 514 {
 515         if (thumb_mode) {
 516                 uint16_t instr16 = *(uint16_t*) &instr;
 517                 if (((instr16 >> 11) & 0x1F) > 0x1C)
 518                         return 4;
 519                 else
 520                         return 2;
 521         } else {
 522                 return 4;
 523         }
 524 }
 525
 526 uint64_t
 527 dtrace_getarg(int arg, int aframes, dtrace_mstate_t *mstate, dtrace_vstate_t *vstate)
 528 {
 529 #pragma unused(arg, aframes, mstate, vstate)
 530 #if 0
 531         /* XXX ARMTODO */
 532         uint64_t val;
 533         uintptr_t *fp = (uintptr_t *)__builtin_frame_address(0);
 534         uintptr_t *stack;
 535         uintptr_t pc;
 536         int i;
 537
 538         for (i = 1; i <= aframes; i++) {
 539                 fp = fp[0];
 540                 pc = fp[1];
 541
 542                 if (dtrace_invop_callsite_pre != NULL
 543                         && pc  >  (uintptr_t)dtrace_invop_callsite_pre
 544                         && pc  <= (uintptr_t)dtrace_invop_callsite_post) {
 545                         /*
 546                          * If we pass through the invalid op handler, we will
 547                          * use the pointer that it passed to the stack as the
 548                          * second argument to dtrace_invop() as the pointer to
 549                          * the frame we're hunting for.
 550                          */
 551
 552                         stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */
 553                         fp = (struct frame *)stack[1]; /* Grab *second* argument */
 554                         stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */
 555                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
 556                         val = (uint64_t)(stack[arg]);
 557                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
 558                         return val;
 559                 }
 560         }
 561
 562         /*
 563          * Arrive here when provider has called dtrace_probe directly.
 564          */
 565         stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */
 566         stack++; /* Advance past probeID */
 567
 568         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
 569         val = *(((uint64_t *)stack) + arg); /* dtrace_probe arguments arg0 .. arg4 are 64bits wide */
 570         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
 571         return (val);
 572 #endif
 573         return 0xfeedfacedeafbeadLL;
 574 }
 575
 576 void
 577 dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which,
 578                 int fltoffs, int fault, uint64_t illval)
 579 {
 580         /* XXX ARMTODO */
 581         /*
 582          * For the case of the error probe firing lets
 583          * stash away "illval" here, and special-case retrieving it in DIF_VARIABLE_ARG.
 584          */
 585         state->dts_arg_error_illval = illval;
 586         dtrace_probe( dtrace_probeid_error, (uint64_t)(uintptr_t)state, epid, which, fltoffs, fault );
 587 }
 588
 589 void
 590 dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit))
 591 {
 592         /* XXX ARMTODO check copied from ppc/x86*/
 593         /*
 594          * "base" is the smallest toxic address in the range, "limit" is the first
 595          * VALID address greater than "base".
 596          */
 597         func(0x0, VM_MIN_KERNEL_ADDRESS);
 598         if (VM_MAX_KERNEL_ADDRESS < ~(uintptr_t)0)
 599                         func(VM_MAX_KERNEL_ADDRESS + 1, ~(uintptr_t)0);
 600 }
 601
 602 int
 603 dtrace_arm_condition_true(int cond, int cpsr)
 604 {
 605         int taken = 0;
 606         int zf = (cpsr & PSR_ZF) ? 1 : 0,
 607             nf = (cpsr & PSR_NF) ? 1 : 0,
 608             cf = (cpsr & PSR_CF) ? 1 : 0,
 609             vf = (cpsr & PSR_VF) ? 1 : 0;
 610
 611         switch(cond) {
 612                 case 0: taken = zf; break;
 613                 case 1: taken = !zf; break;
 614                 case 2: taken = cf; break;
 615                 case 3: taken = !cf; break;
 616                 case 4: taken = nf; break;
 617                 case 5: taken = !nf; break;
 618                 case 6: taken = vf; break;
 619                 case 7: taken = !vf; break;
 620                 case 8: taken = (cf && !zf); break;
 621                 case 9: taken = (!cf || zf); break;
 622                 case 10: taken = (nf == vf); break;
 623                 case 11: taken = (nf != vf); break;
 624                 case 12: taken = (!zf && (nf == vf)); break;
 625                 case 13: taken = (zf || (nf != vf)); break;
 626                 case 14: taken = 1; break;
 627                 case 15: taken = 1; break; /* always "true" for ARM, unpredictable for THUMB. */
 628         }
 629
 630         return taken;
 631 }