bsd/dev/arm64/dtrace_isa.c

   1 /*
   2  * Copyright (c) 2005-2018 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <arm/caches_internal.h>
  30 #include <kern/thread.h>
  31
  32 #if __has_include(<ptrauth.h>)
  33 #include <ptrauth.h>
  34 #endif
  35 #include <stdarg.h>
  36 #include <sys/time.h>
  37 #include <sys/systm.h>
  38 #include <sys/proc.h>
  39 #include <sys/proc_internal.h>
  40 #include <sys/kauth.h>
  41 #include <sys/dtrace.h>
  42 #include <sys/dtrace_impl.h>
  43 #include <machine/atomic.h>
  44 #include <kern/simple_lock.h>
  45 #include <kern/sched_prim.h>            /* for thread_wakeup() */
  46 #include <kern/thread_call.h>
  47 #include <kern/task.h>
  48
  49 extern struct arm_saved_state *find_kern_regs(thread_t);
  50
  51 extern dtrace_id_t      dtrace_probeid_error;   /* special ERROR probe */
  52 typedef arm_saved_state_t savearea_t;
  53
  54 extern lck_attr_t       *dtrace_lck_attr;
  55 extern lck_grp_t        *dtrace_lck_grp;
  56
  57
  58 struct frame {
  59         struct frame *backchain;
  60         uintptr_t retaddr;
  61 };
  62
  63 /*
  64  * Atomicity and synchronization
  65  */
  66 inline void
  67 dtrace_membar_producer(void)
  68 {
  69 #if __ARM_SMP__
  70         __asm__ volatile ("dmb ish" : : : "memory");
  71 #else
  72         __asm__ volatile ("nop" : : : "memory");
  73 #endif
  74 }
  75
  76 inline void
  77 dtrace_membar_consumer(void)
  78 {
  79 #if __ARM_SMP__
  80         __asm__ volatile ("dmb ish" : : : "memory");
  81 #else
  82         __asm__ volatile ("nop" : : : "memory");
  83 #endif
  84 }
  85
  86 /*
  87  * Interrupt manipulation
  88  * XXX dtrace_getipl() can be called from probe context.
  89  */
  90 int
  91 dtrace_getipl(void)
  92 {
  93         /*
  94          * XXX Drat, get_interrupt_level is MACH_KERNEL_PRIVATE
  95          * in osfmk/kern/cpu_data.h
  96          */
  97         /* return get_interrupt_level(); */
  98         return ml_at_interrupt_context() ? 1 : 0;
  99 }
 100
 101 #if __ARM_SMP__
 102 /*
 103  * MP coordination
 104  */
 105
 106 decl_lck_mtx_data(static, dt_xc_lock);
 107 static uint32_t dt_xc_sync;
 108
 109 typedef struct xcArg {
 110         processorid_t   cpu;
 111         dtrace_xcall_t  f;
 112         void           *arg;
 113 } xcArg_t;
 114
 115 static void
 116 xcRemote(void *foo)
 117 {
 118         xcArg_t *pArg = (xcArg_t *) foo;
 119
 120         if (pArg->cpu == CPU->cpu_id || pArg->cpu == DTRACE_CPUALL) {
 121                 (pArg->f)(pArg->arg);
 122         }
 123
 124         if (os_atomic_dec(&dt_xc_sync, relaxed) == 0) {
 125                 thread_wakeup((event_t) &dt_xc_sync);
 126         }
 127 }
 128 #endif
 129
 130 /*
 131  * dtrace_xcall() is not called from probe context.
 132  */
 133 void
 134 dtrace_xcall(processorid_t cpu, dtrace_xcall_t f, void *arg)
 135 {
 136 #if __ARM_SMP__
 137         /* Only one dtrace_xcall in flight allowed */
 138         lck_mtx_lock(&dt_xc_lock);
 139
 140         xcArg_t xcArg;
 141
 142         xcArg.cpu = cpu;
 143         xcArg.f = f;
 144         xcArg.arg = arg;
 145
 146         cpu_broadcast_xcall(&dt_xc_sync, TRUE, xcRemote, (void*) &xcArg);
 147
 148         lck_mtx_unlock(&dt_xc_lock);
 149         return;
 150 #else
 151 #pragma unused(cpu)
 152         /* On uniprocessor systems, the cpu should always be either ourselves or all */
 153         ASSERT(cpu == CPU->cpu_id || cpu == DTRACE_CPUALL);
 154
 155         (*f)(arg);
 156         return;
 157 #endif
 158 }
 159
 160 /*
 161  * Initialization
 162  */
 163 void
 164 dtrace_isa_init(void)
 165 {
 166         lck_mtx_init(&dt_xc_lock, dtrace_lck_grp, dtrace_lck_attr);
 167         return;
 168 }
 169
 170
 171 /**
 172  * Register definitions
 173  */
 174 #define ARM64_FP 29
 175 #define ARM64_LR 30
 176 #define ARM64_SP 31
 177 #define ARM64_PC 32
 178 #define ARM64_CPSR 33
 179
 180 /*
 181  * Runtime and ABI
 182  */
 183 uint64_t
 184 dtrace_getreg(struct regs * savearea, uint_t reg)
 185 {
 186         struct arm_saved_state *regs = (struct arm_saved_state *) savearea;
 187
 188         if (regs == NULL) {
 189                 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
 190                 return 0;
 191         }
 192
 193         if (!check_saved_state_reglimit(regs, reg)) {
 194                 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
 195                 return 0;
 196         }
 197
 198         return (uint64_t)get_saved_state_reg(regs, reg);
 199 }
 200
 201 #define RETURN_OFFSET64 8
 202
 203 static int
 204 dtrace_getustack_common(uint64_t * pcstack, int pcstack_limit, user_addr_t pc,
 205     user_addr_t sp)
 206 {
 207         int ret = 0;
 208
 209         ASSERT(pcstack == NULL || pcstack_limit > 0);
 210
 211         while (pc != 0) {
 212                 ret++;
 213                 if (pcstack != NULL) {
 214                         *pcstack++ = (uint64_t) pc;
 215                         pcstack_limit--;
 216                         if (pcstack_limit <= 0) {
 217                                 break;
 218                         }
 219                 }
 220
 221                 if (sp == 0) {
 222                         break;
 223                 }
 224
 225                 pc = dtrace_fuword64((sp + RETURN_OFFSET64));
 226                 sp = dtrace_fuword64(sp);
 227         }
 228
 229         return ret;
 230 }
 231
 232 void
 233 dtrace_getupcstack(uint64_t * pcstack, int pcstack_limit)
 234 {
 235         thread_t        thread = current_thread();
 236         savearea_t     *regs;
 237         user_addr_t     pc, sp, fp;
 238         volatile uint16_t *flags = (volatile uint16_t *) &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
 239         int n;
 240
 241         if (*flags & CPU_DTRACE_FAULT) {
 242                 return;
 243         }
 244
 245         if (pcstack_limit <= 0) {
 246                 return;
 247         }
 248
 249         /*
 250          * If there's no user context we still need to zero the stack.
 251          */
 252         if (thread == NULL) {
 253                 goto zero;
 254         }
 255
 256         regs = (savearea_t *) find_user_regs(thread);
 257         if (regs == NULL) {
 258                 goto zero;
 259         }
 260
 261         *pcstack++ = (uint64_t)dtrace_proc_selfpid();
 262         pcstack_limit--;
 263
 264         if (pcstack_limit <= 0) {
 265                 return;
 266         }
 267
 268         pc = get_saved_state_pc(regs);
 269         sp = get_saved_state_sp(regs);
 270         fp = get_saved_state_fp(regs);
 271
 272         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
 273                 *pcstack++ = (uint64_t) pc;
 274                 pcstack_limit--;
 275                 if (pcstack_limit <= 0) {
 276                         return;
 277                 }
 278
 279                 pc = get_saved_state_lr(regs);
 280         }
 281
 282         n = dtrace_getustack_common(pcstack, pcstack_limit, pc, fp);
 283
 284         ASSERT(n >= 0);
 285         ASSERT(n <= pcstack_limit);
 286
 287         pcstack += n;
 288         pcstack_limit -= n;
 289
 290 zero:
 291         while (pcstack_limit-- > 0) {
 292                 *pcstack++ = 0ULL;
 293         }
 294 }
 295
 296 int
 297 dtrace_getustackdepth(void)
 298 {
 299         thread_t        thread = current_thread();
 300         savearea_t     *regs;
 301         user_addr_t     pc, sp, fp;
 302         int             n = 0;
 303
 304         if (thread == NULL) {
 305                 return 0;
 306         }
 307
 308         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT)) {
 309                 return -1;
 310         }
 311
 312         regs = (savearea_t *) find_user_regs(thread);
 313         if (regs == NULL) {
 314                 return 0;
 315         }
 316
 317         pc = get_saved_state_pc(regs);
 318         sp = get_saved_state_sp(regs);
 319         fp = get_saved_state_fp(regs);
 320
 321         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
 322                 n++;
 323                 pc = get_saved_state_lr(regs);
 324         }
 325
 326         /*
 327          * Note that unlike ppc, the arm code does not use
 328          * CPU_DTRACE_USTACK_FP. This is because arm always
 329          * traces from the sp, even in syscall/profile/fbt
 330          * providers.
 331          */
 332
 333         n += dtrace_getustack_common(NULL, 0, pc, fp);
 334
 335         return n;
 336 }
 337
 338 void
 339 dtrace_getufpstack(uint64_t * pcstack, uint64_t * fpstack, int pcstack_limit)
 340 {
 341         thread_t        thread = current_thread();
 342         boolean_t       is64bit = proc_is64bit_data(current_proc());
 343         savearea_t      *regs;
 344         user_addr_t     pc, sp;
 345         volatile        uint16_t  *flags = (volatile uint16_t *) &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
 346
 347
 348         if (*flags & CPU_DTRACE_FAULT) {
 349                 return;
 350         }
 351
 352         if (pcstack_limit <= 0) {
 353                 return;
 354         }
 355
 356         /*
 357          * If there's no user context we still need to zero the stack.
 358          */
 359         if (thread == NULL) {
 360                 goto zero;
 361         }
 362
 363         regs = (savearea_t *) find_user_regs(thread);
 364         if (regs == NULL) {
 365                 goto zero;
 366         }
 367
 368         *pcstack++ = (uint64_t)dtrace_proc_selfpid();
 369         pcstack_limit--;
 370
 371         if (pcstack_limit <= 0) {
 372                 return;
 373         }
 374
 375         pc = get_saved_state_pc(regs);
 376         sp = get_saved_state_lr(regs);
 377
 378 #if 0                           /* XXX signal stack crawl */
 379         oldcontext = lwp->lwp_oldcontext;
 380
 381         if (p->p_model == DATAMODEL_NATIVE) {
 382                 s1 = sizeof(struct frame) + 2 * sizeof(long);
 383                 s2 = s1 + sizeof(siginfo_t);
 384         } else {
 385                 s1 = sizeof(struct frame32) + 3 * sizeof(int);
 386                 s2 = s1 + sizeof(siginfo32_t);
 387         }
 388 #endif
 389
 390         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
 391                 *pcstack++ = (uint64_t) pc;
 392                 *fpstack++ = 0;
 393                 pcstack_limit--;
 394                 if (pcstack_limit <= 0) {
 395                         return;
 396                 }
 397
 398                 if (is64bit) {
 399                         pc = dtrace_fuword64(sp);
 400                 } else {
 401                         pc = dtrace_fuword32(sp);
 402                 }
 403         }
 404         while (pc != 0 && sp != 0) {
 405                 *pcstack++ = (uint64_t) pc;
 406                 *fpstack++ = sp;
 407                 pcstack_limit--;
 408                 if (pcstack_limit <= 0) {
 409                         break;
 410                 }
 411
 412 #if 0                           /* XXX signal stack crawl */
 413                 if (oldcontext == sp + s1 || oldcontext == sp + s2) {
 414                         if (p->p_model == DATAMODEL_NATIVE) {
 415                                 ucontext_t     *ucp = (ucontext_t *) oldcontext;
 416                                 greg_t         *gregs = ucp->uc_mcontext.gregs;
 417
 418                                 sp = dtrace_fulword(&gregs[REG_FP]);
 419                                 pc = dtrace_fulword(&gregs[REG_PC]);
 420
 421                                 oldcontext = dtrace_fulword(&ucp->uc_link);
 422                         } else {
 423                                 ucontext_t     *ucp = (ucontext_t *) oldcontext;
 424                                 greg_t         *gregs = ucp->uc_mcontext.gregs;
 425
 426                                 sp = dtrace_fuword32(&gregs[EBP]);
 427                                 pc = dtrace_fuword32(&gregs[EIP]);
 428
 429                                 oldcontext = dtrace_fuword32(&ucp->uc_link);
 430                         }
 431                 } else
 432 #endif
 433                 {
 434                         pc = dtrace_fuword64((sp + RETURN_OFFSET64));
 435                         sp = dtrace_fuword64(sp);
 436                 }
 437
 438 #if 0
 439                 /* XXX ARMTODO*/
 440                 /*
 441                  * This is totally bogus:  if we faulted, we're going to clear
 442                  * the fault and break.  This is to deal with the apparently
 443                  * broken Java stacks on x86.
 444                  */
 445                 if (*flags & CPU_DTRACE_FAULT) {
 446                         *flags &= ~CPU_DTRACE_FAULT;
 447                         break;
 448                 }
 449 #endif
 450         }
 451
 452 zero:
 453         while (pcstack_limit-- > 0) {
 454                 *pcstack++ = 0ULL;
 455         }
 456 }
 457
 458
 459 void
 460 dtrace_getpcstack(pc_t * pcstack, int pcstack_limit, int aframes,
 461     uint32_t * intrpc)
 462 {
 463         struct frame   *fp = (struct frame *) __builtin_frame_address(0);
 464         struct frame   *nextfp, *minfp, *stacktop;
 465         int             depth = 0;
 466         int             on_intr;
 467         int             last = 0;
 468         uintptr_t       pc;
 469         uintptr_t       caller = CPU->cpu_dtrace_caller;
 470
 471         if ((on_intr = CPU_ON_INTR(CPU)) != 0) {
 472                 stacktop = (struct frame *) dtrace_get_cpu_int_stack_top();
 473         }
 474         else {
 475                 stacktop = (struct frame *) (dtrace_get_kernel_stack(current_thread()) + kernel_stack_size);
 476         }
 477
 478         minfp = fp;
 479
 480         aframes++;
 481
 482         if (intrpc != NULL && depth < pcstack_limit) {
 483                 pcstack[depth++] = (pc_t) intrpc;
 484         }
 485
 486         while (depth < pcstack_limit) {
 487                 nextfp = *(struct frame **) fp;
 488                 pc = *(uintptr_t *) (((uintptr_t) fp) + RETURN_OFFSET64);
 489
 490                 if (nextfp <= minfp || nextfp >= stacktop) {
 491                         if (on_intr) {
 492                                 /*
 493                                  * Hop from interrupt stack to thread stack.
 494                                  */
 495                                 arm_saved_state_t *arm_kern_regs = (arm_saved_state_t *) find_kern_regs(current_thread());
 496                                 if (arm_kern_regs) {
 497                                         nextfp = (struct frame *)(saved_state64(arm_kern_regs)->fp);
 498
 499                                         {
 500                                                 vm_offset_t kstack_base = dtrace_get_kernel_stack(current_thread());
 501
 502                                                 minfp = (struct frame *)kstack_base;
 503                                                 stacktop = (struct frame *)(kstack_base + kernel_stack_size);
 504                                         }
 505
 506                                         on_intr = 0;
 507
 508                                         if (nextfp <= minfp || nextfp >= stacktop) {
 509                                                 last = 1;
 510                                         }
 511                                 } else {
 512                                         /*
 513                                          * If this thread was on the interrupt stack, but did not
 514                                          * take an interrupt (i.e, the idle thread), there is no
 515                                          * explicit saved state for us to use.
 516                                          */
 517                                         last = 1;
 518                                 }
 519                         } else {
 520                                 {
 521                                         /*
 522                                          * This is the last frame we can process; indicate
 523                                          * that we should return after processing this frame.
 524                                          */
 525                                         last = 1;
 526                                 }
 527                         }
 528                 }
 529                 if (aframes > 0) {
 530                         if (--aframes == 0 && caller != (uintptr_t)NULL) {
 531                                 /*
 532                                  * We've just run out of artificial frames,
 533                                  * and we have a valid caller -- fill it in
 534                                  * now.
 535                                  */
 536                                 ASSERT(depth < pcstack_limit);
 537                                 pcstack[depth++] = (pc_t) caller;
 538                                 caller = (uintptr_t)NULL;
 539                         }
 540                 } else {
 541                         if (depth < pcstack_limit) {
 542                                 pcstack[depth++] = (pc_t) pc;
 543                         }
 544                 }
 545
 546                 if (last) {
 547                         while (depth < pcstack_limit) {
 548                                 pcstack[depth++] = (pc_t) NULL;
 549                         }
 550                         return;
 551                 }
 552                 fp = nextfp;
 553                 minfp = fp;
 554         }
 555 }
 556
 557 uint64_t
 558 dtrace_getarg(int arg, int aframes, dtrace_mstate_t *mstate, dtrace_vstate_t *vstate)
 559 {
 560 #pragma unused(arg, aframes)
 561         uint64_t val = 0;
 562         struct frame *fp = (struct frame *)__builtin_frame_address(0);
 563         uintptr_t *stack;
 564         uintptr_t pc;
 565         int i;
 566
 567         /*
 568          * A total of 8 arguments are passed via registers; any argument with
 569          * index of 7 or lower is therefore in a register.
 570          */
 571         int inreg = 7;
 572
 573         for (i = 1; i <= aframes; ++i) {
 574                 fp = fp->backchain;
 575 #if __has_feature(ptrauth_returns)
 576                 pc = (uintptr_t)ptrauth_strip((void*)fp->retaddr, ptrauth_key_return_address);
 577 #else
 578                 pc = fp->retaddr;
 579 #endif
 580
 581                 if (dtrace_invop_callsite_pre != NULL
 582                     && pc > (uintptr_t) dtrace_invop_callsite_pre
 583                     && pc <= (uintptr_t) dtrace_invop_callsite_post) {
 584                         /* fp points to frame of dtrace_invop() activation */
 585                         fp = fp->backchain; /* to fbt_perfCallback activation */
 586                         fp = fp->backchain; /* to sleh_synchronous activation */
 587                         fp = fp->backchain; /* to fleh_synchronous activation */
 588
 589                         arm_saved_state_t       *tagged_regs = (arm_saved_state_t*) ((void*) &fp[1]);
 590                         arm_saved_state64_t     *saved_state = saved_state64(tagged_regs);
 591
 592                         if (arg <= inreg) {
 593                                 /* the argument will be found in a register */
 594                                 stack = (uintptr_t*) &saved_state->x[0];
 595                         } else {
 596                                 /* the argument will be found in the stack */
 597                                 fp = (struct frame*) saved_state->sp;
 598                                 stack = (uintptr_t*) &fp[1];
 599                                 arg -= (inreg + 1);
 600                         }
 601
 602                         goto load;
 603                 }
 604         }
 605
 606         /*
 607          * We know that we did not come through a trap to get into
 608          * dtrace_probe() --  We arrive here when the provider has
 609          * called dtrace_probe() directly.
 610          * The probe ID is the first argument to dtrace_probe().
 611          * We must advance beyond that to get the argX.
 612          */
 613         arg++; /* Advance past probeID */
 614
 615         if (arg <= inreg) {
 616                 /*
 617                  * This shouldn't happen.  If the argument is passed in a
 618                  * register then it should have been, well, passed in a
 619                  * register...
 620                  */
 621                 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
 622                 return 0;
 623         }
 624
 625         arg -= (inreg + 1);
 626         stack = (uintptr_t*) &fp[1]; /* Find marshalled arguments */
 627
 628 load:
 629         if (dtrace_canload((uint64_t)(stack + arg), sizeof(uint64_t),
 630             mstate, vstate)) {
 631                 /* dtrace_probe arguments arg0 ... arg4 are 64bits wide */
 632                 val = dtrace_load64((uint64_t)(stack + arg));
 633         }
 634
 635         return val;
 636 }
 637
 638 void
 639 dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which,
 640     int fltoffs, int fault, uint64_t illval)
 641 {
 642         /* XXX ARMTODO */
 643         /*
 644          * For the case of the error probe firing lets
 645          * stash away "illval" here, and special-case retrieving it in DIF_VARIABLE_ARG.
 646          */
 647         state->dts_arg_error_illval = illval;
 648         dtrace_probe( dtrace_probeid_error, (uint64_t)(uintptr_t)state, epid, which, fltoffs, fault );
 649 }
 650
 651 void
 652 dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit))
 653 {
 654         /* XXX ARMTODO check copied from ppc/x86*/
 655         /*
 656          * "base" is the smallest toxic address in the range, "limit" is the first
 657          * VALID address greater than "base".
 658          */
 659         func(0x0, VM_MIN_KERNEL_ADDRESS);
 660         if (VM_MAX_KERNEL_ADDRESS < ~(uintptr_t)0) {
 661                 func(VM_MAX_KERNEL_ADDRESS + 1, ~(uintptr_t)0);
 662         }
 663 }
 664
 665 void
 666 dtrace_flush_caches(void)
 667 {
 668         /* TODO There were some problems with flushing just the cache line that had been modified.
 669          * For now, we'll flush the entire cache, until we figure out how to flush just the patched block.
 670          */
 671         FlushPoU_Dcache();
 672         InvalidatePoU_Icache();
 673 }