bsd/dev/arm64/dtrace_isa.c

   1 /*
   2  * Copyright (c) 2005-2018 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <arm/caches_internal.h>
  30 #include <kern/thread.h>
  31
  32 #if __has_include(<ptrauth.h>)
  33 #include <ptrauth.h>
  34 #endif
  35 #include <stdarg.h>
  36 #include <sys/time.h>
  37 #include <sys/systm.h>
  38 #include <sys/proc.h>
  39 #include <sys/proc_internal.h>
  40 #include <sys/kauth.h>
  41 #include <sys/dtrace.h>
  42 #include <sys/dtrace_impl.h>
  43 #include <machine/atomic.h>
  44 #include <kern/cambria_layout.h>
  45 #include <kern/simple_lock.h>
  46 #include <kern/sched_prim.h>            /* for thread_wakeup() */
  47 #include <kern/thread_call.h>
  48 #include <kern/task.h>
  49
  50 extern struct arm_saved_state *find_kern_regs(thread_t);
  51
  52 extern dtrace_id_t      dtrace_probeid_error;   /* special ERROR probe */
  53 typedef arm_saved_state_t savearea_t;
  54
  55 #if XNU_MONITOR
  56 extern void * pmap_stacks_start;
  57 extern void * pmap_stacks_end;
  58 #endif
  59
  60 struct frame {
  61         struct frame *backchain;
  62         uintptr_t retaddr;
  63 };
  64
  65 /*
  66  * Atomicity and synchronization
  67  */
  68 inline void
  69 dtrace_membar_producer(void)
  70 {
  71         __asm__ volatile ("dmb ish" : : : "memory");
  72 }
  73
  74 inline void
  75 dtrace_membar_consumer(void)
  76 {
  77         __asm__ volatile ("dmb ish" : : : "memory");
  78 }
  79
  80 /*
  81  * Interrupt manipulation
  82  * XXX dtrace_getipl() can be called from probe context.
  83  */
  84 int
  85 dtrace_getipl(void)
  86 {
  87         /*
  88          * XXX Drat, get_interrupt_level is MACH_KERNEL_PRIVATE
  89          * in osfmk/kern/cpu_data.h
  90          */
  91         /* return get_interrupt_level(); */
  92         return ml_at_interrupt_context() ? 1 : 0;
  93 }
  94
  95 /*
  96  * MP coordination
  97  */
  98
  99 static LCK_MTX_DECLARE_ATTR(dt_xc_lock, &dtrace_lck_grp, &dtrace_lck_attr);
 100 static uint32_t dt_xc_sync;
 101
 102 typedef struct xcArg {
 103         processorid_t   cpu;
 104         dtrace_xcall_t  f;
 105         void           *arg;
 106 } xcArg_t;
 107
 108 static void
 109 xcRemote(void *foo)
 110 {
 111         xcArg_t *pArg = (xcArg_t *) foo;
 112
 113         if (pArg->cpu == CPU->cpu_id || pArg->cpu == DTRACE_CPUALL) {
 114                 (pArg->f)(pArg->arg);
 115         }
 116
 117         if (os_atomic_dec(&dt_xc_sync, relaxed) == 0) {
 118                 thread_wakeup((event_t) &dt_xc_sync);
 119         }
 120 }
 121
 122 /*
 123  * dtrace_xcall() is not called from probe context.
 124  */
 125 void
 126 dtrace_xcall(processorid_t cpu, dtrace_xcall_t f, void *arg)
 127 {
 128         /* Only one dtrace_xcall in flight allowed */
 129         lck_mtx_lock(&dt_xc_lock);
 130
 131         xcArg_t xcArg;
 132
 133         xcArg.cpu = cpu;
 134         xcArg.f = f;
 135         xcArg.arg = arg;
 136
 137         cpu_broadcast_xcall(&dt_xc_sync, TRUE, xcRemote, (void*) &xcArg);
 138
 139         lck_mtx_unlock(&dt_xc_lock);
 140         return;
 141 }
 142
 143
 144 /**
 145  * Register definitions
 146  */
 147 #define ARM64_FP 29
 148 #define ARM64_LR 30
 149 #define ARM64_SP 31
 150 #define ARM64_PC 32
 151 #define ARM64_CPSR 33
 152
 153 /*
 154  * Runtime and ABI
 155  */
 156 uint64_t
 157 dtrace_getreg(struct regs * savearea, uint_t reg)
 158 {
 159         struct arm_saved_state *regs = (struct arm_saved_state *) savearea;
 160
 161         if (regs == NULL) {
 162                 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
 163                 return 0;
 164         }
 165
 166         if (!check_saved_state_reglimit(regs, reg)) {
 167                 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
 168                 return 0;
 169         }
 170
 171         return (uint64_t)get_saved_state_reg(regs, reg);
 172 }
 173
 174 uint64_t
 175 dtrace_getvmreg(uint_t ndx)
 176 {
 177 #pragma unused(ndx)
 178         DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
 179         return 0;
 180 }
 181
 182 #define RETURN_OFFSET64 8
 183
 184 static int
 185 dtrace_getustack_common(uint64_t * pcstack, int pcstack_limit, user_addr_t pc,
 186     user_addr_t sp)
 187 {
 188         volatile uint16_t *flags = (volatile uint16_t *) &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
 189         int ret = 0;
 190
 191         ASSERT(pcstack == NULL || pcstack_limit > 0);
 192
 193         while (pc != 0) {
 194                 ret++;
 195                 if (pcstack != NULL) {
 196                         *pcstack++ = (uint64_t) pc;
 197                         pcstack_limit--;
 198                         if (pcstack_limit <= 0) {
 199                                 break;
 200                         }
 201                 }
 202
 203                 if (sp == 0) {
 204                         break;
 205                 }
 206
 207                 pc = dtrace_fuword64((sp + RETURN_OFFSET64));
 208                 sp = dtrace_fuword64(sp);
 209
 210                 /* Truncate ustack if the iterator causes fault. */
 211                 if (*flags & CPU_DTRACE_FAULT) {
 212                         *flags &= ~CPU_DTRACE_FAULT;
 213                         break;
 214                 }
 215         }
 216
 217         return ret;
 218 }
 219
 220 void
 221 dtrace_getupcstack(uint64_t * pcstack, int pcstack_limit)
 222 {
 223         thread_t thread = current_thread();
 224         savearea_t *regs;
 225         user_addr_t pc, sp, fp;
 226         volatile uint16_t *flags = (volatile uint16_t *) &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
 227         int n;
 228
 229         if (*flags & CPU_DTRACE_FAULT) {
 230                 return;
 231         }
 232
 233         if (pcstack_limit <= 0) {
 234                 return;
 235         }
 236
 237         /*
 238          * If there's no user context we still need to zero the stack.
 239          */
 240         if (thread == NULL) {
 241                 goto zero;
 242         }
 243
 244         regs = (savearea_t *) find_user_regs(thread);
 245         if (regs == NULL) {
 246                 goto zero;
 247         }
 248
 249         *pcstack++ = (uint64_t)dtrace_proc_selfpid();
 250         pcstack_limit--;
 251
 252         if (pcstack_limit <= 0) {
 253                 return;
 254         }
 255
 256         pc = get_saved_state_pc(regs);
 257         sp = get_saved_state_sp(regs);
 258
 259         {
 260                 fp = get_saved_state_fp(regs);
 261         }
 262
 263         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
 264                 *pcstack++ = (uint64_t) pc;
 265                 pcstack_limit--;
 266                 if (pcstack_limit <= 0) {
 267                         return;
 268                 }
 269
 270                 pc = get_saved_state_lr(regs);
 271         }
 272
 273         n = dtrace_getustack_common(pcstack, pcstack_limit, pc, fp);
 274
 275         ASSERT(n >= 0);
 276         ASSERT(n <= pcstack_limit);
 277
 278         pcstack += n;
 279         pcstack_limit -= n;
 280
 281 zero:
 282         while (pcstack_limit-- > 0) {
 283                 *pcstack++ = 0ULL;
 284         }
 285 }
 286
 287 int
 288 dtrace_getustackdepth(void)
 289 {
 290         thread_t        thread = current_thread();
 291         savearea_t     *regs;
 292         user_addr_t     pc, sp, fp;
 293         int             n = 0;
 294
 295         if (thread == NULL) {
 296                 return 0;
 297         }
 298
 299         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT)) {
 300                 return -1;
 301         }
 302
 303         regs = (savearea_t *) find_user_regs(thread);
 304         if (regs == NULL) {
 305                 return 0;
 306         }
 307
 308         pc = get_saved_state_pc(regs);
 309         sp = get_saved_state_sp(regs);
 310         fp = get_saved_state_fp(regs);
 311
 312         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
 313                 n++;
 314                 pc = get_saved_state_lr(regs);
 315         }
 316
 317         /*
 318          * Note that unlike ppc, the arm code does not use
 319          * CPU_DTRACE_USTACK_FP. This is because arm always
 320          * traces from the sp, even in syscall/profile/fbt
 321          * providers.
 322          */
 323
 324         n += dtrace_getustack_common(NULL, 0, pc, fp);
 325
 326         return n;
 327 }
 328
 329 void
 330 dtrace_getufpstack(uint64_t * pcstack, uint64_t * fpstack, int pcstack_limit)
 331 {
 332         thread_t        thread = current_thread();
 333         boolean_t       is64bit = proc_is64bit_data(current_proc());
 334         savearea_t      *regs;
 335         user_addr_t     pc, sp;
 336         volatile        uint16_t  *flags = (volatile uint16_t *) &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
 337
 338
 339         if (*flags & CPU_DTRACE_FAULT) {
 340                 return;
 341         }
 342
 343         if (pcstack_limit <= 0) {
 344                 return;
 345         }
 346
 347         /*
 348          * If there's no user context we still need to zero the stack.
 349          */
 350         if (thread == NULL) {
 351                 goto zero;
 352         }
 353
 354         regs = (savearea_t *) find_user_regs(thread);
 355         if (regs == NULL) {
 356                 goto zero;
 357         }
 358
 359         *pcstack++ = (uint64_t)dtrace_proc_selfpid();
 360         pcstack_limit--;
 361
 362         if (pcstack_limit <= 0) {
 363                 return;
 364         }
 365
 366         pc = get_saved_state_pc(regs);
 367         sp = get_saved_state_lr(regs);
 368
 369 #if 0                           /* XXX signal stack crawl */
 370         oldcontext = lwp->lwp_oldcontext;
 371
 372         if (p->p_model == DATAMODEL_NATIVE) {
 373                 s1 = sizeof(struct frame) + 2 * sizeof(long);
 374                 s2 = s1 + sizeof(siginfo_t);
 375         } else {
 376                 s1 = sizeof(struct frame32) + 3 * sizeof(int);
 377                 s2 = s1 + sizeof(siginfo32_t);
 378         }
 379 #endif
 380
 381         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
 382                 *pcstack++ = (uint64_t) pc;
 383                 *fpstack++ = 0;
 384                 pcstack_limit--;
 385                 if (pcstack_limit <= 0) {
 386                         return;
 387                 }
 388
 389                 if (is64bit) {
 390                         pc = dtrace_fuword64(sp);
 391                 } else {
 392                         pc = dtrace_fuword32(sp);
 393                 }
 394         }
 395         while (pc != 0 && sp != 0) {
 396                 *pcstack++ = (uint64_t) pc;
 397                 *fpstack++ = sp;
 398                 pcstack_limit--;
 399                 if (pcstack_limit <= 0) {
 400                         break;
 401                 }
 402
 403 #if 0                           /* XXX signal stack crawl */
 404                 if (oldcontext == sp + s1 || oldcontext == sp + s2) {
 405                         if (p->p_model == DATAMODEL_NATIVE) {
 406                                 ucontext_t     *ucp = (ucontext_t *) oldcontext;
 407                                 greg_t         *gregs = ucp->uc_mcontext.gregs;
 408
 409                                 sp = dtrace_fulword(&gregs[REG_FP]);
 410                                 pc = dtrace_fulword(&gregs[REG_PC]);
 411
 412                                 oldcontext = dtrace_fulword(&ucp->uc_link);
 413                         } else {
 414                                 ucontext_t     *ucp = (ucontext_t *) oldcontext;
 415                                 greg_t         *gregs = ucp->uc_mcontext.gregs;
 416
 417                                 sp = dtrace_fuword32(&gregs[EBP]);
 418                                 pc = dtrace_fuword32(&gregs[EIP]);
 419
 420                                 oldcontext = dtrace_fuword32(&ucp->uc_link);
 421                         }
 422                 } else
 423 #endif
 424                 {
 425                         pc = dtrace_fuword64((sp + RETURN_OFFSET64));
 426                         sp = dtrace_fuword64(sp);
 427                 }
 428
 429                 /* Truncate ustack if the iterator causes fault. */
 430                 if (*flags & CPU_DTRACE_FAULT) {
 431                         *flags &= ~CPU_DTRACE_FAULT;
 432                         break;
 433                 }
 434         }
 435
 436 zero:
 437         while (pcstack_limit-- > 0) {
 438                 *pcstack++ = 0ULL;
 439         }
 440 }
 441
 442 #if XNU_MONITOR
 443 static inline boolean_t
 444 dtrace_frame_in_ppl_stack(struct frame * fp)
 445 {
 446         return ((void *)fp >= pmap_stacks_start) &&
 447                ((void *)fp < pmap_stacks_end);
 448 }
 449 #endif
 450
 451 void
 452 dtrace_getpcstack(pc_t * pcstack, int pcstack_limit, int aframes,
 453     uint32_t * intrpc)
 454 {
 455         struct frame   *fp = (struct frame *) __builtin_frame_address(0);
 456         struct frame   *nextfp, *minfp, *stacktop;
 457         int             depth = 0;
 458         int             on_intr;
 459 #if XNU_MONITOR
 460         int             on_ppl_stack;
 461 #endif
 462         int             last = 0;
 463         uintptr_t       pc;
 464         uintptr_t       caller = CPU->cpu_dtrace_caller;
 465
 466         if ((on_intr = CPU_ON_INTR(CPU)) != 0) {
 467                 stacktop = (struct frame *) dtrace_get_cpu_int_stack_top();
 468         }
 469 #if XNU_MONITOR
 470         else if ((on_ppl_stack = dtrace_frame_in_ppl_stack(fp))) {
 471                 stacktop = (struct frame *) pmap_stacks_end;
 472         }
 473 #endif
 474         else {
 475                 stacktop = (struct frame *) (dtrace_get_kernel_stack(current_thread()) + kernel_stack_size);
 476         }
 477
 478         minfp = fp;
 479
 480         aframes++;
 481
 482         if (intrpc != NULL && depth < pcstack_limit) {
 483                 pcstack[depth++] = (pc_t) intrpc;
 484         }
 485
 486         while (depth < pcstack_limit) {
 487                 nextfp = *(struct frame **) fp;
 488                 pc = *(uintptr_t *) (((uintptr_t) fp) + RETURN_OFFSET64);
 489
 490                 if (nextfp <= minfp || nextfp >= stacktop) {
 491                         if (on_intr) {
 492                                 /*
 493                                  * Hop from interrupt stack to thread stack.
 494                                  */
 495                                 arm_saved_state_t *arm_kern_regs = (arm_saved_state_t *) find_kern_regs(current_thread());
 496                                 if (arm_kern_regs) {
 497                                         nextfp = (struct frame *)(saved_state64(arm_kern_regs)->fp);
 498
 499 #if XNU_MONITOR
 500                                         on_ppl_stack = dtrace_frame_in_ppl_stack(nextfp);
 501
 502                                         if (on_ppl_stack) {
 503                                                 minfp = pmap_stacks_start;
 504                                                 stacktop = pmap_stacks_end;
 505                                         } else
 506 #endif
 507                                         {
 508                                                 vm_offset_t kstack_base = dtrace_get_kernel_stack(current_thread());
 509
 510                                                 minfp = (struct frame *)kstack_base;
 511                                                 stacktop = (struct frame *)(kstack_base + kernel_stack_size);
 512                                         }
 513
 514                                         on_intr = 0;
 515
 516                                         if (nextfp <= minfp || nextfp >= stacktop) {
 517                                                 last = 1;
 518                                         }
 519                                 } else {
 520                                         /*
 521                                          * If this thread was on the interrupt stack, but did not
 522                                          * take an interrupt (i.e, the idle thread), there is no
 523                                          * explicit saved state for us to use.
 524                                          */
 525                                         last = 1;
 526                                 }
 527                         } else {
 528 #if XNU_MONITOR
 529                                 if ((!on_ppl_stack) && dtrace_frame_in_ppl_stack(nextfp)) {
 530                                         /*
 531                                          * We are switching from the kernel stack
 532                                          * to the PPL stack.
 533                                          */
 534                                         on_ppl_stack = 1;
 535                                         minfp = pmap_stacks_start;
 536                                         stacktop = pmap_stacks_end;
 537                                 } else if (on_ppl_stack) {
 538                                         /*
 539                                          * We could be going from the PPL stack
 540                                          * to the kernel stack.
 541                                          */
 542                                         vm_offset_t kstack_base = dtrace_get_kernel_stack(current_thread());
 543
 544                                         minfp = (struct frame *)kstack_base;
 545                                         stacktop = (struct frame *)(kstack_base + kernel_stack_size);
 546
 547                                         if (nextfp <= minfp || nextfp >= stacktop) {
 548                                                 last = 1;
 549                                         }
 550                                 } else
 551 #endif
 552                                 {
 553                                         /*
 554                                          * This is the last frame we can process; indicate
 555                                          * that we should return after processing this frame.
 556                                          */
 557                                         last = 1;
 558                                 }
 559                         }
 560                 }
 561                 if (aframes > 0) {
 562                         if (--aframes == 0 && caller != (uintptr_t)NULL) {
 563                                 /*
 564                                  * We've just run out of artificial frames,
 565                                  * and we have a valid caller -- fill it in
 566                                  * now.
 567                                  */
 568                                 ASSERT(depth < pcstack_limit);
 569                                 pcstack[depth++] = (pc_t) caller;
 570                                 caller = (uintptr_t)NULL;
 571                         }
 572                 } else {
 573                         if (depth < pcstack_limit) {
 574                                 pcstack[depth++] = (pc_t) pc;
 575                         }
 576                 }
 577
 578                 if (last) {
 579                         while (depth < pcstack_limit) {
 580                                 pcstack[depth++] = (pc_t) NULL;
 581                         }
 582                         return;
 583                 }
 584                 fp = nextfp;
 585                 minfp = fp;
 586         }
 587 }
 588
 589 uint64_t
 590 dtrace_getarg(int arg, int aframes, dtrace_mstate_t *mstate, dtrace_vstate_t *vstate)
 591 {
 592 #pragma unused(arg, aframes)
 593         uint64_t val = 0;
 594         struct frame *fp = (struct frame *)__builtin_frame_address(0);
 595         uintptr_t *stack;
 596         uintptr_t pc;
 597         int i;
 598
 599         /*
 600          * A total of 8 arguments are passed via registers; any argument with
 601          * index of 7 or lower is therefore in a register.
 602          */
 603         int inreg = 7;
 604
 605         for (i = 1; i <= aframes; ++i) {
 606                 fp = fp->backchain;
 607 #if __has_feature(ptrauth_returns)
 608                 pc = (uintptr_t)ptrauth_strip((void*)fp->retaddr, ptrauth_key_return_address);
 609 #else
 610                 pc = fp->retaddr;
 611 #endif
 612
 613                 if (dtrace_invop_callsite_pre != NULL
 614                     && pc > (uintptr_t) dtrace_invop_callsite_pre
 615                     && pc <= (uintptr_t) dtrace_invop_callsite_post) {
 616                         /* fp points to frame of dtrace_invop() activation */
 617                         fp = fp->backchain; /* to fbt_perfCallback activation */
 618                         fp = fp->backchain; /* to sleh_synchronous activation */
 619                         fp = fp->backchain; /* to fleh_synchronous activation */
 620
 621                         arm_saved_state_t       *tagged_regs = (arm_saved_state_t*) ((void*) &fp[1]);
 622                         arm_saved_state64_t     *saved_state = saved_state64(tagged_regs);
 623
 624                         if (arg <= inreg) {
 625                                 /* the argument will be found in a register */
 626                                 stack = (uintptr_t*) &saved_state->x[0];
 627                         } else {
 628                                 /* the argument will be found in the stack */
 629                                 fp = (struct frame*) saved_state->sp;
 630                                 stack = (uintptr_t*) &fp[1];
 631                                 arg -= (inreg + 1);
 632                         }
 633
 634                         goto load;
 635                 }
 636         }
 637
 638         /*
 639          * We know that we did not come through a trap to get into
 640          * dtrace_probe() --  We arrive here when the provider has
 641          * called dtrace_probe() directly.
 642          * The probe ID is the first argument to dtrace_probe().
 643          * We must advance beyond that to get the argX.
 644          */
 645         arg++; /* Advance past probeID */
 646
 647         if (arg <= inreg) {
 648                 /*
 649                  * This shouldn't happen.  If the argument is passed in a
 650                  * register then it should have been, well, passed in a
 651                  * register...
 652                  */
 653                 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
 654                 return 0;
 655         }
 656
 657         arg -= (inreg + 1);
 658         stack = (uintptr_t*) &fp[1]; /* Find marshalled arguments */
 659
 660 load:
 661         if (dtrace_canload((uint64_t)(stack + arg), sizeof(uint64_t),
 662             mstate, vstate)) {
 663                 /* dtrace_probe arguments arg0 ... arg4 are 64bits wide */
 664                 val = dtrace_load64((uint64_t)(stack + arg));
 665         }
 666
 667         return val;
 668 }
 669
 670 void
 671 dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which,
 672     int fltoffs, int fault, uint64_t illval)
 673 {
 674         /* XXX ARMTODO */
 675         /*
 676          * For the case of the error probe firing lets
 677          * stash away "illval" here, and special-case retrieving it in DIF_VARIABLE_ARG.
 678          */
 679         state->dts_arg_error_illval = illval;
 680         dtrace_probe( dtrace_probeid_error, (uint64_t)(uintptr_t)state, epid, which, fltoffs, fault );
 681 }
 682
 683 void
 684 dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit))
 685 {
 686         /* XXX ARMTODO check copied from ppc/x86*/
 687         /*
 688          * "base" is the smallest toxic address in the range, "limit" is the first
 689          * VALID address greater than "base".
 690          */
 691         func(0x0, VM_MIN_KERNEL_ADDRESS);
 692         if (VM_MAX_KERNEL_ADDRESS < ~(uintptr_t)0) {
 693                 func(VM_MAX_KERNEL_ADDRESS + 1, ~(uintptr_t)0);
 694         }
 695 }
 696
 697 void
 698 dtrace_flush_caches(void)
 699 {
 700         /* TODO There were some problems with flushing just the cache line that had been modified.
 701          * For now, we'll flush the entire cache, until we figure out how to flush just the patched block.
 702          */
 703         FlushPoU_Dcache();
 704         InvalidatePoU_Icache();
 705 }