[apple/xnu.git] / bsd / dev / arm64 / dtrace_isa.c

/*
 * Copyright (c) 2005-2018 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <arm/caches_internal.h>
#include <kern/thread.h>

#if __has_include(<ptrauth.h>)
#include <ptrauth.h>
#endif
#include <stdarg.h>
#include <sys/time.h>
#include <sys/systm.h>
#include <sys/proc.h>
#include <sys/proc_internal.h>
#include <sys/kauth.h>
#include <sys/dtrace.h>
#include <sys/dtrace_impl.h>
#include <machine/atomic.h>
#include <kern/simple_lock.h>
#include <kern/sched_prim.h>            /* for thread_wakeup() */
#include <kern/thread_call.h>
#include <kern/task.h>

extern struct arm_saved_state *find_kern_regs(thread_t);

extern dtrace_id_t      dtrace_probeid_error;   /* special ERROR probe */
typedef arm_saved_state_t savearea_t;

extern lck_attr_t       *dtrace_lck_attr;
extern lck_grp_t        *dtrace_lck_grp;

#if XNU_MONITOR
extern void * pmap_stacks_start;
extern void * pmap_stacks_end;
#endif

struct frame {
	struct frame *backchain;
	uintptr_t retaddr;
};

/*
 * Atomicity and synchronization
 */
inline void
dtrace_membar_producer(void)
{
#if __ARM_SMP__
	__asm__ volatile ("dmb ish" : : : "memory");
#else
	__asm__ volatile ("nop" : : : "memory");
#endif
}

inline void
dtrace_membar_consumer(void)
{
#if __ARM_SMP__
	__asm__ volatile ("dmb ish" : : : "memory");
#else
	__asm__ volatile ("nop" : : : "memory");
#endif
}

/*
 * Interrupt manipulation
 * XXX dtrace_getipl() can be called from probe context.
 */
int
dtrace_getipl(void)
{
	/*
	 * XXX Drat, get_interrupt_level is MACH_KERNEL_PRIVATE
	 * in osfmk/kern/cpu_data.h
	 */
	/* return get_interrupt_level(); */
	return ml_at_interrupt_context() ? 1 : 0;
}

#if __ARM_SMP__
/*
 * MP coordination
 */

decl_lck_mtx_data(static, dt_xc_lock);
static uint32_t dt_xc_sync;

typedef struct xcArg {
	processorid_t   cpu;
	dtrace_xcall_t  f;
	void           *arg;
} xcArg_t;

static void
xcRemote(void *foo)
{
	xcArg_t *pArg = (xcArg_t *) foo;

	if (pArg->cpu == CPU->cpu_id || pArg->cpu == DTRACE_CPUALL) {
		(pArg->f)(pArg->arg);
	}

	if (os_atomic_dec(&dt_xc_sync, relaxed) == 0) {
		thread_wakeup((event_t) &dt_xc_sync);
	}
}
#endif

/*
 * dtrace_xcall() is not called from probe context.
 */
void
dtrace_xcall(processorid_t cpu, dtrace_xcall_t f, void *arg)
{
#if __ARM_SMP__
	/* Only one dtrace_xcall in flight allowed */
	lck_mtx_lock(&dt_xc_lock);

	xcArg_t xcArg;

	xcArg.cpu = cpu;
	xcArg.f = f;
	xcArg.arg = arg;

	cpu_broadcast_xcall(&dt_xc_sync, TRUE, xcRemote, (void*) &xcArg);

	lck_mtx_unlock(&dt_xc_lock);
	return;
#else
#pragma unused(cpu)
	/* On uniprocessor systems, the cpu should always be either ourselves or all */
	ASSERT(cpu == CPU->cpu_id || cpu == DTRACE_CPUALL);

	(*f)(arg);
	return;
#endif
}

/*
 * Initialization
 */
void
dtrace_isa_init(void)
{
	lck_mtx_init(&dt_xc_lock, dtrace_lck_grp, dtrace_lck_attr);
	return;
}


/**
 * Register definitions
 */
#define ARM64_FP 29
#define ARM64_LR 30
#define ARM64_SP 31
#define ARM64_PC 32
#define ARM64_CPSR 33

/*
 * Runtime and ABI
 */
uint64_t
dtrace_getreg(struct regs * savearea, uint_t reg)
{
	struct arm_saved_state *regs = (struct arm_saved_state *) savearea;

	if (regs == NULL) {
		DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
		return 0;
	}

	if (!check_saved_state_reglimit(regs, reg)) {
		DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
		return 0;
	}

	return (uint64_t)get_saved_state_reg(regs, reg);
}

#define RETURN_OFFSET64 8

static int
dtrace_getustack_common(uint64_t * pcstack, int pcstack_limit, user_addr_t pc,
    user_addr_t sp)
{
	int ret = 0;

	ASSERT(pcstack == NULL || pcstack_limit > 0);

	while (pc != 0) {
		ret++;
		if (pcstack != NULL) {
			*pcstack++ = (uint64_t) pc;
			pcstack_limit--;
			if (pcstack_limit <= 0) {
				break;
			}
		}

		if (sp == 0) {
			break;
		}

		pc = dtrace_fuword64((sp + RETURN_OFFSET64));
		sp = dtrace_fuword64(sp);
	}

	return ret;
}

void
dtrace_getupcstack(uint64_t * pcstack, int pcstack_limit)
{
	thread_t        thread = current_thread();
	savearea_t     *regs;
	user_addr_t     pc, sp, fp;
	volatile uint16_t *flags = (volatile uint16_t *) &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
	int n;

	if (*flags & CPU_DTRACE_FAULT) {
		return;
	}

	if (pcstack_limit <= 0) {
		return;
	}

	/*
	 * If there's no user context we still need to zero the stack.
	 */
	if (thread == NULL) {
		goto zero;
	}

	regs = (savearea_t *) find_user_regs(thread);
	if (regs == NULL) {
		goto zero;
	}

	*pcstack++ = (uint64_t)dtrace_proc_selfpid();
	pcstack_limit--;

	if (pcstack_limit <= 0) {
		return;
	}

	pc = get_saved_state_pc(regs);
	sp = get_saved_state_sp(regs);
	fp = get_saved_state_fp(regs);

	if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
		*pcstack++ = (uint64_t) pc;
		pcstack_limit--;
		if (pcstack_limit <= 0) {
			return;
		}

		pc = get_saved_state_lr(regs);
	}

	n = dtrace_getustack_common(pcstack, pcstack_limit, pc, fp);

	ASSERT(n >= 0);
	ASSERT(n <= pcstack_limit);

	pcstack += n;
	pcstack_limit -= n;

zero:
	while (pcstack_limit-- > 0) {
		*pcstack++ = 0ULL;
	}
}

int
dtrace_getustackdepth(void)
{
	thread_t        thread = current_thread();
	savearea_t     *regs;
	user_addr_t     pc, sp, fp;
	int             n = 0;

	if (thread == NULL) {
		return 0;
	}

	if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT)) {
		return -1;
	}

	regs = (savearea_t *) find_user_regs(thread);
	if (regs == NULL) {
		return 0;
	}

	pc = get_saved_state_pc(regs);
	sp = get_saved_state_sp(regs);
	fp = get_saved_state_fp(regs);

	if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
		n++;
		pc = get_saved_state_lr(regs);
	}

	/*
	 * Note that unlike ppc, the arm code does not use
	 * CPU_DTRACE_USTACK_FP. This is because arm always
	 * traces from the sp, even in syscall/profile/fbt
	 * providers.
	 */

	n += dtrace_getustack_common(NULL, 0, pc, fp);

	return n;
}

void
dtrace_getufpstack(uint64_t * pcstack, uint64_t * fpstack, int pcstack_limit)
{
	thread_t        thread = current_thread();
	boolean_t       is64bit = proc_is64bit_data(current_proc());
	savearea_t      *regs;
	user_addr_t     pc, sp;
	volatile        uint16_t  *flags = (volatile uint16_t *) &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;


	if (*flags & CPU_DTRACE_FAULT) {
		return;
	}

	if (pcstack_limit <= 0) {
		return;
	}

	/*
	 * If there's no user context we still need to zero the stack.
	 */
	if (thread == NULL) {
		goto zero;
	}

	regs = (savearea_t *) find_user_regs(thread);
	if (regs == NULL) {
		goto zero;
	}

	*pcstack++ = (uint64_t)dtrace_proc_selfpid();
	pcstack_limit--;

	if (pcstack_limit <= 0) {
		return;
	}

	pc = get_saved_state_pc(regs);
	sp = get_saved_state_lr(regs);

#if 0                           /* XXX signal stack crawl */
	oldcontext = lwp->lwp_oldcontext;

	if (p->p_model == DATAMODEL_NATIVE) {
		s1 = sizeof(struct frame) + 2 * sizeof(long);
		s2 = s1 + sizeof(siginfo_t);
	} else {
		s1 = sizeof(struct frame32) + 3 * sizeof(int);
		s2 = s1 + sizeof(siginfo32_t);
	}
#endif

	if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
		*pcstack++ = (uint64_t) pc;
		*fpstack++ = 0;
		pcstack_limit--;
		if (pcstack_limit <= 0) {
			return;
		}

		if (is64bit) {
			pc = dtrace_fuword64(sp);
		} else {
			pc = dtrace_fuword32(sp);
		}
	}
	while (pc != 0 && sp != 0) {
		*pcstack++ = (uint64_t) pc;
		*fpstack++ = sp;
		pcstack_limit--;
		if (pcstack_limit <= 0) {
			break;
		}

#if 0                           /* XXX signal stack crawl */
		if (oldcontext == sp + s1 || oldcontext == sp + s2) {
			if (p->p_model == DATAMODEL_NATIVE) {
				ucontext_t     *ucp = (ucontext_t *) oldcontext;
				greg_t         *gregs = ucp->uc_mcontext.gregs;

				sp = dtrace_fulword(&gregs[REG_FP]);
				pc = dtrace_fulword(&gregs[REG_PC]);

				oldcontext = dtrace_fulword(&ucp->uc_link);
			} else {
				ucontext_t     *ucp = (ucontext_t *) oldcontext;
				greg_t         *gregs = ucp->uc_mcontext.gregs;

				sp = dtrace_fuword32(&gregs[EBP]);
				pc = dtrace_fuword32(&gregs[EIP]);

				oldcontext = dtrace_fuword32(&ucp->uc_link);
			}
		} else
#endif
		{
			pc = dtrace_fuword64((sp + RETURN_OFFSET64));
			sp = dtrace_fuword64(sp);
		}

#if 0
		/* XXX ARMTODO*/
		/*
		 * This is totally bogus:  if we faulted, we're going to clear
		 * the fault and break.  This is to deal with the apparently
		 * broken Java stacks on x86.
		 */
		if (*flags & CPU_DTRACE_FAULT) {
			*flags &= ~CPU_DTRACE_FAULT;
			break;
		}
#endif
	}

zero:
	while (pcstack_limit-- > 0) {
		*pcstack++ = 0ULL;
	}
}

#if XNU_MONITOR
static inline boolean_t
dtrace_frame_in_ppl_stack(struct frame * fp)
{
	return ((void *)fp >= pmap_stacks_start) &&
	       ((void *)fp < pmap_stacks_end);
}
#endif

void
dtrace_getpcstack(pc_t * pcstack, int pcstack_limit, int aframes,
    uint32_t * intrpc)
{
	struct frame   *fp = (struct frame *) __builtin_frame_address(0);
	struct frame   *nextfp, *minfp, *stacktop;
	int             depth = 0;
	int             on_intr;
#if XNU_MONITOR
	int             on_ppl_stack;
#endif
	int             last = 0;
	uintptr_t       pc;
	uintptr_t       caller = CPU->cpu_dtrace_caller;

	if ((on_intr = CPU_ON_INTR(CPU)) != 0) {
		stacktop = (struct frame *) dtrace_get_cpu_int_stack_top();
	}
#if XNU_MONITOR
	else if ((on_ppl_stack = dtrace_frame_in_ppl_stack(fp))) {
		stacktop = (struct frame *) pmap_stacks_end;
	}
#endif
	else {
		stacktop = (struct frame *) (dtrace_get_kernel_stack(current_thread()) + kernel_stack_size);
	}

	minfp = fp;

	aframes++;

	if (intrpc != NULL && depth < pcstack_limit) {
		pcstack[depth++] = (pc_t) intrpc;
	}

	while (depth < pcstack_limit) {
		nextfp = *(struct frame **) fp;
		pc = *(uintptr_t *) (((uintptr_t) fp) + RETURN_OFFSET64);

		if (nextfp <= minfp || nextfp >= stacktop) {
			if (on_intr) {
				/*
				 * Hop from interrupt stack to thread stack.
				 */
				arm_saved_state_t *arm_kern_regs = (arm_saved_state_t *) find_kern_regs(current_thread());
				if (arm_kern_regs) {
					nextfp = (struct frame *)(saved_state64(arm_kern_regs)->fp);

#if XNU_MONITOR
					on_ppl_stack = dtrace_frame_in_ppl_stack(nextfp);

					if (on_ppl_stack) {
						minfp = pmap_stacks_start;
						stacktop = pmap_stacks_end;
					} else
#endif
					{
						vm_offset_t kstack_base = dtrace_get_kernel_stack(current_thread());

						minfp = (struct frame *)kstack_base;
						stacktop = (struct frame *)(kstack_base + kernel_stack_size);
					}

					on_intr = 0;

					if (nextfp <= minfp || nextfp >= stacktop) {
						last = 1;
					}
				} else {
					/*
					 * If this thread was on the interrupt stack, but did not
					 * take an interrupt (i.e, the idle thread), there is no
					 * explicit saved state for us to use.
					 */
					last = 1;
				}
			} else {
#if XNU_MONITOR
				if ((!on_ppl_stack) && dtrace_frame_in_ppl_stack(nextfp)) {
					/*
					 * We are switching from the kernel stack
					 * to the PPL stack.
					 */
					on_ppl_stack = 1;
					minfp = pmap_stacks_start;
					stacktop = pmap_stacks_end;
				} else if (on_ppl_stack) {
					/*
					 * We could be going from the PPL stack
					 * to the kernel stack.
					 */
					vm_offset_t kstack_base = dtrace_get_kernel_stack(current_thread());

					minfp = (struct frame *)kstack_base;
					stacktop = (struct frame *)(kstack_base + kernel_stack_size);

					if (nextfp <= minfp || nextfp >= stacktop) {
						last = 1;
					}
				} else
#endif
				{
					/*
					 * This is the last frame we can process; indicate
					 * that we should return after processing this frame.
					 */
					last = 1;
				}
			}
		}
		if (aframes > 0) {
			if (--aframes == 0 && caller != (uintptr_t)NULL) {
				/*
				 * We've just run out of artificial frames,
				 * and we have a valid caller -- fill it in
				 * now.
				 */
				ASSERT(depth < pcstack_limit);
				pcstack[depth++] = (pc_t) caller;
				caller = (uintptr_t)NULL;
			}
		} else {
			if (depth < pcstack_limit) {
				pcstack[depth++] = (pc_t) pc;
			}
		}

		if (last) {
			while (depth < pcstack_limit) {
				pcstack[depth++] = (pc_t) NULL;
			}
			return;
		}
		fp = nextfp;
		minfp = fp;
	}
}

uint64_t
dtrace_getarg(int arg, int aframes, dtrace_mstate_t *mstate, dtrace_vstate_t *vstate)
{
#pragma unused(arg, aframes)
	uint64_t val = 0;
	struct frame *fp = (struct frame *)__builtin_frame_address(0);
	uintptr_t *stack;
	uintptr_t pc;
	int i;

	/*
	 * A total of 8 arguments are passed via registers; any argument with
	 * index of 7 or lower is therefore in a register.
	 */
	int inreg = 7;

	for (i = 1; i <= aframes; ++i) {
		fp = fp->backchain;
#if __has_feature(ptrauth_returns)
		pc = (uintptr_t)ptrauth_strip((void*)fp->retaddr, ptrauth_key_return_address);
#else
		pc = fp->retaddr;
#endif

		if (dtrace_invop_callsite_pre != NULL
		    && pc > (uintptr_t) dtrace_invop_callsite_pre
		    && pc <= (uintptr_t) dtrace_invop_callsite_post) {
			/* fp points to frame of dtrace_invop() activation */
			fp = fp->backchain; /* to fbt_perfCallback activation */
			fp = fp->backchain; /* to sleh_synchronous activation */
			fp = fp->backchain; /* to fleh_synchronous activation */

			arm_saved_state_t       *tagged_regs = (arm_saved_state_t*) ((void*) &fp[1]);
			arm_saved_state64_t     *saved_state = saved_state64(tagged_regs);

			if (arg <= inreg) {
				/* the argument will be found in a register */
				stack = (uintptr_t*) &saved_state->x[0];
			} else {
				/* the argument will be found in the stack */
				fp = (struct frame*) saved_state->sp;
				stack = (uintptr_t*) &fp[1];
				arg -= (inreg + 1);
			}

			goto load;
		}
	}

	/*
	 * We know that we did not come through a trap to get into
	 * dtrace_probe() --  We arrive here when the provider has
	 * called dtrace_probe() directly.
	 * The probe ID is the first argument to dtrace_probe().
	 * We must advance beyond that to get the argX.
	 */
	arg++; /* Advance past probeID */

	if (arg <= inreg) {
		/*
		 * This shouldn't happen.  If the argument is passed in a
		 * register then it should have been, well, passed in a
		 * register...
		 */
		DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
		return 0;
	}

	arg -= (inreg + 1);
	stack = (uintptr_t*) &fp[1]; /* Find marshalled arguments */

load:
	if (dtrace_canload((uint64_t)(stack + arg), sizeof(uint64_t),
	    mstate, vstate)) {
		/* dtrace_probe arguments arg0 ... arg4 are 64bits wide */
		val = dtrace_load64((uint64_t)(stack + arg));
	}

	return val;
}

void
dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which,
    int fltoffs, int fault, uint64_t illval)
{
	/* XXX ARMTODO */
	/*
	 * For the case of the error probe firing lets
	 * stash away "illval" here, and special-case retrieving it in DIF_VARIABLE_ARG.
	 */
	state->dts_arg_error_illval = illval;
	dtrace_probe( dtrace_probeid_error, (uint64_t)(uintptr_t)state, epid, which, fltoffs, fault );
}

void
dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit))
{
	/* XXX ARMTODO check copied from ppc/x86*/
	/*
	 * "base" is the smallest toxic address in the range, "limit" is the first
	 * VALID address greater than "base".
	 */
	func(0x0, VM_MIN_KERNEL_ADDRESS);
	if (VM_MAX_KERNEL_ADDRESS < ~(uintptr_t)0) {
		func(VM_MAX_KERNEL_ADDRESS + 1, ~(uintptr_t)0);
	}
}

void
dtrace_flush_caches(void)
{
	/* TODO There were some problems with flushing just the cache line that had been modified.
	 * For now, we'll flush the entire cache, until we figure out how to flush just the patched block.
	 */
	FlushPoU_Dcache();
	InvalidatePoU_Icache();
}
Commit	Line	Data
5ba3f43e	1	/*
cb323159	2	* Copyright (c) 2005-2018 Apple Computer, Inc. All rights reserved.
5ba3f43e A	3	*
5ba3f43e A	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
0a7de745	5	*
5ba3f43e A	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
0a7de745	14	*
5ba3f43e A	15	* Please obtain a copy of the License at
5ba3f43e A	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
0a7de745	17	*
5ba3f43e A	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
0a7de745	25	*
5ba3f43e A	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
d9a64523	29	#include <arm/caches_internal.h>
5ba3f43e	30	#include <kern/thread.h>
5ba3f43e	31
d9a64523 A	32	#if __has_include(<ptrauth.h>)
	33	#include <ptrauth.h>
	34	#endif
5ba3f43e	35	#include <stdarg.h>
5ba3f43e A	36	#include <sys/time.h>
	37	#include <sys/systm.h>
	38	#include <sys/proc.h>
	39	#include <sys/proc_internal.h>
	40	#include <sys/kauth.h>
	41	#include <sys/dtrace.h>
	42	#include <sys/dtrace_impl.h>
cb323159	43	#include <machine/atomic.h>
5ba3f43e	44	#include <kern/simple_lock.h>
0a7de745	45	#include <kern/sched_prim.h> /* for thread_wakeup() */
5ba3f43e A	46	#include <kern/thread_call.h>
5ba3f43e A	47	#include <kern/task.h>
5ba3f43e A	48
	49	extern struct arm_saved_state *find_kern_regs(thread_t);
	50
	51	extern dtrace_id_t dtrace_probeid_error; /* special ERROR probe */
	52	typedef arm_saved_state_t savearea_t;
	53
0a7de745 A	54	extern lck_attr_t *dtrace_lck_attr;
0a7de745 A	55	extern lck_grp_t *dtrace_lck_grp;
5ba3f43e	56
c6bf4f31 A	57	#if XNU_MONITOR
	58	extern void * pmap_stacks_start;
	59	extern void * pmap_stacks_end;
	60	#endif
5ba3f43e A	61
	62	struct frame {
	63	struct frame *backchain;
	64	uintptr_t retaddr;
	65	};
	66
	67	/*
	68	* Atomicity and synchronization
	69	*/
	70	inline void
	71	dtrace_membar_producer(void)
	72	{
	73	#if __ARM_SMP__
0a7de745	74	__asm__ volatile ("dmb ish" : : : "memory");
5ba3f43e	75	#else
0a7de745	76	__asm__ volatile ("nop" : : : "memory");
5ba3f43e A	77	#endif
	78	}
	79
	80	inline void
	81	dtrace_membar_consumer(void)
	82	{
	83	#if __ARM_SMP__
0a7de745	84	__asm__ volatile ("dmb ish" : : : "memory");
5ba3f43e	85	#else
0a7de745	86	__asm__ volatile ("nop" : : : "memory");
5ba3f43e A	87	#endif
	88	}
	89
	90	/*
	91	* Interrupt manipulation
	92	* XXX dtrace_getipl() can be called from probe context.
	93	*/
	94	int
	95	dtrace_getipl(void)
	96	{
	97	/*
	98	* XXX Drat, get_interrupt_level is MACH_KERNEL_PRIVATE
	99	* in osfmk/kern/cpu_data.h
	100	*/
	101	/* return get_interrupt_level(); */
0a7de745	102	return ml_at_interrupt_context() ? 1 : 0;
5ba3f43e A	103	}
	104
	105	#if __ARM_SMP__
	106	/*
	107	* MP coordination
	108	*/
	109
	110	decl_lck_mtx_data(static, dt_xc_lock);
	111	static uint32_t dt_xc_sync;
	112
	113	typedef struct xcArg {
	114	processorid_t cpu;
	115	dtrace_xcall_t f;
	116	void *arg;
	117	} xcArg_t;
	118
	119	static void
	120	xcRemote(void *foo)
	121	{
	122	xcArg_t pArg = (xcArg_t ) foo;
	123
0a7de745 A	124	if (pArg->cpu == CPU->cpu_id \|\| pArg->cpu == DTRACE_CPUALL) {
	125	(pArg->f)(pArg->arg);
	126	}
5ba3f43e	127
cb323159	128	if (os_atomic_dec(&dt_xc_sync, relaxed) == 0) {
5ba3f43e	129	thread_wakeup((event_t) &dt_xc_sync);
0a7de745	130	}
5ba3f43e A	131	}
	132	#endif
	133
	134	/*
	135	* dtrace_xcall() is not called from probe context.
	136	*/
	137	void
	138	dtrace_xcall(processorid_t cpu, dtrace_xcall_t f, void *arg)
	139	{
	140	#if __ARM_SMP__
	141	/* Only one dtrace_xcall in flight allowed */
	142	lck_mtx_lock(&dt_xc_lock);
	143
	144	xcArg_t xcArg;
	145
	146	xcArg.cpu = cpu;
	147	xcArg.f = f;
	148	xcArg.arg = arg;
	149
	150	cpu_broadcast_xcall(&dt_xc_sync, TRUE, xcRemote, (void*) &xcArg);
	151
	152	lck_mtx_unlock(&dt_xc_lock);
	153	return;
	154	#else
	155	#pragma unused(cpu)
	156	/* On uniprocessor systems, the cpu should always be either ourselves or all */
	157	ASSERT(cpu == CPU->cpu_id \|\| cpu == DTRACE_CPUALL);
	158
	159	(*f)(arg);
	160	return;
	161	#endif
	162	}
	163
	164	/*
	165	* Initialization
	166	*/
	167	void
	168	dtrace_isa_init(void)
	169	{
	170	lck_mtx_init(&dt_xc_lock, dtrace_lck_grp, dtrace_lck_attr);
	171	return;
	172	}
	173
	174
	175	/**
	176	* Register definitions
	177	*/
5ba3f43e A	178	#define ARM64_FP 29
	179	#define ARM64_LR 30
	180	#define ARM64_SP 31
	181	#define ARM64_PC 32
	182	#define ARM64_CPSR 33
	183
	184	/*
	185	* Runtime and ABI
	186	*/
	187	uint64_t
	188	dtrace_getreg(struct regs * savearea, uint_t reg)
	189	{
	190	struct arm_saved_state regs = (struct arm_saved_state ) savearea;
	191
d9a64523 A	192	if (regs == NULL) {
d9a64523 A	193	DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
0a7de745	194	return 0;
d9a64523 A	195	}
d9a64523 A	196
5ba3f43e A	197	if (!check_saved_state_reglimit(regs, reg)) {
5ba3f43e A	198	DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
0a7de745	199	return 0;
5ba3f43e A	200	}
5ba3f43e A	201
0a7de745	202	return (uint64_t)get_saved_state_reg(regs, reg);
5ba3f43e A	203	}
5ba3f43e A	204
5ba3f43e A	205	#define RETURN_OFFSET64 8
	206
	207	static int
	208	dtrace_getustack_common(uint64_t * pcstack, int pcstack_limit, user_addr_t pc,
0a7de745	209	user_addr_t sp)
5ba3f43e A	210	{
5ba3f43e A	211	int ret = 0;
0a7de745	212
5ba3f43e A	213	ASSERT(pcstack == NULL \|\| pcstack_limit > 0);
	214
	215	while (pc != 0) {
	216	ret++;
	217	if (pcstack != NULL) {
	218	*pcstack++ = (uint64_t) pc;
	219	pcstack_limit--;
0a7de745	220	if (pcstack_limit <= 0) {
5ba3f43e	221	break;
0a7de745	222	}
5ba3f43e A	223	}
5ba3f43e A	224
0a7de745	225	if (sp == 0) {
5ba3f43e	226	break;
0a7de745	227	}
5ba3f43e	228
cb323159 A	229	pc = dtrace_fuword64((sp + RETURN_OFFSET64));
cb323159 A	230	sp = dtrace_fuword64(sp);
5ba3f43e A	231	}
5ba3f43e A	232
0a7de745	233	return ret;
5ba3f43e A	234	}
	235
	236	void
	237	dtrace_getupcstack(uint64_t * pcstack, int pcstack_limit)
	238	{
	239	thread_t thread = current_thread();
	240	savearea_t *regs;
	241	user_addr_t pc, sp, fp;
0a7de745	242	volatile uint16_t flags = (volatile uint16_t ) &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
5ba3f43e A	243	int n;
5ba3f43e A	244
0a7de745	245	if (*flags & CPU_DTRACE_FAULT) {
5ba3f43e	246	return;
0a7de745	247	}
5ba3f43e	248
0a7de745	249	if (pcstack_limit <= 0) {
5ba3f43e	250	return;
0a7de745	251	}
5ba3f43e A	252
	253	/*
	254	* If there's no user context we still need to zero the stack.
	255	*/
0a7de745	256	if (thread == NULL) {
5ba3f43e	257	goto zero;
0a7de745	258	}
5ba3f43e A	259
5ba3f43e A	260	regs = (savearea_t *) find_user_regs(thread);
0a7de745	261	if (regs == NULL) {
5ba3f43e	262	goto zero;
0a7de745	263	}
5ba3f43e A	264
	265	*pcstack++ = (uint64_t)dtrace_proc_selfpid();
	266	pcstack_limit--;
	267
0a7de745	268	if (pcstack_limit <= 0) {
5ba3f43e	269	return;
0a7de745	270	}
5ba3f43e A	271
	272	pc = get_saved_state_pc(regs);
	273	sp = get_saved_state_sp(regs);
0a7de745	274	fp = get_saved_state_fp(regs);
5ba3f43e A	275
	276	if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
	277	*pcstack++ = (uint64_t) pc;
	278	pcstack_limit--;
0a7de745	279	if (pcstack_limit <= 0) {
5ba3f43e	280	return;
0a7de745	281	}
5ba3f43e A	282
	283	pc = get_saved_state_lr(regs);
	284	}
	285
	286	n = dtrace_getustack_common(pcstack, pcstack_limit, pc, fp);
	287
	288	ASSERT(n >= 0);
	289	ASSERT(n <= pcstack_limit);
	290
	291	pcstack += n;
	292	pcstack_limit -= n;
	293
	294	zero:
0a7de745	295	while (pcstack_limit-- > 0) {
5ba3f43e	296	*pcstack++ = 0ULL;
0a7de745	297	}
5ba3f43e A	298	}
	299
	300	int
	301	dtrace_getustackdepth(void)
	302	{
	303	thread_t thread = current_thread();
	304	savearea_t *regs;
	305	user_addr_t pc, sp, fp;
	306	int n = 0;
	307
0a7de745	308	if (thread == NULL) {
5ba3f43e	309	return 0;
0a7de745	310	}
5ba3f43e	311
0a7de745 A	312	if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT)) {
	313	return -1;
	314	}
5ba3f43e A	315
5ba3f43e A	316	regs = (savearea_t *) find_user_regs(thread);
0a7de745	317	if (regs == NULL) {
5ba3f43e	318	return 0;
0a7de745 A	319	}
0a7de745 A	320
5ba3f43e A	321	pc = get_saved_state_pc(regs);
	322	sp = get_saved_state_sp(regs);
	323	fp = get_saved_state_fp(regs);
	324
	325	if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
	326	n++;
	327	pc = get_saved_state_lr(regs);
	328	}
	329
	330	/*
	331	* Note that unlike ppc, the arm code does not use
	332	* CPU_DTRACE_USTACK_FP. This is because arm always
	333	* traces from the sp, even in syscall/profile/fbt
	334	* providers.
	335	*/
0a7de745	336
5ba3f43e A	337	n += dtrace_getustack_common(NULL, 0, pc, fp);
5ba3f43e A	338
0a7de745	339	return n;
5ba3f43e A	340	}
	341
	342	void
	343	dtrace_getufpstack(uint64_t * pcstack, uint64_t * fpstack, int pcstack_limit)
	344	{
	345	thread_t thread = current_thread();
d9a64523	346	boolean_t is64bit = proc_is64bit_data(current_proc());
5ba3f43e A	347	savearea_t *regs;
5ba3f43e A	348	user_addr_t pc, sp;
0a7de745	349	volatile uint16_t flags = (volatile uint16_t ) &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
5ba3f43e	350
5ba3f43e	351
0a7de745	352	if (*flags & CPU_DTRACE_FAULT) {
5ba3f43e	353	return;
0a7de745	354	}
5ba3f43e	355
0a7de745	356	if (pcstack_limit <= 0) {
5ba3f43e	357	return;
0a7de745	358	}
5ba3f43e	359
0a7de745	360	/*
5ba3f43e A	361	* If there's no user context we still need to zero the stack.
5ba3f43e A	362	*/
0a7de745	363	if (thread == NULL) {
5ba3f43e	364	goto zero;
0a7de745 A	365	}
0a7de745 A	366
5ba3f43e	367	regs = (savearea_t *) find_user_regs(thread);
0a7de745	368	if (regs == NULL) {
5ba3f43e	369	goto zero;
0a7de745	370	}
5ba3f43e A	371
	372	*pcstack++ = (uint64_t)dtrace_proc_selfpid();
	373	pcstack_limit--;
	374
0a7de745	375	if (pcstack_limit <= 0) {
5ba3f43e	376	return;
0a7de745	377	}
5ba3f43e A	378
	379	pc = get_saved_state_pc(regs);
	380	sp = get_saved_state_lr(regs);
	381
0a7de745	382	#if 0 /* XXX signal stack crawl */
5ba3f43e A	383	oldcontext = lwp->lwp_oldcontext;
	384
	385	if (p->p_model == DATAMODEL_NATIVE) {
	386	s1 = sizeof(struct frame) + 2 * sizeof(long);
	387	s2 = s1 + sizeof(siginfo_t);
	388	} else {
	389	s1 = sizeof(struct frame32) + 3 * sizeof(int);
	390	s2 = s1 + sizeof(siginfo32_t);
	391	}
	392	#endif
	393
	394	if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
	395	*pcstack++ = (uint64_t) pc;
	396	*fpstack++ = 0;
	397	pcstack_limit--;
0a7de745	398	if (pcstack_limit <= 0) {
5ba3f43e	399	return;
0a7de745	400	}
5ba3f43e	401
0a7de745	402	if (is64bit) {
5ba3f43e	403	pc = dtrace_fuword64(sp);
0a7de745	404	} else {
5ba3f43e	405	pc = dtrace_fuword32(sp);
0a7de745	406	}
5ba3f43e A	407	}
	408	while (pc != 0 && sp != 0) {
	409	*pcstack++ = (uint64_t) pc;
	410	*fpstack++ = sp;
	411	pcstack_limit--;
0a7de745	412	if (pcstack_limit <= 0) {
5ba3f43e	413	break;
0a7de745	414	}
5ba3f43e	415
0a7de745	416	#if 0 /* XXX signal stack crawl */
5ba3f43e A	417	if (oldcontext == sp + s1 \|\| oldcontext == sp + s2) {
	418	if (p->p_model == DATAMODEL_NATIVE) {
	419	ucontext_t ucp = (ucontext_t ) oldcontext;
	420	greg_t *gregs = ucp->uc_mcontext.gregs;
	421
	422	sp = dtrace_fulword(&gregs[REG_FP]);
	423	pc = dtrace_fulword(&gregs[REG_PC]);
	424
	425	oldcontext = dtrace_fulword(&ucp->uc_link);
	426	} else {
	427	ucontext_t ucp = (ucontext_t ) oldcontext;
	428	greg_t *gregs = ucp->uc_mcontext.gregs;
	429
	430	sp = dtrace_fuword32(&gregs[EBP]);
	431	pc = dtrace_fuword32(&gregs[EIP]);
	432
	433	oldcontext = dtrace_fuword32(&ucp->uc_link);
	434	}
	435	} else
	436	#endif
	437	{
cb323159 A	438	pc = dtrace_fuword64((sp + RETURN_OFFSET64));
cb323159 A	439	sp = dtrace_fuword64(sp);
5ba3f43e A	440	}
	441
	442	#if 0
	443	/* XXX ARMTODO*/
	444	/*
	445	* This is totally bogus: if we faulted, we're going to clear
	446	* the fault and break. This is to deal with the apparently
	447	* broken Java stacks on x86.
	448	*/
	449	if (*flags & CPU_DTRACE_FAULT) {
	450	*flags &= ~CPU_DTRACE_FAULT;
	451	break;
	452	}
	453	#endif
	454	}
	455
0a7de745 A	456	zero:
0a7de745 A	457	while (pcstack_limit-- > 0) {
5ba3f43e	458	*pcstack++ = 0ULL;
0a7de745	459	}
5ba3f43e A	460	}
5ba3f43e A	461
c6bf4f31 A	462	#if XNU_MONITOR
	463	static inline boolean_t
	464	dtrace_frame_in_ppl_stack(struct frame * fp)
	465	{
	466	return ((void *)fp >= pmap_stacks_start) &&
	467	((void *)fp < pmap_stacks_end);
	468	}
	469	#endif
5ba3f43e A	470
	471	void
	472	dtrace_getpcstack(pc_t * pcstack, int pcstack_limit, int aframes,
0a7de745	473	uint32_t * intrpc)
5ba3f43e A	474	{
	475	struct frame fp = (struct frame ) __builtin_frame_address(0);
	476	struct frame nextfp, minfp, *stacktop;
	477	int depth = 0;
	478	int on_intr;
c6bf4f31 A	479	#if XNU_MONITOR
	480	int on_ppl_stack;
	481	#endif
5ba3f43e A	482	int last = 0;
	483	uintptr_t pc;
	484	uintptr_t caller = CPU->cpu_dtrace_caller;
	485
0a7de745	486	if ((on_intr = CPU_ON_INTR(CPU)) != 0) {
5ba3f43e	487	stacktop = (struct frame *) dtrace_get_cpu_int_stack_top();
0a7de745	488	}
c6bf4f31 A	489	#if XNU_MONITOR
	490	else if ((on_ppl_stack = dtrace_frame_in_ppl_stack(fp))) {
	491	stacktop = (struct frame *) pmap_stacks_end;
	492	}
	493	#endif
0a7de745	494	else {
5ba3f43e	495	stacktop = (struct frame *) (dtrace_get_kernel_stack(current_thread()) + kernel_stack_size);
0a7de745	496	}
5ba3f43e A	497
	498	minfp = fp;
	499
	500	aframes++;
	501
0a7de745	502	if (intrpc != NULL && depth < pcstack_limit) {
5ba3f43e	503	pcstack[depth++] = (pc_t) intrpc;
0a7de745	504	}
5ba3f43e A	505
	506	while (depth < pcstack_limit) {
	507	nextfp = (struct frame *) fp;
	508	pc = (uintptr_t ) (((uintptr_t) fp) + RETURN_OFFSET64);
	509
	510	if (nextfp <= minfp \|\| nextfp >= stacktop) {
	511	if (on_intr) {
	512	/*
	513	* Hop from interrupt stack to thread stack.
	514	*/
	515	arm_saved_state_t arm_kern_regs = (arm_saved_state_t ) find_kern_regs(current_thread());
	516	if (arm_kern_regs) {
	517	nextfp = (struct frame *)(saved_state64(arm_kern_regs)->fp);
	518
c6bf4f31 A	519	#if XNU_MONITOR
	520	on_ppl_stack = dtrace_frame_in_ppl_stack(nextfp);
	521
	522	if (on_ppl_stack) {
	523	minfp = pmap_stacks_start;
	524	stacktop = pmap_stacks_end;
	525	} else
	526	#endif
5ba3f43e A	527	{
	528	vm_offset_t kstack_base = dtrace_get_kernel_stack(current_thread());
	529
	530	minfp = (struct frame *)kstack_base;
	531	stacktop = (struct frame *)(kstack_base + kernel_stack_size);
	532	}
	533
	534	on_intr = 0;
	535
	536	if (nextfp <= minfp \|\| nextfp >= stacktop) {
	537	last = 1;
	538	}
	539	} else {
	540	/*
	541	* If this thread was on the interrupt stack, but did not
	542	* take an interrupt (i.e, the idle thread), there is no
	543	* explicit saved state for us to use.
	544	*/
	545	last = 1;
	546	}
	547	} else {
c6bf4f31 A	548	#if XNU_MONITOR
	549	if ((!on_ppl_stack) && dtrace_frame_in_ppl_stack(nextfp)) {
	550	/*
	551	* We are switching from the kernel stack
	552	* to the PPL stack.
	553	*/
	554	on_ppl_stack = 1;
	555	minfp = pmap_stacks_start;
	556	stacktop = pmap_stacks_end;
	557	} else if (on_ppl_stack) {
	558	/*
	559	* We could be going from the PPL stack
	560	* to the kernel stack.
	561	*/
	562	vm_offset_t kstack_base = dtrace_get_kernel_stack(current_thread());
	563
	564	minfp = (struct frame *)kstack_base;
	565	stacktop = (struct frame *)(kstack_base + kernel_stack_size);
	566
	567	if (nextfp <= minfp \|\| nextfp >= stacktop) {
	568	last = 1;
	569	}
	570	} else
	571	#endif
5ba3f43e A	572	{
	573	/*
	574	* This is the last frame we can process; indicate
	575	* that we should return after processing this frame.
	576	*/
	577	last = 1;
	578	}
	579	}
	580	}
	581	if (aframes > 0) {
	582	if (--aframes == 0 && caller != (uintptr_t)NULL) {
	583	/*
	584	* We've just run out of artificial frames,
	585	* and we have a valid caller -- fill it in
	586	* now.
	587	*/
	588	ASSERT(depth < pcstack_limit);
	589	pcstack[depth++] = (pc_t) caller;
	590	caller = (uintptr_t)NULL;
	591	}
	592	} else {
0a7de745	593	if (depth < pcstack_limit) {
5ba3f43e	594	pcstack[depth++] = (pc_t) pc;
0a7de745	595	}
5ba3f43e A	596	}
	597
	598	if (last) {
0a7de745	599	while (depth < pcstack_limit) {
5ba3f43e	600	pcstack[depth++] = (pc_t) NULL;
0a7de745	601	}
5ba3f43e A	602	return;
	603	}
	604	fp = nextfp;
	605	minfp = fp;
	606	}
	607	}
	608
5ba3f43e A	609	uint64_t
	610	dtrace_getarg(int arg, int aframes, dtrace_mstate_t mstate, dtrace_vstate_t vstate)
	611	{
	612	#pragma unused(arg, aframes)
	613	uint64_t val = 0;
	614	struct frame fp = (struct frame )__builtin_frame_address(0);
	615	uintptr_t *stack;
	616	uintptr_t pc;
	617	int i;
	618
	619	/*
	620	* A total of 8 arguments are passed via registers; any argument with
	621	* index of 7 or lower is therefore in a register.
	622	*/
	623	int inreg = 7;
	624
	625	for (i = 1; i <= aframes; ++i) {
	626	fp = fp->backchain;
d9a64523 A	627	#if __has_feature(ptrauth_returns)
	628	pc = (uintptr_t)ptrauth_strip((void*)fp->retaddr, ptrauth_key_return_address);
	629	#else
5ba3f43e	630	pc = fp->retaddr;
d9a64523	631	#endif
5ba3f43e A	632
5ba3f43e A	633	if (dtrace_invop_callsite_pre != NULL
0a7de745 A	634	&& pc > (uintptr_t) dtrace_invop_callsite_pre
0a7de745 A	635	&& pc <= (uintptr_t) dtrace_invop_callsite_post) {
5ba3f43e A	636	/* fp points to frame of dtrace_invop() activation */
	637	fp = fp->backchain; /* to fbt_perfCallback activation */
	638	fp = fp->backchain; /* to sleh_synchronous activation */
	639	fp = fp->backchain; /* to fleh_synchronous activation */
	640
0a7de745 A	641	arm_saved_state_t tagged_regs = (arm_saved_state_t) ((void*) &fp[1]);
0a7de745 A	642	arm_saved_state64_t *saved_state = saved_state64(tagged_regs);
5ba3f43e A	643
	644	if (arg <= inreg) {
	645	/* the argument will be found in a register */
	646	stack = (uintptr_t*) &saved_state->x[0];
	647	} else {
	648	/* the argument will be found in the stack */
	649	fp = (struct frame*) saved_state->sp;
d9a64523	650	stack = (uintptr_t*) &fp[1];
5ba3f43e A	651	arg -= (inreg + 1);
	652	}
	653
	654	goto load;
	655	}
	656	}
	657
	658	/*
	659	* We know that we did not come through a trap to get into
	660	* dtrace_probe() -- We arrive here when the provider has
	661	* called dtrace_probe() directly.
	662	* The probe ID is the first argument to dtrace_probe().
	663	* We must advance beyond that to get the argX.
	664	*/
	665	arg++; /* Advance past probeID */
	666
	667	if (arg <= inreg) {
	668	/*
	669	* This shouldn't happen. If the argument is passed in a
	670	* register then it should have been, well, passed in a
	671	* register...
	672	*/
	673	DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
0a7de745	674	return 0;
5ba3f43e A	675	}
	676
	677	arg -= (inreg + 1);
	678	stack = (uintptr_t) &fp[1]; / Find marshalled arguments */
	679
	680	load:
	681	if (dtrace_canload((uint64_t)(stack + arg), sizeof(uint64_t),
0a7de745	682	mstate, vstate)) {
5ba3f43e A	683	/* dtrace_probe arguments arg0 ... arg4 are 64bits wide */
	684	val = dtrace_load64((uint64_t)(stack + arg));
	685	}
	686
0a7de745	687	return val;
5ba3f43e A	688	}
	689
	690	void
	691	dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which,
0a7de745	692	int fltoffs, int fault, uint64_t illval)
5ba3f43e A	693	{
	694	/* XXX ARMTODO */
	695	/*
	696	* For the case of the error probe firing lets
	697	* stash away "illval" here, and special-case retrieving it in DIF_VARIABLE_ARG.
	698	*/
	699	state->dts_arg_error_illval = illval;
	700	dtrace_probe( dtrace_probeid_error, (uint64_t)(uintptr_t)state, epid, which, fltoffs, fault );
	701	}
	702
	703	void
	704	dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit))
	705	{
	706	/* XXX ARMTODO check copied from ppc/x86*/
0a7de745	707	/*
5ba3f43e A	708	* "base" is the smallest toxic address in the range, "limit" is the first
5ba3f43e A	709	* VALID address greater than "base".
0a7de745	710	*/
5ba3f43e	711	func(0x0, VM_MIN_KERNEL_ADDRESS);
0a7de745 A	712	if (VM_MAX_KERNEL_ADDRESS < ~(uintptr_t)0) {
	713	func(VM_MAX_KERNEL_ADDRESS + 1, ~(uintptr_t)0);
	714	}
5ba3f43e A	715	}
5ba3f43e A	716
0a7de745 A	717	void
0a7de745 A	718	dtrace_flush_caches(void)
d9a64523 A	719	{
	720	/* TODO There were some problems with flushing just the cache line that had been modified.
	721	* For now, we'll flush the entire cache, until we figure out how to flush just the patched block.
	722	*/
	723	FlushPoU_Dcache();
	724	InvalidatePoU_Icache();
	725	}