[apple/xnu.git] / osfmk / kperf / action.c

/*
 * Copyright (c) 2011 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

/*
 * Called from a trigger. Actually takes the data from the different
 * modules and puts them in a buffer
 */

#include <mach/mach_types.h>
#include <machine/machine_routines.h>
// #include <libkern/libkern.h>
#include <kern/kalloc.h>
#include <kern/debug.h> /* panic */
#include <kern/thread.h>
#include <sys/errno.h>

#include <chud/chud_xnu.h>
#include <kperf/kperf.h>

#include <kperf/buffer.h>
#include <kperf/timetrigger.h>
#include <kperf/threadinfo.h>
#include <kperf/callstack.h>
#include <kperf/sample.h>
#include <kperf/action.h>
#include <kperf/context.h>
#include <kperf/ast.h>

#define ACTION_MAX 32

/* the list of different actions to take */
struct action
{
	uint32_t sample;
	uint32_t userdata;
	int pid_filter;
};

/* the list of actions */
static unsigned actionc = 0;
static struct action *actionv = NULL;

/* whether to record callstacks on kdebug events */
static int kdebug_callstack_action = 0;

/* whether we get a callback on a thread switch */
int  kperf_cswitch_hook = 0;

/* indirect hooks to play nice with CHUD for the transition to kperf */
kern_return_t chudxnu_kdebug_callback_enter(chudxnu_kdebug_callback_func_t fn);
kern_return_t chudxnu_kdebug_callback_cancel(void);

/* Do the real work! */
/* this can be called in any context ... right? */
static kern_return_t
kperf_sample_internal( struct kperf_sample *sbuf,
                       struct kperf_context *context,
                       unsigned sample_what, unsigned sample_flags,
                       unsigned actionid )
{
	boolean_t enabled;
	int did_ucallstack = 0, did_tinfo_extra = 0;
	uint32_t userdata;

	/* not much point continuing here, but what to do ? return
	 * Shutdown? cut a tracepoint and continue?
	 */
	if( sample_what == 0 )
		return SAMPLE_CONTINUE;

	int is_kernel = (context->cur_pid == 0);

	sbuf->kcallstack.nframes = 0;
	sbuf->kcallstack.flags = CALLSTACK_VALID;
	sbuf->ucallstack.nframes = 0;
	sbuf->ucallstack.flags = CALLSTACK_VALID;

	/*  an event occurred. Sample everything and dump it in a
	 *  buffer.
	 */

	/* collect data from samplers */
	if( sample_what & SAMPLER_TINFO ) {
		kperf_threadinfo_sample( &sbuf->threadinfo, context );
		
		/* See if we should drop idle thread samples */
		if( !(sample_flags & SAMPLE_FLAG_IDLE_THREADS) )
			if (sbuf->threadinfo.runmode & 0x40)
				return SAMPLE_CONTINUE;
	}

	if( (sample_what & SAMPLER_KSTACK) && !(sample_flags & SAMPLE_FLAG_EMPTY_CALLSTACK) )
		kperf_kcallstack_sample( &sbuf->kcallstack, context );

	/* sensitive ones */
	if ( !is_kernel ) {
		if( sample_flags & SAMPLE_FLAG_PEND_USER )
		{
			if( (sample_what & SAMPLER_USTACK) && !(sample_flags & SAMPLE_FLAG_EMPTY_CALLSTACK) )
				did_ucallstack = kperf_ucallstack_pend( context );

			if( sample_what & SAMPLER_TINFOEX )
				did_tinfo_extra = kperf_threadinfo_extra_pend( context );
		}
		else
		{
			if( (sample_what & SAMPLER_USTACK) && !(sample_flags & SAMPLE_FLAG_EMPTY_CALLSTACK) )
				kperf_ucallstack_sample( &sbuf->ucallstack, context );

			if( sample_what & SAMPLER_TINFOEX )
				kperf_threadinfo_extra_sample( &sbuf->tinfo_ex,
							       context );
		}
	}

#if KPC
	if ( sample_what & SAMPLER_PMC_CPU )
		kperf_kpc_cpu_sample( &sbuf->kpcdata, 
		                      (sample_what & SAMPLER_PMC_CPU) != 0 );
#endif

	/* lookup the user tag, if any */
	if( actionid 
	    && (actionid <= actionc) )
		userdata = actionv[actionid-1].userdata;
	else
		userdata = actionid;

	/* stash the data into the buffer
	 * interrupts off to ensure we don't get split
	 */
	enabled = ml_set_interrupts_enabled(FALSE);

	BUF_DATA( PERF_GEN_EVENT | DBG_FUNC_START, sample_what, 
	           actionid, userdata, sample_flags );

	/* dump threadinfo */
	if( sample_what & SAMPLER_TINFO )
		kperf_threadinfo_log( &sbuf->threadinfo );

	/* dump kcallstack */
	if( sample_what & SAMPLER_KSTACK )
		kperf_kcallstack_log( &sbuf->kcallstack );


	/* dump user stuff */
	if ( !is_kernel ) {
		if ( sample_flags & SAMPLE_FLAG_PEND_USER )
		{
			if ( did_ucallstack )
				BUF_INFO1( PERF_CS_UPEND, 0 );

			if ( did_tinfo_extra )
				BUF_INFO1( PERF_TI_XPEND, 0 );
		}
		else
		{
			if( sample_what & SAMPLER_USTACK )
				kperf_ucallstack_log( &sbuf->ucallstack );

			if( sample_what & SAMPLER_TINFOEX )
				kperf_threadinfo_extra_log( &sbuf->tinfo_ex );
		}
	}

#if KPC
	if ( sample_what & SAMPLER_PMC_CPU )
		kperf_kpc_cpu_log( &sbuf->kpcdata );
	
#endif

	BUF_DATA1( PERF_GEN_EVENT | DBG_FUNC_END, sample_what );

	/* intrs back on */
	ml_set_interrupts_enabled(enabled);

	return SAMPLE_CONTINUE;
}

/* Translate actionid into sample bits and take a sample */
kern_return_t
kperf_sample( struct kperf_sample *sbuf,
	      struct kperf_context *context,
              unsigned actionid, unsigned sample_flags )
{
	unsigned sample_what = 0;
	int pid_filter;

	/* work out what to sample, if anything */
	if( (actionid > actionc) || (actionid == 0) )
		return SAMPLE_SHUTDOWN;

	/* check the pid filter against the context's current pid.
	 * filter pid == -1 means any pid
	 */
	pid_filter = actionv[actionid-1].pid_filter;
	if( (pid_filter != -1)
	    && (pid_filter != context->cur_pid) )
		return SAMPLE_CONTINUE;

	/* the samplers to run */
	sample_what = actionv[actionid-1].sample;

	/* do the actual sample operation */
	return kperf_sample_internal( sbuf, context, sample_what, 
	                              sample_flags, actionid );
}

/* ast callback on a thread */
void
kperf_thread_ast_handler( thread_t thread )
{
	int r;
	uint32_t t_chud;
	unsigned sample_what = 0;
	/* we know we're on a thread, so let's do stuff */
	task_t task = NULL;

	BUF_INFO1(PERF_AST_HNDLR | DBG_FUNC_START, thread);

	/* use ~2kb of the stack for the sample, should be ok since we're in the ast */
	struct kperf_sample sbuf;
	bzero(&sbuf, sizeof(struct kperf_sample));

	/* make a context, take a sample */
	struct kperf_context ctx;
	ctx.cur_thread = thread;
	ctx.cur_pid = -1;

	task = chudxnu_task_for_thread(thread);
	if(task)
		ctx.cur_pid = chudxnu_pid_for_task(task);

	/* decode the chud bits so we know what to sample */
	t_chud = kperf_get_thread_bits(thread);
	
	if (t_chud & T_AST_NAME)
		sample_what |= SAMPLER_TINFOEX;
	
	if (t_chud & T_AST_CALLSTACK)
	{
		sample_what |= SAMPLER_USTACK;
		sample_what |= SAMPLER_TINFO;
	}

	/* do the sample, just of the user stuff */
	r = kperf_sample_internal( &sbuf, &ctx, sample_what, 0, 0 );

	BUF_INFO1(PERF_AST_HNDLR | DBG_FUNC_END, r);
}

/* register AST bits */
int
kperf_ast_pend( thread_t cur_thread, uint32_t check_bits,
		uint32_t set_bits )
{
	/* pend on the thread */
	uint32_t t_chud, set_done = 0;
 
	/* can only pend on the current thread */
	if( cur_thread != chudxnu_current_thread() )
		panic("pending to non-current thread");

	/* get our current bits */
	t_chud = kperf_get_thread_bits(cur_thread);

	/* see if it's already been done or pended */
	if( !(t_chud & check_bits ) )
	{
		/* set the bit on the thread */
		t_chud |= set_bits;
		kperf_set_thread_bits(cur_thread, t_chud);

		/* set the actual AST */
		kperf_set_thread_ast( cur_thread );

		set_done = 1;
	}

	return set_done;

//	BUF_INFO3( dbg_code, (uintptr_t)cur_thread, t_chud, set_done );
}

/*
 * kdebug callback & stack management
 */

#define IS_END(debugid)           ((debugid & 3) == DBG_FUNC_END)
#define IS_MIG(debugid)           (IS_END(debugid) && ((debugid & 0xff000000U) == KDBG_CLASS_ENCODE((unsigned)DBG_MIG, 0U)))
#define IS_MACH_SYSCALL(debugid)  (IS_END(debugid) && (KDBG_CLASS_DECODE(debugid) == KDBG_CLASS_ENCODE(DBG_MACH, DBG_MACH_EXCP_SC)))
#define IS_VM_FAULT(debugid)      (IS_END(debugid) && (KDBG_CLASS_DECODE(debugid) == KDBG_CLASS_ENCODE(DBG_MACH, DBG_MACH_VM)))
#define IS_BSD_SYSCTLL(debugid)   (IS_END(debugid) && (KDBG_CLASS_DECODE(debugid) == KDBG_CLASS_ENCODE(DBG_BSD, DBG_BSD_EXCP_SC)))
#define IS_APPS_SIGNPOST(debugid) (IS_END(debugid) && (KDBG_CLASS_DECODE(debugid) == KDBG_CLASS_ENCODE(DBG_APPS, DBG_MACH_CHUD)))
#define IS_MACH_SIGNPOST(debugid) (IS_END(debugid) && (KDBG_CLASS_DECODE(debugid) == KDBG_CLASS_ENCODE(DBG_MACH, DBG_MACH_CHUD)))

void
kperf_kdebug_callback(uint32_t debugid)
{
	int cur_pid = 0;
	task_t task = NULL;

	/* if we're not doing kperf callback stacks, return */
	if( !kdebug_callstack_action )
		return;

	/* if we're looking at a kperf tracepoint, don't recurse */
	if( (debugid & 0xff000000) == KDBG_CLASS_ENCODE(DBG_PERF, 0) )
		return;

	/* ensure interrupts are already off thanks to kdebug */
	if( ml_get_interrupts_enabled() )
		return;

	/* make sure we're not being called recursively.  */
#if NOTYET
	if( kperf_kdbg_recurse(KPERF_RECURSE_IN) )
		return;
#endif

	/* check the happy list of trace codes */
	if( !( IS_MIG(debugid)
	       || IS_MACH_SYSCALL(debugid)
	       || IS_VM_FAULT(debugid)
	       || IS_BSD_SYSCTLL(debugid)
	       || IS_MACH_SIGNPOST(debugid)
	       || IS_APPS_SIGNPOST(debugid) ) )
		return;

	/* check for kernel */
	thread_t thread = chudxnu_current_thread();
	task = chudxnu_task_for_thread(thread);
	if(task)
		cur_pid = chudxnu_pid_for_task(task);
	if( !cur_pid )
		return;

#if NOTYET
	/* setup a context */
	struct kperf_context ctx;
	struct kperf_sample *intbuf = NULL;

	ctx.cur_thread = thread;
	ctx.cur_pid = cur_pid;
	ctx.trigger_type = TRIGGER_TYPE_TRACE;
	ctx.trigger_id = 0;

	/* CPU sample buffer -- only valid with interrupts off (above)
	 * Technically this isn't true -- tracepoints can, and often
	 * are, cut from interrupt handlers, but none of those tracepoints
	 * should make it this far.
	 */
	intbuf = kperf_intr_sample_buffer();

	/* do the sample */
	kperf_sample( intbuf, &ctx, kdebug_callstack_action, SAMPLE_FLAG_PEND_USER );
	
	/* no longer recursive */
	kperf_kdbg_recurse(KPERF_RECURSE_OUT);
#else
	/* dicing with death */
	BUF_INFO2(PERF_KDBG_HNDLR, debugid, cur_pid);

	/* pend the AST */
	kperf_ast_pend( thread, T_AST_CALLSTACK, T_AST_CALLSTACK );
#endif

}

int
kperf_kdbg_get_stacks(void)
{
	return kdebug_callstack_action;
}

int
kperf_kdbg_set_stacks(int newval)
{
	/* set the value */
	kdebug_callstack_action = newval;

	/* enable the callback from kdebug */
	if( newval )
		chudxnu_kdebug_callback_enter(NULL);
	else
		chudxnu_kdebug_callback_cancel();

	return 0;
}

/*
 * Thread switch
 */

/* called from context switch handler */
void
kperf_switch_context( __unused thread_t old, thread_t new )
{
	task_t task = get_threadtask(new);
	int pid = chudxnu_pid_for_task(task);

	/* cut a tracepoint to tell us what the new thread's PID is
	 * for Instruments
	 */
	BUF_DATA2( PERF_TI_CSWITCH, thread_tid(new), pid );
}

/*
 * Action configuration
 */
unsigned
kperf_action_get_count(void)
{
	return actionc;
}

int
kperf_action_set_samplers( unsigned actionid, uint32_t samplers )
{
	if( (actionid > actionc) || (actionid == 0) )
		return EINVAL;

	actionv[actionid-1].sample = samplers;

	return 0;
}

int
kperf_action_get_samplers( unsigned actionid, uint32_t *samplers_out )
{
	if( (actionid > actionc) )
		return EINVAL;

	if( actionid == 0 )
		*samplers_out = 0; /* "NULL" action */
	else
		*samplers_out = actionv[actionid-1].sample;

	return 0;
}

int
kperf_action_set_userdata( unsigned actionid, uint32_t userdata )
{
	if( (actionid > actionc) || (actionid == 0) )
		return EINVAL;

	actionv[actionid-1].userdata = userdata;

	return 0;
}

int
kperf_action_get_userdata( unsigned actionid, uint32_t *userdata_out )
{
	if( (actionid > actionc) )
		return EINVAL;

	if( actionid == 0 )
		*userdata_out = 0; /* "NULL" action */
	else
		*userdata_out = actionv[actionid-1].userdata;

	return 0;
}

int
kperf_action_set_filter( unsigned actionid,
			 int pid )
{
	if( (actionid > actionc) || (actionid == 0) )
		return EINVAL;

	actionv[actionid-1].pid_filter = pid;

	return 0;
}

int
kperf_action_get_filter( unsigned actionid,
			 int *pid_out )
{
	if( (actionid > actionc) )
		return EINVAL;

	if( actionid == 0 )
		*pid_out = -1; /* "NULL" action */
	else
		*pid_out = actionv[actionid-1].pid_filter;

	return 0;
}

int
kperf_action_set_count(unsigned count)
{
	struct action *new_actionv = NULL, *old_actionv = NULL;
	unsigned old_count, i;

	/* easy no-op */
	if( count == actionc )
		return 0;

	/* TODO: allow shrinking? */
	if( count < actionc )
		return EINVAL;

	/* cap it for good measure */
	if( count > ACTION_MAX )
		return EINVAL;

	/* creating the action arror for the first time. create a few
	 * more things, too.
	 */
       	if( actionc == 0 )
	{
		int r;
		r = kperf_init();

		if( r != 0 )
			return r;
	}

	/* create a new array */
	new_actionv = kalloc( count * sizeof(*new_actionv) );
	if( new_actionv == NULL )
		return ENOMEM;

	old_actionv = actionv;
	old_count = actionc;

	if( old_actionv != NULL )
		bcopy( actionv, new_actionv, actionc * sizeof(*actionv) );

	bzero( &new_actionv[actionc], (count - old_count) * sizeof(*actionv) );

	for( i = old_count; i < count; i++ )
		new_actionv[i].pid_filter = -1;

	actionv = new_actionv;
	actionc = count;

	if( old_actionv != NULL )
		kfree( old_actionv, old_count * sizeof(*actionv) );

	return 0;
}
Commit	Line	Data
316670eb A	1	/*
	2	* Copyright (c) 2011 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	/*
	30	* Called from a trigger. Actually takes the data from the different
	31	* modules and puts them in a buffer
	32	*/
	33
	34	#include <mach/mach_types.h>
	35	#include <machine/machine_routines.h>
	36	// #include <libkern/libkern.h>
	37	#include <kern/kalloc.h>
	38	#include <kern/debug.h> /* panic */
	39	#include <kern/thread.h>
	40	#include <sys/errno.h>
	41
	42	#include <chud/chud_xnu.h>
	43	#include <kperf/kperf.h>
	44
	45	#include <kperf/buffer.h>
	46	#include <kperf/timetrigger.h>
	47	#include <kperf/threadinfo.h>
	48	#include <kperf/callstack.h>
	49	#include <kperf/sample.h>
316670eb A	50	#include <kperf/action.h>
	51	#include <kperf/context.h>
	52	#include <kperf/ast.h>
	53
	54	#define ACTION_MAX 32
	55
316670eb A	56	/* the list of different actions to take */
	57	struct action
	58	{
39236c6e A	59	uint32_t sample;
	60	uint32_t userdata;
	61	int pid_filter;
316670eb A	62	};
	63
	64	/* the list of actions */
	65	static unsigned actionc = 0;
	66	static struct action *actionv = NULL;
	67
39236c6e A	68	/* whether to record callstacks on kdebug events */
	69	static int kdebug_callstack_action = 0;
	70
	71	/* whether we get a callback on a thread switch */
	72	int kperf_cswitch_hook = 0;
	73
	74	/* indirect hooks to play nice with CHUD for the transition to kperf */
	75	kern_return_t chudxnu_kdebug_callback_enter(chudxnu_kdebug_callback_func_t fn);
	76	kern_return_t chudxnu_kdebug_callback_cancel(void);
316670eb A	77
	78	/* Do the real work! */
	79	/* this can be called in any context ... right? */
	80	static kern_return_t
	81	kperf_sample_internal( struct kperf_sample *sbuf,
39236c6e A	82	struct kperf_context *context,
	83	unsigned sample_what, unsigned sample_flags,
	84	unsigned actionid )
316670eb A	85	{
	86	boolean_t enabled;
	87	int did_ucallstack = 0, did_tinfo_extra = 0;
39236c6e	88	uint32_t userdata;
316670eb A	89
	90	/* not much point continuing here, but what to do ? return
	91	* Shutdown? cut a tracepoint and continue?
	92	*/
	93	if( sample_what == 0 )
	94	return SAMPLE_CONTINUE;
	95
	96	int is_kernel = (context->cur_pid == 0);
	97
39236c6e A	98	sbuf->kcallstack.nframes = 0;
	99	sbuf->kcallstack.flags = CALLSTACK_VALID;
	100	sbuf->ucallstack.nframes = 0;
	101	sbuf->ucallstack.flags = CALLSTACK_VALID;
	102
316670eb A	103	/* an event occurred. Sample everything and dump it in a
	104	* buffer.
	105	*/
	106
	107	/* collect data from samplers */
	108	if( sample_what & SAMPLER_TINFO ) {
	109	kperf_threadinfo_sample( &sbuf->threadinfo, context );
	110
39236c6e A	111	/* See if we should drop idle thread samples */
	112	if( !(sample_flags & SAMPLE_FLAG_IDLE_THREADS) )
	113	if (sbuf->threadinfo.runmode & 0x40)
	114	return SAMPLE_CONTINUE;
316670eb A	115	}
316670eb A	116
39236c6e	117	if( (sample_what & SAMPLER_KSTACK) && !(sample_flags & SAMPLE_FLAG_EMPTY_CALLSTACK) )
316670eb A	118	kperf_kcallstack_sample( &sbuf->kcallstack, context );
	119
	120	/* sensitive ones */
	121	if ( !is_kernel ) {
39236c6e	122	if( sample_flags & SAMPLE_FLAG_PEND_USER )
316670eb	123	{
39236c6e	124	if( (sample_what & SAMPLER_USTACK) && !(sample_flags & SAMPLE_FLAG_EMPTY_CALLSTACK) )
316670eb A	125	did_ucallstack = kperf_ucallstack_pend( context );
	126
	127	if( sample_what & SAMPLER_TINFOEX )
	128	did_tinfo_extra = kperf_threadinfo_extra_pend( context );
	129	}
	130	else
	131	{
39236c6e	132	if( (sample_what & SAMPLER_USTACK) && !(sample_flags & SAMPLE_FLAG_EMPTY_CALLSTACK) )
316670eb A	133	kperf_ucallstack_sample( &sbuf->ucallstack, context );
	134
	135	if( sample_what & SAMPLER_TINFOEX )
	136	kperf_threadinfo_extra_sample( &sbuf->tinfo_ex,
	137	context );
	138	}
	139	}
	140
39236c6e A	141	#if KPC
	142	if ( sample_what & SAMPLER_PMC_CPU )
	143	kperf_kpc_cpu_sample( &sbuf->kpcdata,
	144	(sample_what & SAMPLER_PMC_CPU) != 0 );
	145	#endif
	146
	147	/* lookup the user tag, if any */
	148	if( actionid
	149	&& (actionid <= actionc) )
	150	userdata = actionv[actionid-1].userdata;
	151	else
	152	userdata = actionid;
	153
316670eb A	154	/* stash the data into the buffer
	155	* interrupts off to ensure we don't get split
	156	*/
	157	enabled = ml_set_interrupts_enabled(FALSE);
	158
39236c6e A	159	BUF_DATA( PERF_GEN_EVENT \| DBG_FUNC_START, sample_what,
39236c6e A	160	actionid, userdata, sample_flags );
316670eb A	161
	162	/* dump threadinfo */
	163	if( sample_what & SAMPLER_TINFO )
	164	kperf_threadinfo_log( &sbuf->threadinfo );
	165
	166	/* dump kcallstack */
	167	if( sample_what & SAMPLER_KSTACK )
	168	kperf_kcallstack_log( &sbuf->kcallstack );
	169
	170
	171	/* dump user stuff */
	172	if ( !is_kernel ) {
39236c6e	173	if ( sample_flags & SAMPLE_FLAG_PEND_USER )
316670eb A	174	{
	175	if ( did_ucallstack )
	176	BUF_INFO1( PERF_CS_UPEND, 0 );
	177
	178	if ( did_tinfo_extra )
	179	BUF_INFO1( PERF_TI_XPEND, 0 );
	180	}
	181	else
	182	{
	183	if( sample_what & SAMPLER_USTACK )
	184	kperf_ucallstack_log( &sbuf->ucallstack );
	185
	186	if( sample_what & SAMPLER_TINFOEX )
	187	kperf_threadinfo_extra_log( &sbuf->tinfo_ex );
	188	}
	189	}
	190
39236c6e A	191	#if KPC
	192	if ( sample_what & SAMPLER_PMC_CPU )
	193	kperf_kpc_cpu_log( &sbuf->kpcdata );
	194
	195	#endif
	196
	197	BUF_DATA1( PERF_GEN_EVENT \| DBG_FUNC_END, sample_what );
316670eb A	198
	199	/* intrs back on */
	200	ml_set_interrupts_enabled(enabled);
	201
	202	return SAMPLE_CONTINUE;
	203	}
	204
	205	/* Translate actionid into sample bits and take a sample */
	206	kern_return_t
	207	kperf_sample( struct kperf_sample *sbuf,
	208	struct kperf_context *context,
39236c6e	209	unsigned actionid, unsigned sample_flags )
316670eb A	210	{
316670eb A	211	unsigned sample_what = 0;
39236c6e	212	int pid_filter;
316670eb A	213
316670eb A	214	/* work out what to sample, if anything */
39236c6e	215	if( (actionid > actionc) \|\| (actionid == 0) )
316670eb A	216	return SAMPLE_SHUTDOWN;
316670eb A	217
39236c6e A	218	/* check the pid filter against the context's current pid.
	219	* filter pid == -1 means any pid
	220	*/
	221	pid_filter = actionv[actionid-1].pid_filter;
	222	if( (pid_filter != -1)
	223	&& (pid_filter != context->cur_pid) )
	224	return SAMPLE_CONTINUE;
	225
	226	/* the samplers to run */
	227	sample_what = actionv[actionid-1].sample;
316670eb	228
39236c6e A	229	/* do the actual sample operation */
	230	return kperf_sample_internal( sbuf, context, sample_what,
	231	sample_flags, actionid );
316670eb A	232	}
	233
	234	/* ast callback on a thread */
	235	void
	236	kperf_thread_ast_handler( thread_t thread )
	237	{
	238	int r;
	239	uint32_t t_chud;
	240	unsigned sample_what = 0;
	241	/* we know we're on a thread, so let's do stuff */
	242	task_t task = NULL;
	243
316670eb A	244	BUF_INFO1(PERF_AST_HNDLR \| DBG_FUNC_START, thread);
316670eb A	245
39236c6e A	246	/* use ~2kb of the stack for the sample, should be ok since we're in the ast */
	247	struct kperf_sample sbuf;
	248	bzero(&sbuf, sizeof(struct kperf_sample));
316670eb A	249
	250	/* make a context, take a sample */
	251	struct kperf_context ctx;
	252	ctx.cur_thread = thread;
	253	ctx.cur_pid = -1;
	254
	255	task = chudxnu_task_for_thread(thread);
	256	if(task)
	257	ctx.cur_pid = chudxnu_pid_for_task(task);
	258
	259	/* decode the chud bits so we know what to sample */
	260	t_chud = kperf_get_thread_bits(thread);
	261
	262	if (t_chud & T_AST_NAME)
	263	sample_what \|= SAMPLER_TINFOEX;
	264
	265	if (t_chud & T_AST_CALLSTACK)
39236c6e	266	{
316670eb	267	sample_what \|= SAMPLER_USTACK;
39236c6e A	268	sample_what \|= SAMPLER_TINFO;
39236c6e A	269	}
316670eb A	270
316670eb A	271	/* do the sample, just of the user stuff */
39236c6e	272	r = kperf_sample_internal( &sbuf, &ctx, sample_what, 0, 0 );
316670eb	273
316670eb	274	BUF_INFO1(PERF_AST_HNDLR \| DBG_FUNC_END, r);
316670eb A	275	}
	276
	277	/* register AST bits */
	278	int
	279	kperf_ast_pend( thread_t cur_thread, uint32_t check_bits,
	280	uint32_t set_bits )
	281	{
	282	/* pend on the thread */
	283	uint32_t t_chud, set_done = 0;
	284
	285	/* can only pend on the current thread */
	286	if( cur_thread != chudxnu_current_thread() )
	287	panic("pending to non-current thread");
	288
	289	/* get our current bits */
	290	t_chud = kperf_get_thread_bits(cur_thread);
	291
	292	/* see if it's already been done or pended */
	293	if( !(t_chud & check_bits ) )
	294	{
	295	/* set the bit on the thread */
	296	t_chud \|= set_bits;
	297	kperf_set_thread_bits(cur_thread, t_chud);
	298
	299	/* set the actual AST */
	300	kperf_set_thread_ast( cur_thread );
	301
	302	set_done = 1;
	303	}
	304
	305	return set_done;
	306
	307	// BUF_INFO3( dbg_code, (uintptr_t)cur_thread, t_chud, set_done );
	308	}
	309
39236c6e A	310	/*
	311	* kdebug callback & stack management
	312	*/
	313
	314	#define IS_END(debugid) ((debugid & 3) == DBG_FUNC_END)
	315	#define IS_MIG(debugid) (IS_END(debugid) && ((debugid & 0xff000000U) == KDBG_CLASS_ENCODE((unsigned)DBG_MIG, 0U)))
	316	#define IS_MACH_SYSCALL(debugid) (IS_END(debugid) && (KDBG_CLASS_DECODE(debugid) == KDBG_CLASS_ENCODE(DBG_MACH, DBG_MACH_EXCP_SC)))
	317	#define IS_VM_FAULT(debugid) (IS_END(debugid) && (KDBG_CLASS_DECODE(debugid) == KDBG_CLASS_ENCODE(DBG_MACH, DBG_MACH_VM)))
	318	#define IS_BSD_SYSCTLL(debugid) (IS_END(debugid) && (KDBG_CLASS_DECODE(debugid) == KDBG_CLASS_ENCODE(DBG_BSD, DBG_BSD_EXCP_SC)))
	319	#define IS_APPS_SIGNPOST(debugid) (IS_END(debugid) && (KDBG_CLASS_DECODE(debugid) == KDBG_CLASS_ENCODE(DBG_APPS, DBG_MACH_CHUD)))
	320	#define IS_MACH_SIGNPOST(debugid) (IS_END(debugid) && (KDBG_CLASS_DECODE(debugid) == KDBG_CLASS_ENCODE(DBG_MACH, DBG_MACH_CHUD)))
	321
	322	void
	323	kperf_kdebug_callback(uint32_t debugid)
	324	{
	325	int cur_pid = 0;
	326	task_t task = NULL;
	327
	328	/* if we're not doing kperf callback stacks, return */
	329	if( !kdebug_callstack_action )
	330	return;
	331
	332	/* if we're looking at a kperf tracepoint, don't recurse */
	333	if( (debugid & 0xff000000) == KDBG_CLASS_ENCODE(DBG_PERF, 0) )
	334	return;
	335
	336	/* ensure interrupts are already off thanks to kdebug */
	337	if( ml_get_interrupts_enabled() )
	338	return;
	339
	340	/* make sure we're not being called recursively. */
	341	#if NOTYET
	342	if( kperf_kdbg_recurse(KPERF_RECURSE_IN) )
	343	return;
	344	#endif
	345
	346	/* check the happy list of trace codes */
	347	if( !( IS_MIG(debugid)
	348	\|\| IS_MACH_SYSCALL(debugid)
	349	\|\| IS_VM_FAULT(debugid)
	350	\|\| IS_BSD_SYSCTLL(debugid)
	351	\|\| IS_MACH_SIGNPOST(debugid)
	352	\|\| IS_APPS_SIGNPOST(debugid) ) )
	353	return;
	354
	355	/* check for kernel */
	356	thread_t thread = chudxnu_current_thread();
	357	task = chudxnu_task_for_thread(thread);
	358	if(task)
	359	cur_pid = chudxnu_pid_for_task(task);
	360	if( !cur_pid )
	361	return;
	362
	363	#if NOTYET
	364	/* setup a context */
	365	struct kperf_context ctx;
	366	struct kperf_sample *intbuf = NULL;
	367
	368	ctx.cur_thread = thread;
	369	ctx.cur_pid = cur_pid;
	370	ctx.trigger_type = TRIGGER_TYPE_TRACE;
	371	ctx.trigger_id = 0;
	372
	373	/* CPU sample buffer -- only valid with interrupts off (above)
374	* Technically this isn't true -- tracepoints can, and often
375	* are, cut from interrupt handlers, but none of those tracepoints
376	* should make it this far.
377	*/
378	intbuf = kperf_intr_sample_buffer();
379
380	/* do the sample */
381	kperf_sample( intbuf, &ctx, kdebug_callstack_action, SAMPLE_FLAG_PEND_USER );
382
383	/* no longer recursive */
384	kperf_kdbg_recurse(KPERF_RECURSE_OUT);
385	#else
386	/* dicing with death */
387	BUF_INFO2(PERF_KDBG_HNDLR, debugid, cur_pid);
388
389	/* pend the AST */
390	kperf_ast_pend( thread, T_AST_CALLSTACK, T_AST_CALLSTACK );
391	#endif
392
393	}
394
395	int
396	kperf_kdbg_get_stacks(void)
397	{
398	return kdebug_callstack_action;
399	}
400
401	int
402	kperf_kdbg_set_stacks(int newval)
403	{
404	/* set the value */
405	kdebug_callstack_action = newval;
406
407	/* enable the callback from kdebug */
408	if( newval )
409	chudxnu_kdebug_callback_enter(NULL);
410	else
411	chudxnu_kdebug_callback_cancel();
412
413	return 0;
414	}
415
416	/*
417	* Thread switch
418	*/
419
420	/* called from context switch handler */
421	void
422	kperf_switch_context( __unused thread_t old, thread_t new )
423	{
424	task_t task = get_threadtask(new);
425	int pid = chudxnu_pid_for_task(task);
426
427	/* cut a tracepoint to tell us what the new thread's PID is
428	* for Instruments
429	*/
430	BUF_DATA2( PERF_TI_CSWITCH, thread_tid(new), pid );
431	}
432
433	/*
434	* Action configuration
435	*/
316670eb A	436	unsigned
	437	kperf_action_get_count(void)
	438	{
	439	return actionc;
	440	}
	441
	442	int
	443	kperf_action_set_samplers( unsigned actionid, uint32_t samplers )
	444	{
39236c6e	445	if( (actionid > actionc) \|\| (actionid == 0) )
316670eb A	446	return EINVAL;
316670eb A	447
39236c6e	448	actionv[actionid-1].sample = samplers;
316670eb A	449
	450	return 0;
	451	}
	452
	453	int
	454	kperf_action_get_samplers( unsigned actionid, uint32_t *samplers_out )
	455	{
39236c6e	456	if( (actionid > actionc) )
316670eb A	457	return EINVAL;
316670eb A	458
39236c6e A	459	if( actionid == 0 )
	460	samplers_out = 0; / "NULL" action */
	461	else
	462	*samplers_out = actionv[actionid-1].sample;
	463
	464	return 0;
	465	}
	466
	467	int
	468	kperf_action_set_userdata( unsigned actionid, uint32_t userdata )
	469	{
	470	if( (actionid > actionc) \|\| (actionid == 0) )
	471	return EINVAL;
	472
	473	actionv[actionid-1].userdata = userdata;
	474
	475	return 0;
	476	}
	477
	478	int
	479	kperf_action_get_userdata( unsigned actionid, uint32_t *userdata_out )
	480	{
	481	if( (actionid > actionc) )
	482	return EINVAL;
	483
	484	if( actionid == 0 )
	485	userdata_out = 0; / "NULL" action */
	486	else
	487	*userdata_out = actionv[actionid-1].userdata;
	488
	489	return 0;
	490	}
	491
	492	int
	493	kperf_action_set_filter( unsigned actionid,
	494	int pid )
	495	{
	496	if( (actionid > actionc) \|\| (actionid == 0) )
	497	return EINVAL;
	498
	499	actionv[actionid-1].pid_filter = pid;
	500
	501	return 0;
	502	}
	503
	504	int
	505	kperf_action_get_filter( unsigned actionid,
	506	int *pid_out )
	507	{
	508	if( (actionid > actionc) )
	509	return EINVAL;
	510
	511	if( actionid == 0 )
	512	pid_out = -1; / "NULL" action */
	513	else
	514	*pid_out = actionv[actionid-1].pid_filter;
316670eb A	515
	516	return 0;
	517	}
	518
	519	int
	520	kperf_action_set_count(unsigned count)
	521	{
	522	struct action new_actionv = NULL, old_actionv = NULL;
39236c6e	523	unsigned old_count, i;
316670eb A	524
	525	/* easy no-op */
	526	if( count == actionc )
	527	return 0;
	528
	529	/* TODO: allow shrinking? */
	530	if( count < actionc )
	531	return EINVAL;
	532
	533	/* cap it for good measure */
	534	if( count > ACTION_MAX )
	535	return EINVAL;
	536
	537	/* creating the action arror for the first time. create a few
	538	* more things, too.
	539	*/
	540	if( actionc == 0 )
	541	{
	542	int r;
	543	r = kperf_init();
	544
	545	if( r != 0 )
	546	return r;
	547	}
	548
	549	/* create a new array */
	550	new_actionv = kalloc( count * sizeof(*new_actionv) );
	551	if( new_actionv == NULL )
	552	return ENOMEM;
	553
	554	old_actionv = actionv;
	555	old_count = actionc;
	556
	557	if( old_actionv != NULL )
	558	bcopy( actionv, new_actionv, actionc * sizeof(*actionv) );
	559
	560	bzero( &new_actionv[actionc], (count - old_count) * sizeof(*actionv) );
	561
39236c6e A	562	for( i = old_count; i < count; i++ )
	563	new_actionv[i].pid_filter = -1;
	564
316670eb A	565	actionv = new_actionv;
	566	actionc = count;
	567
	568	if( old_actionv != NULL )
	569	kfree( old_actionv, old_count * sizeof(*actionv) );
	570
316670eb A	571	return 0;
316670eb A	572	}