[apple/xnu.git] / osfmk / kern / kpc_thread.c

/*
 * Copyright (c) 2012 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <mach/mach_types.h>
#include <kern/processor.h>
#include <kern/thread.h>
#include <kern/assert.h>
#include <kern/locks.h>
#include <sys/errno.h>

#include <kperf/kperf.h>
#include <kperf/buffer.h>
#include <kperf/context.h>
#include <kperf/sample.h>
#include <kperf/action.h>
#include <kperf/kperf_kpc.h>
#include <kern/kpc.h>

#if defined (__arm64__)
#include <arm/cpu_data_internal.h>
#elif defined (__arm__)
#include <arm/cpu_data_internal.h>
#endif

/* global for whether to read PMCs on context switch */
int kpc_threads_counting = 0;

/* whether to call into KPC when a thread goes off CPU */
boolean_t kpc_off_cpu_active = FALSE;

/* current config and number of counters in that config */
static uint32_t kpc_thread_classes = 0;
static uint32_t kpc_thread_classes_count = 0;

static lck_grp_attr_t *kpc_thread_lckgrp_attr = NULL;
static lck_grp_t      *kpc_thread_lckgrp = NULL;
static lck_mtx_t       kpc_thread_lock;

void
kpc_thread_init(void)
{
	kpc_thread_lckgrp_attr = lck_grp_attr_alloc_init();
	kpc_thread_lckgrp = lck_grp_alloc_init("kpc", kpc_thread_lckgrp_attr);
	lck_mtx_init(&kpc_thread_lock, kpc_thread_lckgrp, LCK_ATTR_NULL);
}

uint32_t
kpc_get_thread_counting(void)
{
	uint32_t kpc_thread_classes_tmp;
	int kpc_threads_counting_tmp;

	/* Make sure we get a consistent snapshot of these values */
	lck_mtx_lock(&kpc_thread_lock);

	kpc_thread_classes_tmp = kpc_thread_classes;
	kpc_threads_counting_tmp = kpc_threads_counting;

	lck_mtx_unlock(&kpc_thread_lock);

	if( kpc_threads_counting_tmp )
		return kpc_thread_classes_tmp;
	else
		return 0;
}

int
kpc_set_thread_counting(uint32_t classes)
{
	uint32_t count;

	lck_mtx_lock(&kpc_thread_lock);

	count = kpc_get_counter_count(classes);

	if( (classes == 0)
	    || (count == 0) )
	{
		/* shut down */
		kpc_threads_counting = FALSE;
	}
	else
	{
		/* stash the config */
		kpc_thread_classes = classes;

		/* work out the size */
		kpc_thread_classes_count = count;
		assert(kpc_thread_classes_count <= KPC_MAX_COUNTERS);

		/* enable switch */
		kpc_threads_counting = TRUE;

		/* and schedule an AST for this thread... */
		if( !current_thread()->kpc_buf )
		{
			current_thread()->kperf_flags |= T_KPC_ALLOC;
			act_set_kperf(current_thread());
		}
	}

    kpc_off_cpu_update();
	lck_mtx_unlock(&kpc_thread_lock);

	return 0;
}

/* snapshot current PMCs and update counters in the current thread */
static void
kpc_update_thread_counters( thread_t thread )
{
	uint32_t i;
	uint64_t *tmp = NULL;
	cpu_data_t *cpu = NULL;

	cpu = current_cpu_datap();

	/* 1. stash current PMCs into latest CPU block */
	kpc_get_cpu_counters( FALSE, kpc_thread_classes, 
	                      NULL, cpu->cpu_kpc_buf[1] );

	/* 2. apply delta to old thread */
	if( thread->kpc_buf )
		for( i = 0; i < kpc_thread_classes_count; i++ )
			thread->kpc_buf[i] += cpu->cpu_kpc_buf[1][i] - cpu->cpu_kpc_buf[0][i];

	/* schedule any necessary allocations */
	if( !current_thread()->kpc_buf )
	{
		current_thread()->kperf_flags |= T_KPC_ALLOC;
		act_set_kperf(current_thread());
	}

	/* 3. switch the PMC block pointers */
	tmp = cpu->cpu_kpc_buf[1];
	cpu->cpu_kpc_buf[1] = cpu->cpu_kpc_buf[0];
	cpu->cpu_kpc_buf[0] = tmp;
}

/* get counter values for a thread */
int
kpc_get_curthread_counters(uint32_t *inoutcount, uint64_t *buf)
{
	thread_t thread = current_thread();
	boolean_t enabled;

	/* buffer too small :( */
	if( *inoutcount < kpc_thread_classes_count )
		return EINVAL;

	/* copy data and actual size */
	if( !thread->kpc_buf )
		return EINVAL;

	enabled = ml_set_interrupts_enabled(FALSE);

	/* snap latest version of counters for this thread */
	kpc_update_thread_counters( current_thread() );
	
	/* copy out */
	memcpy( buf, thread->kpc_buf, 
	        kpc_thread_classes_count * sizeof(*buf) );
	*inoutcount = kpc_thread_classes_count;

	ml_set_interrupts_enabled(enabled);

	return 0;
}

void
kpc_off_cpu_update(void)
{
	kpc_off_cpu_active = kpc_threads_counting;
}

void
kpc_off_cpu_internal(thread_t thread)
{
	if (kpc_threads_counting) {
		kpc_update_thread_counters(thread);
	}
}

void
kpc_thread_create(thread_t thread)
{
	/* nothing to do if we're not counting */
	if(!kpc_threads_counting)
		return;

	/* give the new thread a counterbuf */
	thread->kpc_buf = kpc_counterbuf_alloc();
}

void
kpc_thread_destroy(thread_t thread)
{
	uint64_t *buf = NULL;

	/* usual case: no kpc buf, just return */
	if( !thread->kpc_buf )
		return;

	/* otherwise, don't leak */
	buf = thread->kpc_buf;
	thread->kpc_buf = NULL;
	kpc_counterbuf_free(buf);
}

/* ast callback on a thread */
void
kpc_thread_ast_handler( thread_t thread )
{
	/* see if we want an alloc */
	if( thread->kperf_flags & T_KPC_ALLOC )
		thread->kpc_buf = kpc_counterbuf_alloc();
}
Commit	Line	Data
39236c6e A	1	/*
	2	* Copyright (c) 2012 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	#include <mach/mach_types.h>
	30	#include <kern/processor.h>
	31	#include <kern/thread.h>
	32	#include <kern/assert.h>
	33	#include <kern/locks.h>
	34	#include <sys/errno.h>
	35
3e170ce0 A	36	#include <kperf/kperf.h>
	37	#include <kperf/buffer.h>
	38	#include <kperf/context.h>
	39	#include <kperf/sample.h>
	40	#include <kperf/action.h>
	41	#include <kperf/kperf_kpc.h>
39236c6e A	42	#include <kern/kpc.h>
39236c6e A	43
5ba3f43e A	44	#if defined (__arm64__)
	45	#include <arm/cpu_data_internal.h>
	46	#elif defined (__arm__)
	47	#include <arm/cpu_data_internal.h>
	48	#endif
39236c6e A	49
39236c6e A	50	/* global for whether to read PMCs on context switch */
3e170ce0	51	int kpc_threads_counting = 0;
39236c6e	52
39037602 A	53	/* whether to call into KPC when a thread goes off CPU */
	54	boolean_t kpc_off_cpu_active = FALSE;
	55
39236c6e A	56	/* current config and number of counters in that config */
	57	static uint32_t kpc_thread_classes = 0;
	58	static uint32_t kpc_thread_classes_count = 0;
	59
	60	static lck_grp_attr_t *kpc_thread_lckgrp_attr = NULL;
	61	static lck_grp_t *kpc_thread_lckgrp = NULL;
	62	static lck_mtx_t kpc_thread_lock;
	63
39236c6e A	64	void
	65	kpc_thread_init(void)
	66	{
	67	kpc_thread_lckgrp_attr = lck_grp_attr_alloc_init();
	68	kpc_thread_lckgrp = lck_grp_alloc_init("kpc", kpc_thread_lckgrp_attr);
	69	lck_mtx_init(&kpc_thread_lock, kpc_thread_lckgrp, LCK_ATTR_NULL);
	70	}
	71
	72	uint32_t
	73	kpc_get_thread_counting(void)
	74	{
	75	uint32_t kpc_thread_classes_tmp;
	76	int kpc_threads_counting_tmp;
	77
	78	/* Make sure we get a consistent snapshot of these values */
	79	lck_mtx_lock(&kpc_thread_lock);
	80
	81	kpc_thread_classes_tmp = kpc_thread_classes;
	82	kpc_threads_counting_tmp = kpc_threads_counting;
	83
	84	lck_mtx_unlock(&kpc_thread_lock);
	85
	86	if( kpc_threads_counting_tmp )
	87	return kpc_thread_classes_tmp;
	88	else
	89	return 0;
	90	}
	91
	92	int
	93	kpc_set_thread_counting(uint32_t classes)
	94	{
	95	uint32_t count;
	96
	97	lck_mtx_lock(&kpc_thread_lock);
	98
	99	count = kpc_get_counter_count(classes);
	100
	101	if( (classes == 0)
	102	\|\| (count == 0) )
	103	{
	104	/* shut down */
	105	kpc_threads_counting = FALSE;
	106	}
	107	else
	108	{
	109	/* stash the config */
	110	kpc_thread_classes = classes;
	111
	112	/* work out the size */
	113	kpc_thread_classes_count = count;
	114	assert(kpc_thread_classes_count <= KPC_MAX_COUNTERS);
	115
	116	/* enable switch */
	117	kpc_threads_counting = TRUE;
	118
	119	/* and schedule an AST for this thread... */
	120	if( !current_thread()->kpc_buf )
	121	{
39037602	122	current_thread()->kperf_flags \|= T_KPC_ALLOC;
39236c6e	123	act_set_kperf(current_thread());
39037602	124	}
39236c6e A	125	}
39236c6e A	126
39037602	127	kpc_off_cpu_update();
39236c6e A	128	lck_mtx_unlock(&kpc_thread_lock);
	129
	130	return 0;
	131	}
	132
	133	/* snapshot current PMCs and update counters in the current thread */
	134	static void
	135	kpc_update_thread_counters( thread_t thread )
	136	{
	137	uint32_t i;
	138	uint64_t *tmp = NULL;
	139	cpu_data_t *cpu = NULL;
	140
39236c6e	141	cpu = current_cpu_datap();
39236c6e A	142
	143	/* 1. stash current PMCs into latest CPU block */
	144	kpc_get_cpu_counters( FALSE, kpc_thread_classes,
	145	NULL, cpu->cpu_kpc_buf[1] );
	146
	147	/* 2. apply delta to old thread */
	148	if( thread->kpc_buf )
	149	for( i = 0; i < kpc_thread_classes_count; i++ )
	150	thread->kpc_buf[i] += cpu->cpu_kpc_buf[1][i] - cpu->cpu_kpc_buf[0][i];
	151
39236c6e A	152	/* schedule any necessary allocations */
	153	if( !current_thread()->kpc_buf )
	154	{
39037602	155	current_thread()->kperf_flags \|= T_KPC_ALLOC;
39236c6e	156	act_set_kperf(current_thread());
39037602	157	}
39236c6e A	158
	159	/* 3. switch the PMC block pointers */
	160	tmp = cpu->cpu_kpc_buf[1];
	161	cpu->cpu_kpc_buf[1] = cpu->cpu_kpc_buf[0];
	162	cpu->cpu_kpc_buf[0] = tmp;
	163	}
	164
39236c6e A	165	/* get counter values for a thread */
	166	int
	167	kpc_get_curthread_counters(uint32_t inoutcount, uint64_t buf)
	168	{
	169	thread_t thread = current_thread();
	170	boolean_t enabled;
	171
	172	/* buffer too small :( */
	173	if( *inoutcount < kpc_thread_classes_count )
	174	return EINVAL;
	175
	176	/* copy data and actual size */
	177	if( !thread->kpc_buf )
	178	return EINVAL;
	179
	180	enabled = ml_set_interrupts_enabled(FALSE);
	181
	182	/* snap latest version of counters for this thread */
	183	kpc_update_thread_counters( current_thread() );
	184
	185	/* copy out */
	186	memcpy( buf, thread->kpc_buf,
	187	kpc_thread_classes_count * sizeof(*buf) );
	188	*inoutcount = kpc_thread_classes_count;
	189
	190	ml_set_interrupts_enabled(enabled);
	191
	192	return 0;
	193	}
	194
39037602 A	195	void
	196	kpc_off_cpu_update(void)
	197	{
	198	kpc_off_cpu_active = kpc_threads_counting;
	199	}
	200
	201	void
	202	kpc_off_cpu_internal(thread_t thread)
	203	{
	204	if (kpc_threads_counting) {
	205	kpc_update_thread_counters(thread);
	206	}
	207	}
39236c6e A	208
	209	void
	210	kpc_thread_create(thread_t thread)
	211	{
	212	/* nothing to do if we're not counting */
	213	if(!kpc_threads_counting)
	214	return;
	215
	216	/* give the new thread a counterbuf */
	217	thread->kpc_buf = kpc_counterbuf_alloc();
	218	}
	219
	220	void
	221	kpc_thread_destroy(thread_t thread)
	222	{
	223	uint64_t *buf = NULL;
	224
	225	/* usual case: no kpc buf, just return */
	226	if( !thread->kpc_buf )
	227	return;
	228
	229	/* otherwise, don't leak */
	230	buf = thread->kpc_buf;
	231	thread->kpc_buf = NULL;
	232	kpc_counterbuf_free(buf);
	233	}
	234
	235	/* ast callback on a thread */
	236	void
	237	kpc_thread_ast_handler( thread_t thread )
	238	{
	239	/* see if we want an alloc */
39037602	240	if( thread->kperf_flags & T_KPC_ALLOC )
39236c6e A	241	thread->kpc_buf = kpc_counterbuf_alloc();
39236c6e A	242	}