[apple/xnu.git] / osfmk / kperf / kperf_timer.c

/*
 * Copyright (c) 2011 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

/*  Manage timers */

#include <mach/mach_types.h>
#include <kern/cpu_data.h> /* current_thread() */
#include <kern/kalloc.h>
#include <stdatomic.h>
#include <sys/errno.h>
#include <sys/vm.h>
#include <sys/ktrace.h>

#include <machine/machine_routines.h>
#if defined(__x86_64__)
#include <i386/mp.h>
#endif /* defined(__x86_64__) */

#include <kperf/kperf.h>
#include <kperf/buffer.h>
#include <kperf/context.h>
#include <kperf/action.h>
#include <kperf/kperf_timer.h>
#include <kperf/kperf_arch.h>
#include <kperf/pet.h>
#include <kperf/sample.h>

/* the list of timers */
struct kperf_timer *kperf_timerv = NULL;
unsigned int kperf_timerc = 0;

static unsigned int pet_timer_id = 999;

/* maximum number of timers we can construct */
#define TIMER_MAX (16)

static uint64_t min_period_abstime;
static uint64_t min_period_bg_abstime;
static uint64_t min_period_pet_abstime;
static uint64_t min_period_pet_bg_abstime;

static uint64_t
kperf_timer_min_period_abstime(void)
{
	if (ktrace_background_active()) {
		return min_period_bg_abstime;
	} else {
		return min_period_abstime;
	}
}

static uint64_t
kperf_timer_min_pet_period_abstime(void)
{
	if (ktrace_background_active()) {
		return min_period_pet_bg_abstime;
	} else {
		return min_period_pet_abstime;
	}
}

static void
kperf_timer_schedule(struct kperf_timer *timer, uint64_t now)
{
	BUF_INFO(PERF_TM_SCHED, timer->period);

	/* if we re-programmed the timer to zero, just drop it */
	if (timer->period == 0) {
		return;
	}

	/* calculate deadline */
	uint64_t deadline = now + timer->period;

	/* re-schedule the timer, making sure we don't apply slop */
	timer_call_enter(&timer->tcall, deadline, TIMER_CALL_SYS_CRITICAL);
}

static void
kperf_sample_cpu(struct kperf_timer *timer, bool system_sample,
		bool only_system)
{
	assert(timer != NULL);

	/* Always cut a tracepoint to show a sample event occurred */
	BUF_DATA(PERF_TM_HNDLR | DBG_FUNC_START, 0);

	int ncpu = cpu_number();

	struct kperf_sample *intbuf = kperf_intr_sample_buffer();
#if DEVELOPMENT || DEBUG
	intbuf->sample_time = mach_absolute_time();
#endif /* DEVELOPMENT || DEBUG */

	/* On a timer, we can see the "real" current thread */
	struct kperf_context ctx = {
		.cur_thread = current_thread(),
		.trigger_type = TRIGGER_TYPE_TIMER,
		.trigger_id = (unsigned int)(timer - kperf_timerv),
	};
	ctx.cur_pid = task_pid(get_threadtask(ctx.cur_thread));

	if (ctx.trigger_id == pet_timer_id && ncpu < machine_info.logical_cpu_max) {
		kperf_tid_on_cpus[ncpu] = thread_tid(ctx.cur_thread);
	}

	/* make sure sampling is on */
	unsigned int status = kperf_sampling_status();
	if (status == KPERF_SAMPLING_OFF) {
		BUF_INFO(PERF_TM_HNDLR | DBG_FUNC_END, SAMPLE_OFF);
		return;
	} else if (status == KPERF_SAMPLING_SHUTDOWN) {
		BUF_INFO(PERF_TM_HNDLR | DBG_FUNC_END, SAMPLE_SHUTDOWN);
		return;
	}

	/* call the action -- kernel-only from interrupt, pend user */
	int r = kperf_sample(intbuf, &ctx, timer->actionid,
			SAMPLE_FLAG_PEND_USER | (system_sample ? SAMPLE_FLAG_SYSTEM : 0) |
			(only_system ? SAMPLE_FLAG_ONLY_SYSTEM : 0));

	/* end tracepoint is informational */
	BUF_INFO(PERF_TM_HNDLR | DBG_FUNC_END, r);

	(void)atomic_fetch_and_explicit(&timer->pending_cpus,
			~(UINT64_C(1) << ncpu), memory_order_relaxed);
}

void
kperf_ipi_handler(void *param)
{
	kperf_sample_cpu((struct kperf_timer *)param, false, false);
}

static void
kperf_timer_handler(void *param0, __unused void *param1)
{
	struct kperf_timer *timer = param0;
	unsigned int ntimer = (unsigned int)(timer - kperf_timerv);
	unsigned int ncpus  = machine_info.logical_cpu_max;
	bool system_only_self = true;

	if (timer->actionid == 0) {
		return;
	}

	timer->active = 1;
#if DEVELOPMENT || DEBUG
	timer->fire_time = mach_absolute_time();
#endif /* DEVELOPMENT || DEBUG */

	/* along the lines of do not ipi if we are all shutting down */
	if (kperf_sampling_status() == KPERF_SAMPLING_SHUTDOWN) {
		goto deactivate;
	}

	BUF_DATA(PERF_TM_FIRE, ntimer, ntimer == pet_timer_id, timer->period,
	                       timer->actionid);

	if (ntimer == pet_timer_id) {
		kperf_pet_fire_before();

		/* clean-up the thread-on-CPUs cache */
		bzero(kperf_tid_on_cpus, ncpus * sizeof(*kperf_tid_on_cpus));
	}

	/*
	 * IPI other cores only if the action has non-system samplers.
	 */
	if (kperf_sample_has_non_system(timer->actionid)) {
		/*
		 * If the core that's handling the timer is not scheduling
		 * threads, only run system samplers.
		 */
		system_only_self = kperf_mp_broadcast_other_running(timer);
	}
	kperf_sample_cpu(timer, true, system_only_self);

	/* release the pet thread? */
	if (ntimer == pet_timer_id) {
		/* PET mode is responsible for rearming the timer */
		kperf_pet_fire_after();
	} else {
		/*
		  * FIXME: Get the current time from elsewhere.  The next
		  * timer's period now includes the time taken to reach this
		  * point.  This causes a bias towards longer sampling periods
		  * than requested.
		  */
		kperf_timer_schedule(timer, mach_absolute_time());
	}

deactivate:
	timer->active = 0;
}

/* program the timer from the PET thread */
void
kperf_timer_pet_rearm(uint64_t elapsed_ticks)
{
	struct kperf_timer *timer = NULL;
	uint64_t period = 0;
	uint64_t deadline;

	/*
	 * If the pet_timer_id is invalid, it has been disabled, so this should
	 * do nothing.
	 */
	if (pet_timer_id >= kperf_timerc) {
		return;
	}

	unsigned int status = kperf_sampling_status();
	/* do not reprogram the timer if it has been shutdown or sampling is off */
	if (status == KPERF_SAMPLING_OFF) {
		BUF_INFO(PERF_PET_END, SAMPLE_OFF);
		return;
	} else if (status == KPERF_SAMPLING_SHUTDOWN) {
		BUF_INFO(PERF_PET_END, SAMPLE_SHUTDOWN);
		return;
	}

	timer = &(kperf_timerv[pet_timer_id]);

	/* if we re-programmed the timer to zero, just drop it */
	if (!timer->period) {
		return;
	}

	/* subtract the time the pet sample took being careful not to underflow */
	if (timer->period > elapsed_ticks) {
		period = timer->period - elapsed_ticks;
	}

	/* make sure we don't set the next PET sample to happen too soon */
	if (period < min_period_pet_abstime) {
		period = min_period_pet_abstime;
	}

	/* we probably took so long in the PET thread, it makes sense to take
	 * the time again.
	 */
	deadline = mach_absolute_time() + period;

	BUF_INFO(PERF_PET_SCHED, timer->period, period, elapsed_ticks, deadline);

	/* re-schedule the timer, making sure we don't apply slop */
	timer_call_enter(&timer->tcall, deadline, TIMER_CALL_SYS_CRITICAL);

	return;
}

/* turn on all the timers */
void
kperf_timer_go(void)
{
	/* get the PET thread going */
	if (pet_timer_id < kperf_timerc) {
		kperf_pet_config(kperf_timerv[pet_timer_id].actionid);
	}

	uint64_t now = mach_absolute_time();

	for (unsigned int i = 0; i < kperf_timerc; i++) {
		if (kperf_timerv[i].period == 0) {
			continue;
		}

		kperf_timer_schedule(&(kperf_timerv[i]), now);
	}
}

void
kperf_timer_stop(void)
{
	for (unsigned int i = 0; i < kperf_timerc; i++) {
		if (kperf_timerv[i].period == 0) {
			continue;
		}

		/* wait for the timer to stop */
		while (kperf_timerv[i].active);

		timer_call_cancel(&kperf_timerv[i].tcall);
	}

	/* wait for PET to stop, too */
	kperf_pet_config(0);
}

unsigned int
kperf_timer_get_petid(void)
{
	return pet_timer_id;
}

int
kperf_timer_set_petid(unsigned int timerid)
{
	if (timerid < kperf_timerc) {
		uint64_t min_period;

		min_period = kperf_timer_min_pet_period_abstime();
		if (kperf_timerv[timerid].period < min_period) {
			kperf_timerv[timerid].period = min_period;
		}
		kperf_pet_config(kperf_timerv[timerid].actionid);
	} else {
		/* clear the PET trigger if it's a bogus ID */
		kperf_pet_config(0);
	}

	pet_timer_id = timerid;

	return 0;
}

int
kperf_timer_get_period(unsigned int timerid, uint64_t *period_abstime)
{
	if (timerid >= kperf_timerc) {
		return EINVAL;
	}

	*period_abstime = kperf_timerv[timerid].period;
	return 0;
}

int
kperf_timer_set_period(unsigned int timerid, uint64_t period_abstime)
{
	uint64_t min_period;

	if (timerid >= kperf_timerc) {
		return EINVAL;
	}

	if (pet_timer_id == timerid) {
		min_period = kperf_timer_min_pet_period_abstime();
	} else {
		min_period = kperf_timer_min_period_abstime();
	}

	if (period_abstime > 0 && period_abstime < min_period) {
		period_abstime = min_period;
	}

	kperf_timerv[timerid].period = period_abstime;

	/* FIXME: re-program running timers? */

	return 0;
}

int
kperf_timer_get_action(unsigned int timerid, uint32_t *action)
{
	if (timerid >= kperf_timerc) {
		return EINVAL;
	}

	*action = kperf_timerv[timerid].actionid;
	return 0;
}

int
kperf_timer_set_action(unsigned int timerid, uint32_t action)
{
	if (timerid >= kperf_timerc) {
		return EINVAL;
	}

	kperf_timerv[timerid].actionid = action;
	return 0;
}

unsigned int
kperf_timer_get_count(void)
{
	return kperf_timerc;
}

void
kperf_timer_reset(void)
{
	kperf_timer_set_petid(999);
	kperf_set_pet_idle_rate(KPERF_PET_DEFAULT_IDLE_RATE);
	kperf_set_lightweight_pet(0);
	for (unsigned int i = 0; i < kperf_timerc; i++) {
		kperf_timerv[i].period = 0;
		kperf_timerv[i].actionid = 0;
		kperf_timerv[i].pending_cpus = 0;
	}
}

extern int
kperf_timer_set_count(unsigned int count)
{
	struct kperf_timer *new_timerv = NULL, *old_timerv = NULL;
	unsigned int old_count;

	if (min_period_abstime == 0) {
		nanoseconds_to_absolutetime(KP_MIN_PERIOD_NS, &min_period_abstime);
		nanoseconds_to_absolutetime(KP_MIN_PERIOD_BG_NS, &min_period_bg_abstime);
		nanoseconds_to_absolutetime(KP_MIN_PERIOD_PET_NS, &min_period_pet_abstime);
		nanoseconds_to_absolutetime(KP_MIN_PERIOD_PET_BG_NS,
			&min_period_pet_bg_abstime);
		assert(min_period_abstime > 0);
	}

	if (count == kperf_timerc) {
		return 0;
	}
	if (count > TIMER_MAX) {
		return EINVAL;
	}

	/* TODO: allow shrinking? */
	if (count < kperf_timerc) {
		return EINVAL;
	}

	/*
	 * Make sure kperf is initialized when creating the array for the first
	 * time.
	 */
	if (kperf_timerc == 0) {
		int r;

		/* main kperf */
		if ((r = kperf_init())) {
			return r;
		}
	}

	/*
	 * Shut down any running timers since we will be messing with the timer
	 * call structures.
	 */
	kperf_timer_stop();

	/* create a new array */
	new_timerv = kalloc_tag(count * sizeof(struct kperf_timer),
		VM_KERN_MEMORY_DIAG);
	if (new_timerv == NULL) {
		return ENOMEM;
	}
	old_timerv = kperf_timerv;
	old_count = kperf_timerc;

	if (old_timerv != NULL) {
		bcopy(kperf_timerv, new_timerv,
			kperf_timerc * sizeof(struct kperf_timer));
	}

	/* zero the new entries */
	bzero(&(new_timerv[kperf_timerc]),
		(count - old_count) * sizeof(struct kperf_timer));

	/* (re-)setup the timer call info for all entries */
	for (unsigned int i = 0; i < count; i++) {
		timer_call_setup(&new_timerv[i].tcall, kperf_timer_handler, &new_timerv[i]);
	}

	kperf_timerv = new_timerv;
	kperf_timerc = count;

	if (old_timerv != NULL) {
		kfree(old_timerv, old_count * sizeof(struct kperf_timer));
	}

	return 0;
}
Commit	Line	Data
39037602 A	1	/*
	2	* Copyright (c) 2011 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	/* Manage timers */
	30
	31	#include <mach/mach_types.h>
	32	#include <kern/cpu_data.h> /* current_thread() */
	33	#include <kern/kalloc.h>
5ba3f43e	34	#include <stdatomic.h>
39037602 A	35	#include <sys/errno.h>
	36	#include <sys/vm.h>
	37	#include <sys/ktrace.h>
	38
	39	#include <machine/machine_routines.h>
	40	#if defined(__x86_64__)
	41	#include <i386/mp.h>
	42	#endif /* defined(__x86_64__) */
	43
	44	#include <kperf/kperf.h>
	45	#include <kperf/buffer.h>
	46	#include <kperf/context.h>
	47	#include <kperf/action.h>
	48	#include <kperf/kperf_timer.h>
	49	#include <kperf/kperf_arch.h>
	50	#include <kperf/pet.h>
	51	#include <kperf/sample.h>
	52
	53	/* the list of timers */
	54	struct kperf_timer *kperf_timerv = NULL;
	55	unsigned int kperf_timerc = 0;
	56
	57	static unsigned int pet_timer_id = 999;
	58
	59	/* maximum number of timers we can construct */
	60	#define TIMER_MAX (16)
	61
39037602 A	62	static uint64_t min_period_abstime;
	63	static uint64_t min_period_bg_abstime;
	64	static uint64_t min_period_pet_abstime;
	65	static uint64_t min_period_pet_bg_abstime;
	66
	67	static uint64_t
	68	kperf_timer_min_period_abstime(void)
	69	{
	70	if (ktrace_background_active()) {
	71	return min_period_bg_abstime;
	72	} else {
	73	return min_period_abstime;
	74	}
	75	}
	76
	77	static uint64_t
	78	kperf_timer_min_pet_period_abstime(void)
	79	{
	80	if (ktrace_background_active()) {
	81	return min_period_pet_bg_abstime;
	82	} else {
	83	return min_period_pet_abstime;
	84	}
	85	}
	86
	87	static void
	88	kperf_timer_schedule(struct kperf_timer *timer, uint64_t now)
	89	{
	90	BUF_INFO(PERF_TM_SCHED, timer->period);
	91
	92	/* if we re-programmed the timer to zero, just drop it */
	93	if (timer->period == 0) {
	94	return;
	95	}
	96
	97	/* calculate deadline */
	98	uint64_t deadline = now + timer->period;
	99
	100	/* re-schedule the timer, making sure we don't apply slop */
	101	timer_call_enter(&timer->tcall, deadline, TIMER_CALL_SYS_CRITICAL);
	102	}
	103
5ba3f43e A	104	static void
	105	kperf_sample_cpu(struct kperf_timer *timer, bool system_sample,
	106	bool only_system)
39037602	107	{
39037602 A	108	assert(timer != NULL);
	109
	110	/* Always cut a tracepoint to show a sample event occurred */
	111	BUF_DATA(PERF_TM_HNDLR \| DBG_FUNC_START, 0);
	112
	113	int ncpu = cpu_number();
	114
	115	struct kperf_sample *intbuf = kperf_intr_sample_buffer();
a39ff7e2 A	116	#if DEVELOPMENT \|\| DEBUG
	117	intbuf->sample_time = mach_absolute_time();
	118	#endif /* DEVELOPMENT \|\| DEBUG */
39037602 A	119
39037602 A	120	/* On a timer, we can see the "real" current thread */
a39ff7e2 A	121	struct kperf_context ctx = {
	122	.cur_thread = current_thread(),
	123	.trigger_type = TRIGGER_TYPE_TIMER,
	124	.trigger_id = (unsigned int)(timer - kperf_timerv),
	125	};
39037602 A	126	ctx.cur_pid = task_pid(get_threadtask(ctx.cur_thread));
39037602 A	127
39037602	128	if (ctx.trigger_id == pet_timer_id && ncpu < machine_info.logical_cpu_max) {
5ba3f43e	129	kperf_tid_on_cpus[ncpu] = thread_tid(ctx.cur_thread);
39037602 A	130	}
	131
	132	/* make sure sampling is on */
	133	unsigned int status = kperf_sampling_status();
	134	if (status == KPERF_SAMPLING_OFF) {
	135	BUF_INFO(PERF_TM_HNDLR \| DBG_FUNC_END, SAMPLE_OFF);
	136	return;
	137	} else if (status == KPERF_SAMPLING_SHUTDOWN) {
	138	BUF_INFO(PERF_TM_HNDLR \| DBG_FUNC_END, SAMPLE_SHUTDOWN);
	139	return;
	140	}
	141
	142	/* call the action -- kernel-only from interrupt, pend user */
5ba3f43e A	143	int r = kperf_sample(intbuf, &ctx, timer->actionid,
	144	SAMPLE_FLAG_PEND_USER \| (system_sample ? SAMPLE_FLAG_SYSTEM : 0) \|
	145	(only_system ? SAMPLE_FLAG_ONLY_SYSTEM : 0));
39037602 A	146
	147	/* end tracepoint is informational */
	148	BUF_INFO(PERF_TM_HNDLR \| DBG_FUNC_END, r);
	149
5ba3f43e A	150	(void)atomic_fetch_and_explicit(&timer->pending_cpus,
	151	~(UINT64_C(1) << ncpu), memory_order_relaxed);
	152	}
	153
	154	void
	155	kperf_ipi_handler(void *param)
	156	{
	157	kperf_sample_cpu((struct kperf_timer *)param, false, false);
39037602 A	158	}
	159
	160	static void
	161	kperf_timer_handler(void param0, __unused void param1)
	162	{
	163	struct kperf_timer *timer = param0;
	164	unsigned int ntimer = (unsigned int)(timer - kperf_timerv);
	165	unsigned int ncpus = machine_info.logical_cpu_max;
5ba3f43e A	166	bool system_only_self = true;
	167
	168	if (timer->actionid == 0) {
	169	return;
	170	}
39037602 A	171
39037602 A	172	timer->active = 1;
a39ff7e2 A	173	#if DEVELOPMENT \|\| DEBUG
	174	timer->fire_time = mach_absolute_time();
	175	#endif /* DEVELOPMENT \|\| DEBUG */
39037602 A	176
	177	/* along the lines of do not ipi if we are all shutting down */
	178	if (kperf_sampling_status() == KPERF_SAMPLING_SHUTDOWN) {
	179	goto deactivate;
	180	}
	181
	182	BUF_DATA(PERF_TM_FIRE, ntimer, ntimer == pet_timer_id, timer->period,
	183	timer->actionid);
	184
	185	if (ntimer == pet_timer_id) {
	186	kperf_pet_fire_before();
	187
	188	/* clean-up the thread-on-CPUs cache */
5ba3f43e	189	bzero(kperf_tid_on_cpus, ncpus * sizeof(*kperf_tid_on_cpus));
39037602 A	190	}
39037602 A	191
5ba3f43e A	192	/*
	193	* IPI other cores only if the action has non-system samplers.
	194	*/
	195	if (kperf_sample_has_non_system(timer->actionid)) {
	196	/*
	197	* If the core that's handling the timer is not scheduling
	198	* threads, only run system samplers.
	199	*/
	200	system_only_self = kperf_mp_broadcast_other_running(timer);
	201	}
	202	kperf_sample_cpu(timer, true, system_only_self);
39037602 A	203
	204	/* release the pet thread? */
	205	if (ntimer == pet_timer_id) {
	206	/* PET mode is responsible for rearming the timer */
	207	kperf_pet_fire_after();
	208	} else {
	209	/*
	210	* FIXME: Get the current time from elsewhere. The next
	211	* timer's period now includes the time taken to reach this
	212	* point. This causes a bias towards longer sampling periods
	213	* than requested.
	214	*/
	215	kperf_timer_schedule(timer, mach_absolute_time());
	216	}
	217
	218	deactivate:
	219	timer->active = 0;
	220	}
	221
	222	/* program the timer from the PET thread */
	223	void
	224	kperf_timer_pet_rearm(uint64_t elapsed_ticks)
	225	{
	226	struct kperf_timer *timer = NULL;
	227	uint64_t period = 0;
	228	uint64_t deadline;
	229
	230	/*
	231	* If the pet_timer_id is invalid, it has been disabled, so this should
	232	* do nothing.
	233	*/
	234	if (pet_timer_id >= kperf_timerc) {
	235	return;
	236	}
	237
	238	unsigned int status = kperf_sampling_status();
	239	/* do not reprogram the timer if it has been shutdown or sampling is off */
	240	if (status == KPERF_SAMPLING_OFF) {
	241	BUF_INFO(PERF_PET_END, SAMPLE_OFF);
	242	return;
	243	} else if (status == KPERF_SAMPLING_SHUTDOWN) {
	244	BUF_INFO(PERF_PET_END, SAMPLE_SHUTDOWN);
	245	return;
	246	}
	247
	248	timer = &(kperf_timerv[pet_timer_id]);
	249
	250	/* if we re-programmed the timer to zero, just drop it */
	251	if (!timer->period) {
	252	return;
	253	}
	254
	255	/* subtract the time the pet sample took being careful not to underflow */
	256	if (timer->period > elapsed_ticks) {
	257	period = timer->period - elapsed_ticks;
	258	}
	259
	260	/* make sure we don't set the next PET sample to happen too soon */
	261	if (period < min_period_pet_abstime) {
	262	period = min_period_pet_abstime;
	263	}
	264
	265	/* we probably took so long in the PET thread, it makes sense to take
	266	* the time again.
267	*/
268	deadline = mach_absolute_time() + period;
269
270	BUF_INFO(PERF_PET_SCHED, timer->period, period, elapsed_ticks, deadline);
271
272	/* re-schedule the timer, making sure we don't apply slop */
5ba3f43e	273	timer_call_enter(&timer->tcall, deadline, TIMER_CALL_SYS_CRITICAL);
39037602 A	274
	275	return;
	276	}
	277
	278	/* turn on all the timers */
	279	void
	280	kperf_timer_go(void)
	281	{
	282	/* get the PET thread going */
	283	if (pet_timer_id < kperf_timerc) {
	284	kperf_pet_config(kperf_timerv[pet_timer_id].actionid);
	285	}
	286
	287	uint64_t now = mach_absolute_time();
	288
	289	for (unsigned int i = 0; i < kperf_timerc; i++) {
	290	if (kperf_timerv[i].period == 0) {
	291	continue;
	292	}
	293
	294	kperf_timer_schedule(&(kperf_timerv[i]), now);
	295	}
	296	}
	297
	298	void
	299	kperf_timer_stop(void)
	300	{
	301	for (unsigned int i = 0; i < kperf_timerc; i++) {
	302	if (kperf_timerv[i].period == 0) {
	303	continue;
	304	}
	305
	306	/* wait for the timer to stop */
	307	while (kperf_timerv[i].active);
	308
5ba3f43e	309	timer_call_cancel(&kperf_timerv[i].tcall);
39037602 A	310	}
	311
	312	/* wait for PET to stop, too */
	313	kperf_pet_config(0);
	314	}
	315
	316	unsigned int
	317	kperf_timer_get_petid(void)
	318	{
	319	return pet_timer_id;
	320	}
	321
	322	int
	323	kperf_timer_set_petid(unsigned int timerid)
	324	{
	325	if (timerid < kperf_timerc) {
	326	uint64_t min_period;
	327
	328	min_period = kperf_timer_min_pet_period_abstime();
	329	if (kperf_timerv[timerid].period < min_period) {
	330	kperf_timerv[timerid].period = min_period;
	331	}
	332	kperf_pet_config(kperf_timerv[timerid].actionid);
	333	} else {
	334	/* clear the PET trigger if it's a bogus ID */
	335	kperf_pet_config(0);
	336	}
	337
	338	pet_timer_id = timerid;
	339
	340	return 0;
	341	}
	342
	343	int
	344	kperf_timer_get_period(unsigned int timerid, uint64_t *period_abstime)
	345	{
	346	if (timerid >= kperf_timerc) {
	347	return EINVAL;
	348	}
	349
	350	*period_abstime = kperf_timerv[timerid].period;
	351	return 0;
	352	}
	353
	354	int
	355	kperf_timer_set_period(unsigned int timerid, uint64_t period_abstime)
	356	{
	357	uint64_t min_period;
	358
	359	if (timerid >= kperf_timerc) {
	360	return EINVAL;
	361	}
	362
	363	if (pet_timer_id == timerid) {
	364	min_period = kperf_timer_min_pet_period_abstime();
	365	} else {
	366	min_period = kperf_timer_min_period_abstime();
	367	}
	368
	369	if (period_abstime > 0 && period_abstime < min_period) {
	370	period_abstime = min_period;
	371	}
	372
	373	kperf_timerv[timerid].period = period_abstime;
374
375	/* FIXME: re-program running timers? */
376
377	return 0;
378	}
379
380	int
381	kperf_timer_get_action(unsigned int timerid, uint32_t *action)
382	{
383	if (timerid >= kperf_timerc) {
384	return EINVAL;
385	}
386
387	*action = kperf_timerv[timerid].actionid;
388	return 0;
389	}
390
391	int
392	kperf_timer_set_action(unsigned int timerid, uint32_t action)
393	{
394	if (timerid >= kperf_timerc) {
395	return EINVAL;
396	}
397
398	kperf_timerv[timerid].actionid = action;
399	return 0;
400	}
401
402	unsigned int
403	kperf_timer_get_count(void)
404	{
405	return kperf_timerc;
406	}
407
408	void
409	kperf_timer_reset(void)
410	{
411	kperf_timer_set_petid(999);
412	kperf_set_pet_idle_rate(KPERF_PET_DEFAULT_IDLE_RATE);
413	kperf_set_lightweight_pet(0);
414	for (unsigned int i = 0; i < kperf_timerc; i++) {
415	kperf_timerv[i].period = 0;
416	kperf_timerv[i].actionid = 0;
39037602	417	kperf_timerv[i].pending_cpus = 0;
39037602 A	418	}
	419	}
	420
	421	extern int
	422	kperf_timer_set_count(unsigned int count)
	423	{
	424	struct kperf_timer new_timerv = NULL, old_timerv = NULL;
	425	unsigned int old_count;
	426
	427	if (min_period_abstime == 0) {
5ba3f43e A	428	nanoseconds_to_absolutetime(KP_MIN_PERIOD_NS, &min_period_abstime);
	429	nanoseconds_to_absolutetime(KP_MIN_PERIOD_BG_NS, &min_period_bg_abstime);
	430	nanoseconds_to_absolutetime(KP_MIN_PERIOD_PET_NS, &min_period_pet_abstime);
	431	nanoseconds_to_absolutetime(KP_MIN_PERIOD_PET_BG_NS,
39037602 A	432	&min_period_pet_bg_abstime);
	433	assert(min_period_abstime > 0);
	434	}
	435
	436	if (count == kperf_timerc) {
	437	return 0;
	438	}
	439	if (count > TIMER_MAX) {
	440	return EINVAL;
	441	}
	442
	443	/* TODO: allow shrinking? */
	444	if (count < kperf_timerc) {
	445	return EINVAL;
	446	}
	447
	448	/*
	449	* Make sure kperf is initialized when creating the array for the first
	450	* time.
	451	*/
	452	if (kperf_timerc == 0) {
	453	int r;
	454
	455	/* main kperf */
	456	if ((r = kperf_init())) {
	457	return r;
	458	}
	459	}
	460
	461	/*
	462	* Shut down any running timers since we will be messing with the timer
	463	* call structures.
	464	*/
	465	kperf_timer_stop();
	466
	467	/* create a new array */
	468	new_timerv = kalloc_tag(count * sizeof(struct kperf_timer),
	469	VM_KERN_MEMORY_DIAG);
	470	if (new_timerv == NULL) {
	471	return ENOMEM;
	472	}
	473	old_timerv = kperf_timerv;
	474	old_count = kperf_timerc;
	475
	476	if (old_timerv != NULL) {
	477	bcopy(kperf_timerv, new_timerv,
	478	kperf_timerc * sizeof(struct kperf_timer));
	479	}
	480
	481	/* zero the new entries */
	482	bzero(&(new_timerv[kperf_timerc]),
	483	(count - old_count) * sizeof(struct kperf_timer));
	484
	485	/* (re-)setup the timer call info for all entries */
	486	for (unsigned int i = 0; i < count; i++) {
5ba3f43e	487	timer_call_setup(&new_timerv[i].tcall, kperf_timer_handler, &new_timerv[i]);
39037602 A	488	}
	489
	490	kperf_timerv = new_timerv;
	491	kperf_timerc = count;
	492
	493	if (old_timerv != NULL) {
	494	kfree(old_timerv, old_count * sizeof(struct kperf_timer));
	495	}
	496
	497	return 0;
	498	}