[apple/xnu.git] / osfmk / i386 / i386_timer.c

/*
 * Copyright (c) 2000-2008 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
/*
 * @OSF_COPYRIGHT@
 */
/*
 * @APPLE_FREE_COPYRIGHT@
 */
/*
 *	File:		timer.c
 *	Purpose:	Routines for handling the machine independent timer.
 */

#include <mach/mach_types.h>

#include <kern/timer_queue.h>
#include <kern/timer_call.h>
#include <kern/clock.h>
#include <kern/thread.h>
#include <kern/processor.h>
#include <kern/macro_help.h>
#include <kern/spl.h>
#include <kern/timer_queue.h>
#include <kern/pms.h>

#include <machine/commpage.h>
#include <machine/machine_routines.h>

#include <sys/kdebug.h>
#include <i386/cpu_data.h>
#include <i386/cpu_topology.h>
#include <i386/cpu_threads.h>

uint32_t spurious_timers;

/*
 *      Event timer interrupt.
 *
 * XXX a drawback of this implementation is that events serviced earlier must not set deadlines
 *     that occur before the entire chain completes.
 *
 * XXX a better implementation would use a set of generic callouts and iterate over them
 */
void
timer_intr(int          user_mode,
    uint64_t    rip)
{
	uint64_t                abstime;
	rtclock_timer_t         *mytimer;
	cpu_data_t              *pp;
	int64_t                 latency;
	uint64_t                pmdeadline;
	boolean_t               timer_processed = FALSE;

	pp = current_cpu_datap();

	SCHED_STATS_TIMER_POP(current_processor());

	abstime = mach_absolute_time();         /* Get the time now */

	/* has a pending clock timer expired? */
	mytimer = &pp->rtclock_timer;           /* Point to the event timer */

	if ((timer_processed = ((mytimer->deadline <= abstime) ||
	    (abstime >= (mytimer->queue.earliest_soft_deadline))))) {
		/*
		 * Log interrupt service latency (-ve value expected by tool)
		 * a non-PM event is expected next.
		 * The requested deadline may be earlier than when it was set
		 * - use MAX to avoid reporting bogus latencies.
		 */
		latency = (int64_t) (abstime - MAX(mytimer->deadline,
		    mytimer->when_set));
		/* Log zero timer latencies when opportunistically processing
		 * coalesced timers.
		 */
		if (latency < 0) {
			TCOAL_DEBUG(0xEEEE0000, abstime, mytimer->queue.earliest_soft_deadline, abstime - mytimer->queue.earliest_soft_deadline, 0, 0);
			latency = 0;
		}

		KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
		    DECR_TRAP_LATENCY | DBG_FUNC_NONE,
		    -latency,
		    ((user_mode != 0) ? rip : VM_KERNEL_UNSLIDE(rip)),
		    user_mode, 0, 0);

		mytimer->has_expired = TRUE;    /* Remember that we popped */
		mytimer->deadline = timer_queue_expire(&mytimer->queue, abstime);
		mytimer->has_expired = FALSE;

		/* Get the time again since we ran a bit */
		abstime = mach_absolute_time();
		mytimer->when_set = abstime;
	}

	/* is it time for power management state change? */
	if ((pmdeadline = pmCPUGetDeadline(pp)) && (pmdeadline <= abstime)) {
		KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
		    DECR_PM_DEADLINE | DBG_FUNC_START,
		    0, 0, 0, 0, 0);
		pmCPUDeadline(pp);
		KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
		    DECR_PM_DEADLINE | DBG_FUNC_END,
		    0, 0, 0, 0, 0);
		timer_processed = TRUE;
		abstime = mach_absolute_time(); /* Get the time again since we ran a bit */
	}

	uint64_t quantum_deadline = pp->quantum_timer_deadline;
	/* is it the quantum timer expiration? */
	if ((quantum_deadline <= abstime) && (quantum_deadline > 0)) {
		pp->quantum_timer_deadline = 0;
		quantum_timer_expire(abstime);
	}

	/* schedule our next deadline */
	x86_lcpu()->rtcDeadline = EndOfAllTime;
	timer_resync_deadlines();

	if (__improbable(timer_processed == FALSE)) {
		spurious_timers++;
	}
}

/*
 * Set the clock deadline.
 */
void
timer_set_deadline(uint64_t deadline)
{
	rtclock_timer_t         *mytimer;
	spl_t                   s;
	cpu_data_t              *pp;

	s = splclock();                         /* no interruptions */
	pp = current_cpu_datap();

	mytimer = &pp->rtclock_timer;           /* Point to the timer itself */
	mytimer->deadline = deadline;           /* Set new expiration time */
	mytimer->when_set = mach_absolute_time();

	timer_resync_deadlines();

	splx(s);
}

void
quantum_timer_set_deadline(uint64_t deadline)
{
	cpu_data_t              *pp;
	/* We should've only come into this path with interrupts disabled */
	assert(ml_get_interrupts_enabled() == FALSE);

	pp = current_cpu_datap();
	pp->quantum_timer_deadline = deadline;
	timer_resync_deadlines();
}

/*
 * Re-evaluate the outstanding deadlines and select the most proximate.
 *
 * Should be called at splclock.
 */
void
timer_resync_deadlines(void)
{
	uint64_t                deadline = EndOfAllTime;
	uint64_t                pmdeadline;
	uint64_t                quantum_deadline;
	rtclock_timer_t         *mytimer;
	spl_t                   s = splclock();
	cpu_data_t              *pp;
	uint32_t                decr;

	pp = current_cpu_datap();
	if (!pp->cpu_running) {
		/* There's really nothing to do if this processor is down */
		return;
	}

	/*
	 * If we have a clock timer set, pick that.
	 */
	mytimer = &pp->rtclock_timer;
	if (!mytimer->has_expired &&
	    0 < mytimer->deadline && mytimer->deadline < EndOfAllTime) {
		deadline = mytimer->deadline;
	}

	/*
	 * If we have a power management deadline, see if that's earlier.
	 */
	pmdeadline = pmCPUGetDeadline(pp);
	if (0 < pmdeadline && pmdeadline < deadline) {
		deadline = pmdeadline;
	}

	/* If we have the quantum timer setup, check that */
	quantum_deadline = pp->quantum_timer_deadline;
	if ((quantum_deadline > 0) &&
	    (quantum_deadline < deadline)) {
		deadline = quantum_deadline;
	}


	/*
	 * Go and set the "pop" event.
	 */
	decr = (uint32_t) setPop(deadline);

	/* Record non-PM deadline for latency tool */
	if (decr != 0 && deadline != pmdeadline) {
		uint64_t queue_count = 0;
		if (deadline != quantum_deadline) {
			/*
			 * For non-quantum timer put the queue count
			 * in the tracepoint.
			 */
			queue_count = mytimer->queue.count;
		}
		KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
		    DECR_SET_DEADLINE | DBG_FUNC_NONE,
		    decr, 2,
		    deadline,
		    queue_count, 0);
	}
	splx(s);
}

void
timer_queue_expire_local(
	__unused void                   *arg)
{
	rtclock_timer_t         *mytimer;
	uint64_t                        abstime;
	cpu_data_t                      *pp;

	pp = current_cpu_datap();

	mytimer = &pp->rtclock_timer;
	abstime = mach_absolute_time();

	mytimer->has_expired = TRUE;
	mytimer->deadline = timer_queue_expire(&mytimer->queue, abstime);
	mytimer->has_expired = FALSE;
	mytimer->when_set = mach_absolute_time();

	timer_resync_deadlines();
}

void
timer_queue_expire_rescan(
	__unused void                   *arg)
{
	rtclock_timer_t         *mytimer;
	uint64_t                abstime;
	cpu_data_t              *pp;

	assert(ml_get_interrupts_enabled() == FALSE);
	pp = current_cpu_datap();

	mytimer = &pp->rtclock_timer;
	abstime = mach_absolute_time();

	mytimer->has_expired = TRUE;
	mytimer->deadline = timer_queue_expire_with_options(&mytimer->queue, abstime, TRUE);
	mytimer->has_expired = FALSE;
	mytimer->when_set = mach_absolute_time();

	timer_resync_deadlines();
}

#define TIMER_RESORT_THRESHOLD_ABSTIME (50 * NSEC_PER_MSEC)

#if TCOAL_PRIO_STATS
int32_t nc_tcl, rt_tcl, bg_tcl, kt_tcl, fp_tcl, ts_tcl, qos_tcl;
#define TCOAL_PRIO_STAT(x) (x++)
#else
#define TCOAL_PRIO_STAT(x)
#endif

boolean_t
timer_resort_threshold(uint64_t skew)
{
	if (skew >= TIMER_RESORT_THRESHOLD_ABSTIME) {
		return TRUE;
	} else {
		return FALSE;
	}
}

/*
 * Return the local timer queue for a running processor
 * else return the boot processor's timer queue.
 */
mpqueue_head_t *
timer_queue_assign(
	uint64_t        deadline)
{
	cpu_data_t              *cdp = current_cpu_datap();
	mpqueue_head_t          *queue;

	if (cdp->cpu_running) {
		queue = &cdp->rtclock_timer.queue;

		if (deadline < cdp->rtclock_timer.deadline) {
			timer_set_deadline(deadline);
		}
	} else {
		queue = &cpu_datap(master_cpu)->rtclock_timer.queue;
	}

	return queue;
}

void
timer_queue_cancel(
	mpqueue_head_t  *queue,
	uint64_t        deadline,
	uint64_t        new_deadline)
{
	if (queue == &current_cpu_datap()->rtclock_timer.queue) {
		if (deadline < new_deadline) {
			timer_set_deadline(new_deadline);
		}
	}
}

/*
 * timer_queue_migrate_cpu() is called from the Power-Management kext
 * when a logical processor goes idle (in a deep C-state) with a distant
 * deadline so that it's timer queue can be moved to another processor.
 * This target processor should be the least idle (most busy) --
 * currently this is the primary processor for the calling thread's package.
 * Locking restrictions demand that the target cpu must be the boot cpu.
 */
uint32_t
timer_queue_migrate_cpu(int target_cpu)
{
	cpu_data_t      *target_cdp = cpu_datap(target_cpu);
	cpu_data_t      *cdp = current_cpu_datap();
	int             ntimers_moved;

	assert(!ml_get_interrupts_enabled());
	assert(target_cpu != cdp->cpu_number);
	assert(target_cpu == master_cpu);

	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
	    DECR_TIMER_MIGRATE | DBG_FUNC_START,
	    target_cpu,
	    cdp->rtclock_timer.deadline, (cdp->rtclock_timer.deadline >> 32),
	    0, 0);

	/*
	 * Move timer requests from the local queue to the target processor's.
	 * The return value is the number of requests moved. If this is 0,
	 * it indicates that the first (i.e. earliest) timer is earlier than
	 * the earliest for the target processor. Since this would force a
	 * resync, the move of this and all later requests is aborted.
	 */
	ntimers_moved = timer_queue_migrate(&cdp->rtclock_timer.queue,
	    &target_cdp->rtclock_timer.queue);

	/*
	 * Assuming we moved stuff, clear local deadline.
	 */
	if (ntimers_moved > 0) {
		cdp->rtclock_timer.deadline = EndOfAllTime;
		setPop(EndOfAllTime);
	}

	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
	    DECR_TIMER_MIGRATE | DBG_FUNC_END,
	    target_cpu, ntimers_moved, 0, 0, 0);

	return ntimers_moved;
}

mpqueue_head_t *
timer_queue_cpu(int cpu)
{
	return &cpu_datap(cpu)->rtclock_timer.queue;
}

void
timer_call_cpu(int cpu, void (*fn)(void *), void *arg)
{
	mp_cpus_call(cpu_to_cpumask(cpu), SYNC, fn, arg);
}

void
timer_call_nosync_cpu(int cpu, void (*fn)(void *), void *arg)
{
	/* XXX Needs error checking and retry */
	mp_cpus_call(cpu_to_cpumask(cpu), NOSYNC, fn, arg);
}


static timer_coalescing_priority_params_ns_t tcoal_prio_params_init =
{
	.idle_entry_timer_processing_hdeadline_threshold_ns = 5000ULL * NSEC_PER_USEC,
	.interrupt_timer_coalescing_ilat_threshold_ns = 30ULL * NSEC_PER_USEC,
	.timer_resort_threshold_ns = 50 * NSEC_PER_MSEC,
	.timer_coalesce_rt_shift = 0,
	.timer_coalesce_bg_shift = -5,
	.timer_coalesce_kt_shift = 3,
	.timer_coalesce_fp_shift = 3,
	.timer_coalesce_ts_shift = 3,
	.timer_coalesce_rt_ns_max = 0ULL,
	.timer_coalesce_bg_ns_max = 100 * NSEC_PER_MSEC,
	.timer_coalesce_kt_ns_max = 1 * NSEC_PER_MSEC,
	.timer_coalesce_fp_ns_max = 1 * NSEC_PER_MSEC,
	.timer_coalesce_ts_ns_max = 1 * NSEC_PER_MSEC,
	.latency_qos_scale = {3, 2, 1, -2, -15, -15},
	.latency_qos_ns_max = {1 * NSEC_PER_MSEC, 5 * NSEC_PER_MSEC, 20 * NSEC_PER_MSEC,
		               75 * NSEC_PER_MSEC, 10000 * NSEC_PER_MSEC, 10000 * NSEC_PER_MSEC},
	.latency_tier_rate_limited = {FALSE, FALSE, FALSE, FALSE, TRUE, TRUE},
};

timer_coalescing_priority_params_ns_t *
timer_call_get_priority_params(void)
{
	return &tcoal_prio_params_init;
}
Commit	Line	Data
39236c6e A	1	/*
	2	* Copyright (c) 2000-2008 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
0a7de745	5	*
39236c6e A	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
0a7de745	14	*
39236c6e A	15	* Please obtain a copy of the License at
39236c6e A	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
0a7de745	17	*
39236c6e A	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
0a7de745	25	*
39236c6e A	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28	/*
	29	* @OSF_COPYRIGHT@
	30	*/
	31	/*
	32	* @APPLE_FREE_COPYRIGHT@
	33	*/
	34	/*
	35	* File: timer.c
	36	* Purpose: Routines for handling the machine independent timer.
	37	*/
	38
	39	#include <mach/mach_types.h>
	40
	41	#include <kern/timer_queue.h>
	42	#include <kern/timer_call.h>
	43	#include <kern/clock.h>
	44	#include <kern/thread.h>
	45	#include <kern/processor.h>
	46	#include <kern/macro_help.h>
	47	#include <kern/spl.h>
	48	#include <kern/timer_queue.h>
	49	#include <kern/pms.h>
	50
	51	#include <machine/commpage.h>
	52	#include <machine/machine_routines.h>
	53
	54	#include <sys/kdebug.h>
	55	#include <i386/cpu_data.h>
	56	#include <i386/cpu_topology.h>
	57	#include <i386/cpu_threads.h>
	58
	59	uint32_t spurious_timers;
	60
	61	/*
0a7de745	62	* Event timer interrupt.
39236c6e A	63	*
	64	* XXX a drawback of this implementation is that events serviced earlier must not set deadlines
	65	* that occur before the entire chain completes.
	66	*
	67	* XXX a better implementation would use a set of generic callouts and iterate over them
	68	*/
	69	void
0a7de745 A	70	timer_intr(int user_mode,
0a7de745 A	71	uint64_t rip)
39236c6e	72	{
0a7de745 A	73	uint64_t abstime;
	74	rtclock_timer_t *mytimer;
	75	cpu_data_t *pp;
	76	int64_t latency;
	77	uint64_t pmdeadline;
	78	boolean_t timer_processed = FALSE;
39236c6e A	79
	80	pp = current_cpu_datap();
	81
	82	SCHED_STATS_TIMER_POP(current_processor());
	83
0a7de745	84	abstime = mach_absolute_time(); /* Get the time now */
39236c6e A	85
39236c6e A	86	/* has a pending clock timer expired? */
0a7de745	87	mytimer = &pp->rtclock_timer; /* Point to the event timer */
39236c6e A	88
39236c6e A	89	if ((timer_processed = ((mytimer->deadline <= abstime) \|\|
0a7de745	90	(abstime >= (mytimer->queue.earliest_soft_deadline))))) {
39236c6e A	91	/*
	92	* Log interrupt service latency (-ve value expected by tool)
	93	* a non-PM event is expected next.
0a7de745	94	* The requested deadline may be earlier than when it was set
39236c6e A	95	* - use MAX to avoid reporting bogus latencies.
	96	*/
	97	latency = (int64_t) (abstime - MAX(mytimer->deadline,
0a7de745	98	mytimer->when_set));
39236c6e A	99	/* Log zero timer latencies when opportunistically processing
	100	* coalesced timers.
	101	*/
	102	if (latency < 0) {
	103	TCOAL_DEBUG(0xEEEE0000, abstime, mytimer->queue.earliest_soft_deadline, abstime - mytimer->queue.earliest_soft_deadline, 0, 0);
	104	latency = 0;
	105	}
	106
	107	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
0a7de745 A	108	DECR_TRAP_LATENCY \| DBG_FUNC_NONE,
	109	-latency,
	110	((user_mode != 0) ? rip : VM_KERNEL_UNSLIDE(rip)),
	111	user_mode, 0, 0);
39236c6e	112
0a7de745	113	mytimer->has_expired = TRUE; /* Remember that we popped */
39236c6e A	114	mytimer->deadline = timer_queue_expire(&mytimer->queue, abstime);
	115	mytimer->has_expired = FALSE;
	116
	117	/* Get the time again since we ran a bit */
	118	abstime = mach_absolute_time();
	119	mytimer->when_set = abstime;
	120	}
	121
	122	/* is it time for power management state change? */
	123	if ((pmdeadline = pmCPUGetDeadline(pp)) && (pmdeadline <= abstime)) {
	124	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
0a7de745 A	125	DECR_PM_DEADLINE \| DBG_FUNC_START,
0a7de745 A	126	0, 0, 0, 0, 0);
39236c6e A	127	pmCPUDeadline(pp);
39236c6e A	128	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
0a7de745 A	129	DECR_PM_DEADLINE \| DBG_FUNC_END,
0a7de745 A	130	0, 0, 0, 0, 0);
39236c6e	131	timer_processed = TRUE;
5ba3f43e	132	abstime = mach_absolute_time(); /* Get the time again since we ran a bit */
39236c6e A	133	}
39236c6e A	134
5ba3f43e A	135	uint64_t quantum_deadline = pp->quantum_timer_deadline;
	136	/* is it the quantum timer expiration? */
	137	if ((quantum_deadline <= abstime) && (quantum_deadline > 0)) {
	138	pp->quantum_timer_deadline = 0;
	139	quantum_timer_expire(abstime);
	140	}
0a7de745	141
39236c6e A	142	/* schedule our next deadline */
	143	x86_lcpu()->rtcDeadline = EndOfAllTime;
	144	timer_resync_deadlines();
	145
0a7de745	146	if (__improbable(timer_processed == FALSE)) {
39236c6e	147	spurious_timers++;
0a7de745	148	}
39236c6e A	149	}
	150
	151	/*
	152	* Set the clock deadline.
	153	*/
0a7de745 A	154	void
0a7de745 A	155	timer_set_deadline(uint64_t deadline)
39236c6e	156	{
0a7de745 A	157	rtclock_timer_t *mytimer;
	158	spl_t s;
	159	cpu_data_t *pp;
39236c6e	160
0a7de745	161	s = splclock(); /* no interruptions */
39236c6e A	162	pp = current_cpu_datap();
39236c6e A	163
0a7de745 A	164	mytimer = &pp->rtclock_timer; /* Point to the timer itself */
0a7de745 A	165	mytimer->deadline = deadline; /* Set new expiration time */
39236c6e A	166	mytimer->when_set = mach_absolute_time();
	167
	168	timer_resync_deadlines();
	169
	170	splx(s);
	171	}
	172
5ba3f43e A	173	void
	174	quantum_timer_set_deadline(uint64_t deadline)
	175	{
0a7de745 A	176	cpu_data_t *pp;
	177	/* We should've only come into this path with interrupts disabled */
	178	assert(ml_get_interrupts_enabled() == FALSE);
5ba3f43e	179
0a7de745 A	180	pp = current_cpu_datap();
	181	pp->quantum_timer_deadline = deadline;
	182	timer_resync_deadlines();
5ba3f43e A	183	}
5ba3f43e A	184
39236c6e A	185	/*
	186	* Re-evaluate the outstanding deadlines and select the most proximate.
	187	*
	188	* Should be called at splclock.
	189	*/
	190	void
	191	timer_resync_deadlines(void)
	192	{
0a7de745 A	193	uint64_t deadline = EndOfAllTime;
	194	uint64_t pmdeadline;
	195	uint64_t quantum_deadline;
	196	rtclock_timer_t *mytimer;
	197	spl_t s = splclock();
	198	cpu_data_t *pp;
	199	uint32_t decr;
39236c6e A	200
39236c6e A	201	pp = current_cpu_datap();
0a7de745	202	if (!pp->cpu_running) {
39236c6e A	203	/* There's really nothing to do if this processor is down */
39236c6e A	204	return;
0a7de745	205	}
39236c6e A	206
	207	/*
	208	* If we have a clock timer set, pick that.
	209	*/
	210	mytimer = &pp->rtclock_timer;
	211	if (!mytimer->has_expired &&
0a7de745	212	0 < mytimer->deadline && mytimer->deadline < EndOfAllTime) {
39236c6e	213	deadline = mytimer->deadline;
0a7de745	214	}
39236c6e A	215
	216	/*
	217	* If we have a power management deadline, see if that's earlier.
	218	*/
	219	pmdeadline = pmCPUGetDeadline(pp);
0a7de745	220	if (0 < pmdeadline && pmdeadline < deadline) {
39236c6e	221	deadline = pmdeadline;
0a7de745	222	}
39236c6e	223
5ba3f43e A	224	/* If we have the quantum timer setup, check that */
5ba3f43e A	225	quantum_deadline = pp->quantum_timer_deadline;
0a7de745 A	226	if ((quantum_deadline > 0) &&
0a7de745 A	227	(quantum_deadline < deadline)) {
5ba3f43e	228	deadline = quantum_deadline;
0a7de745	229	}
5ba3f43e A	230
5ba3f43e A	231
39236c6e A	232	/*
	233	* Go and set the "pop" event.
	234	*/
	235	decr = (uint32_t) setPop(deadline);
	236
	237	/* Record non-PM deadline for latency tool */
	238	if (decr != 0 && deadline != pmdeadline) {
5ba3f43e A	239	uint64_t queue_count = 0;
5ba3f43e A	240	if (deadline != quantum_deadline) {
0a7de745	241	/*
5ba3f43e A	242	* For non-quantum timer put the queue count
	243	* in the tracepoint.
	244	*/
	245	queue_count = mytimer->queue.count;
	246	}
39236c6e	247	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
5ba3f43e A	248	DECR_SET_DEADLINE \| DBG_FUNC_NONE,
	249	decr, 2,
	250	deadline,
	251	queue_count, 0);
39236c6e A	252	}
	253	splx(s);
	254	}
	255
	256	void
	257	timer_queue_expire_local(
0a7de745	258	__unused void *arg)
39236c6e	259	{
0a7de745 A	260	rtclock_timer_t *mytimer;
	261	uint64_t abstime;
	262	cpu_data_t *pp;
39236c6e A	263
	264	pp = current_cpu_datap();
	265
	266	mytimer = &pp->rtclock_timer;
	267	abstime = mach_absolute_time();
	268
	269	mytimer->has_expired = TRUE;
	270	mytimer->deadline = timer_queue_expire(&mytimer->queue, abstime);
	271	mytimer->has_expired = FALSE;
	272	mytimer->when_set = mach_absolute_time();
	273
	274	timer_resync_deadlines();
	275	}
	276
	277	void
	278	timer_queue_expire_rescan(
0a7de745	279	__unused void *arg)
39236c6e	280	{
0a7de745 A	281	rtclock_timer_t *mytimer;
	282	uint64_t abstime;
	283	cpu_data_t *pp;
39236c6e A	284
	285	assert(ml_get_interrupts_enabled() == FALSE);
	286	pp = current_cpu_datap();
	287
	288	mytimer = &pp->rtclock_timer;
	289	abstime = mach_absolute_time();
	290
	291	mytimer->has_expired = TRUE;
	292	mytimer->deadline = timer_queue_expire_with_options(&mytimer->queue, abstime, TRUE);
	293	mytimer->has_expired = FALSE;
	294	mytimer->when_set = mach_absolute_time();
	295
	296	timer_resync_deadlines();
	297	}
	298
39236c6e A	299	#define TIMER_RESORT_THRESHOLD_ABSTIME (50 * NSEC_PER_MSEC)
	300
	301	#if TCOAL_PRIO_STATS
	302	int32_t nc_tcl, rt_tcl, bg_tcl, kt_tcl, fp_tcl, ts_tcl, qos_tcl;
	303	#define TCOAL_PRIO_STAT(x) (x++)
	304	#else
	305	#define TCOAL_PRIO_STAT(x)
	306	#endif
	307
39236c6e	308	boolean_t
0a7de745 A	309	timer_resort_threshold(uint64_t skew)
	310	{
	311	if (skew >= TIMER_RESORT_THRESHOLD_ABSTIME) {
39236c6e	312	return TRUE;
0a7de745	313	} else {
39236c6e	314	return FALSE;
0a7de745	315	}
39236c6e A	316	}
39236c6e A	317
39236c6e A	318	/*
	319	* Return the local timer queue for a running processor
	320	* else return the boot processor's timer queue.
	321	*/
	322	mpqueue_head_t *
	323	timer_queue_assign(
0a7de745	324	uint64_t deadline)
39236c6e	325	{
0a7de745 A	326	cpu_data_t *cdp = current_cpu_datap();
0a7de745 A	327	mpqueue_head_t *queue;
39236c6e A	328
	329	if (cdp->cpu_running) {
	330	queue = &cdp->rtclock_timer.queue;
	331
0a7de745	332	if (deadline < cdp->rtclock_timer.deadline) {
39236c6e	333	timer_set_deadline(deadline);
0a7de745 A	334	}
0a7de745 A	335	} else {
39236c6e	336	queue = &cpu_datap(master_cpu)->rtclock_timer.queue;
0a7de745	337	}
39236c6e	338
0a7de745	339	return queue;
39236c6e A	340	}
	341
	342	void
	343	timer_queue_cancel(
0a7de745 A	344	mpqueue_head_t *queue,
	345	uint64_t deadline,
	346	uint64_t new_deadline)
39236c6e	347	{
0a7de745 A	348	if (queue == &current_cpu_datap()->rtclock_timer.queue) {
	349	if (deadline < new_deadline) {
	350	timer_set_deadline(new_deadline);
	351	}
	352	}
39236c6e A	353	}
	354
	355	/*
	356	* timer_queue_migrate_cpu() is called from the Power-Management kext
	357	* when a logical processor goes idle (in a deep C-state) with a distant
	358	* deadline so that it's timer queue can be moved to another processor.
	359	* This target processor should be the least idle (most busy) --
	360	* currently this is the primary processor for the calling thread's package.
	361	* Locking restrictions demand that the target cpu must be the boot cpu.
	362	*/
	363	uint32_t
	364	timer_queue_migrate_cpu(int target_cpu)
	365	{
0a7de745 A	366	cpu_data_t *target_cdp = cpu_datap(target_cpu);
	367	cpu_data_t *cdp = current_cpu_datap();
	368	int ntimers_moved;
39236c6e A	369
	370	assert(!ml_get_interrupts_enabled());
	371	assert(target_cpu != cdp->cpu_number);
	372	assert(target_cpu == master_cpu);
	373
	374	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
0a7de745 A	375	DECR_TIMER_MIGRATE \| DBG_FUNC_START,
	376	target_cpu,
	377	cdp->rtclock_timer.deadline, (cdp->rtclock_timer.deadline >> 32),
	378	0, 0);
39236c6e A	379
	380	/*
	381	* Move timer requests from the local queue to the target processor's.
	382	* The return value is the number of requests moved. If this is 0,
	383	* it indicates that the first (i.e. earliest) timer is earlier than
	384	* the earliest for the target processor. Since this would force a
	385	* resync, the move of this and all later requests is aborted.
	386	*/
	387	ntimers_moved = timer_queue_migrate(&cdp->rtclock_timer.queue,
0a7de745	388	&target_cdp->rtclock_timer.queue);
39236c6e A	389
	390	/*
	391	* Assuming we moved stuff, clear local deadline.
	392	*/
	393	if (ntimers_moved > 0) {
	394	cdp->rtclock_timer.deadline = EndOfAllTime;
	395	setPop(EndOfAllTime);
	396	}
0a7de745	397
39236c6e	398	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
0a7de745 A	399	DECR_TIMER_MIGRATE \| DBG_FUNC_END,
0a7de745 A	400	target_cpu, ntimers_moved, 0, 0, 0);
39236c6e A	401
	402	return ntimers_moved;
	403	}
	404
	405	mpqueue_head_t *
	406	timer_queue_cpu(int cpu)
	407	{
	408	return &cpu_datap(cpu)->rtclock_timer.queue;
	409	}
	410
	411	void
	412	timer_call_cpu(int cpu, void (fn)(void ), void *arg)
	413	{
	414	mp_cpus_call(cpu_to_cpumask(cpu), SYNC, fn, arg);
	415	}
	416
	417	void
	418	timer_call_nosync_cpu(int cpu, void (fn)(void ), void *arg)
	419	{
	420	/* XXX Needs error checking and retry */
	421	mp_cpus_call(cpu_to_cpumask(cpu), NOSYNC, fn, arg);
	422	}
	423
fe8ab488 A	424
	425	static timer_coalescing_priority_params_ns_t tcoal_prio_params_init =
	426	{
	427	.idle_entry_timer_processing_hdeadline_threshold_ns = 5000ULL * NSEC_PER_USEC,
	428	.interrupt_timer_coalescing_ilat_threshold_ns = 30ULL * NSEC_PER_USEC,
	429	.timer_resort_threshold_ns = 50 * NSEC_PER_MSEC,
	430	.timer_coalesce_rt_shift = 0,
	431	.timer_coalesce_bg_shift = -5,
	432	.timer_coalesce_kt_shift = 3,
	433	.timer_coalesce_fp_shift = 3,
	434	.timer_coalesce_ts_shift = 3,
	435	.timer_coalesce_rt_ns_max = 0ULL,
	436	.timer_coalesce_bg_ns_max = 100 * NSEC_PER_MSEC,
	437	.timer_coalesce_kt_ns_max = 1 * NSEC_PER_MSEC,
	438	.timer_coalesce_fp_ns_max = 1 * NSEC_PER_MSEC,
	439	.timer_coalesce_ts_ns_max = 1 * NSEC_PER_MSEC,
	440	.latency_qos_scale = {3, 2, 1, -2, -15, -15},
0a7de745 A	441	.latency_qos_ns_max = {1 * NSEC_PER_MSEC, 5 * NSEC_PER_MSEC, 20 * NSEC_PER_MSEC,
0a7de745 A	442	75 * NSEC_PER_MSEC, 10000 * NSEC_PER_MSEC, 10000 * NSEC_PER_MSEC},
fe8ab488 A	443	.latency_tier_rate_limited = {FALSE, FALSE, FALSE, FALSE, TRUE, TRUE},
	444	};
	445
0a7de745 A	446	timer_coalescing_priority_params_ns_t *
0a7de745 A	447	timer_call_get_priority_params(void)
fe8ab488 A	448	{
	449	return &tcoal_prio_params_init;
	450	}