[apple/xnu.git] / osfmk / arm / rtclock.c

/*
 * Copyright (c) 2007 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
/*
 * @OSF_COPYRIGHT@
 */
/*
 * @APPLE_FREE_COPYRIGHT@
 */
/*
 * File: arm/rtclock.c
 * Purpose: Routines for handling the machine dependent
 *   real-time clock.
 */

#include <mach/mach_types.h>

#include <kern/clock.h>
#include <kern/thread.h>
#include <kern/macro_help.h>
#include <kern/spl.h>
#include <kern/timer_queue.h>

#include <kern/host_notify.h>

#include <machine/commpage.h>
#include <machine/machine_routines.h>
#include <machine/config.h>
#include <arm/exception.h>
#include <arm/cpu_data_internal.h>
#if __arm64__
#include <arm64/proc_reg.h>
#elif __arm__
#include <arm/proc_reg.h>
#else
#error Unsupported arch
#endif
#include <arm/rtclock.h>

#include <IOKit/IOPlatformExpert.h>
#include <libkern/OSAtomic.h>

#include <sys/kdebug.h>

#define MAX_TIMEBASE_TRIES 10

int rtclock_init(void);

static int
deadline_to_decrementer(uint64_t deadline,
    uint64_t now);
static void
timebase_callback(struct timebase_freq_t * freq);

#if DEVELOPMENT || DEBUG
uint32_t absolute_time_validation = 0;
#endif

/*
 * Configure the real-time clock device at boot
 */
void
rtclock_early_init(void)
{
	PE_register_timebase_callback(timebase_callback);
#if DEVELOPMENT || DEBUG
	uint32_t tmp_mv = 1;

#if defined(APPLE_ARM64_ARCH_FAMILY)
	/* Enable MAT validation on A0 hardware by default. */
	absolute_time_validation = ml_get_topology_info()->chip_revision == CPU_VERSION_A0;
#endif

	if (kern_feature_override(KF_MATV_OVRD)) {
		absolute_time_validation = 0;
	}
	if (PE_parse_boot_argn("timebase_validation", &tmp_mv, sizeof(tmp_mv))) {
		absolute_time_validation = tmp_mv;
	}
#endif
}

static void
timebase_callback(struct timebase_freq_t * freq)
{
	unsigned long numer, denom;
	uint64_t      t64_1, t64_2;
	uint32_t      divisor;

	if (freq->timebase_den < 1 || freq->timebase_den > 4 ||
	    freq->timebase_num < freq->timebase_den) {
		panic("rtclock timebase_callback: invalid constant %ld / %ld",
		    freq->timebase_num, freq->timebase_den);
	}

	denom = freq->timebase_num;
	numer = freq->timebase_den * NSEC_PER_SEC;
	// reduce by the greatest common denominator to minimize overflow
	if (numer > denom) {
		t64_1 = numer;
		t64_2 = denom;
	} else {
		t64_1 = denom;
		t64_2 = numer;
	}
	while (t64_2 != 0) {
		uint64_t temp = t64_2;
		t64_2 = t64_1 % t64_2;
		t64_1 = temp;
	}
	numer /= t64_1;
	denom /= t64_1;

	rtclock_timebase_const.numer = (uint32_t)numer;
	rtclock_timebase_const.denom = (uint32_t)denom;
	divisor = (uint32_t)(freq->timebase_num / freq->timebase_den);

	rtclock_sec_divisor = divisor;
	rtclock_usec_divisor = divisor / USEC_PER_SEC;
}

/*
 * Initialize the system clock device for the current cpu
 */
int
rtclock_init(void)
{
	uint64_t     abstime;
	cpu_data_t * cdp;

	clock_timebase_init();

	if (cpu_number() == master_cpu) {
		ml_init_lock_timeout();
	}

	cdp = getCpuDatap();

	abstime = mach_absolute_time();
	cdp->rtcPop = EndOfAllTime;                                     /* Init Pop time */
	timer_resync_deadlines();                                       /* Start the timers going */

	return 1;
}

uint64_t
mach_absolute_time(void)
{
#if DEVELOPMENT || DEBUG
	if (__improbable(absolute_time_validation == 1)) {
		static volatile uint64_t s_last_absolute_time = 0;
		uint64_t                 new_absolute_time, old_absolute_time;
		int                      attempts = 0;

		/* ARM 64: We need a dsb here to ensure that the load of s_last_absolute_time
		 * completes before the timebase read. Were the load to complete after the
		 * timebase read, there would be a window for another CPU to update
		 * s_last_absolute_time and leave us in an inconsistent state. Consider the
		 * following interleaving:
		 *
		 *   Let s_last_absolute_time = t0
		 *   CPU0: Read timebase at t1
		 *   CPU1: Read timebase at t2
		 *   CPU1: Update s_last_absolute_time to t2
		 *   CPU0: Load completes
		 *   CPU0: Update s_last_absolute_time to t1
		 *
		 * This would cause the assertion to fail even though time did not go
		 * backwards. Thus, we use a dsb to guarantee completion of the load before
		 * the timebase read.
		 */
		do {
			attempts++;
			old_absolute_time = s_last_absolute_time;

#if __arm64__
			__asm__ volatile ("dsb ld" ::: "memory");
#else
			OSSynchronizeIO(); // See osfmk/arm64/rtclock.c
#endif

			new_absolute_time = ml_get_timebase();
		} while (attempts < MAX_TIMEBASE_TRIES && !OSCompareAndSwap64(old_absolute_time, new_absolute_time, &s_last_absolute_time));

		if (attempts < MAX_TIMEBASE_TRIES && old_absolute_time > new_absolute_time) {
			panic("mach_absolute_time returning non-monotonically increasing value 0x%llx (old value 0x%llx\n)\n",
			    new_absolute_time, old_absolute_time);
		}
		return new_absolute_time;
	} else {
		return ml_get_timebase();
	}
#else
	return ml_get_timebase();
#endif
}

uint64_t
mach_approximate_time(void)
{
#if __ARM_TIME__ || __ARM_TIME_TIMEBASE_ONLY__ || __arm64__
	/* Hardware supports a fast timestamp, so grab it without asserting monotonicity */
	return ml_get_timebase();
#else
	processor_t processor;
	uint64_t    approx_time;

	disable_preemption();
	processor = current_processor();
	approx_time = processor->last_dispatch;
	enable_preemption();

	return approx_time;
#endif
}

void
clock_get_system_microtime(clock_sec_t *  secs,
    clock_usec_t * microsecs)
{
	absolutetime_to_microtime(mach_absolute_time(), secs, microsecs);
}

void
clock_get_system_nanotime(clock_sec_t *  secs,
    clock_nsec_t * nanosecs)
{
	uint64_t abstime;
	uint64_t t64;

	abstime = mach_absolute_time();
	*secs = (t64 = abstime / rtclock_sec_divisor);
	abstime -= (t64 * rtclock_sec_divisor);

	*nanosecs = (clock_nsec_t)((abstime * NSEC_PER_SEC) / rtclock_sec_divisor);
}

void
clock_gettimeofday_set_commpage(uint64_t abstime,
    uint64_t sec,
    uint64_t frac,
    uint64_t scale,
    uint64_t tick_per_sec)
{
	commpage_set_timestamp(abstime, sec, frac, scale, tick_per_sec);
}

void
clock_timebase_info(mach_timebase_info_t info)
{
	*info = rtclock_timebase_const;
}

/*
 * Real-time clock device interrupt.
 */
void
rtclock_intr(__unused unsigned int is_user_context)
{
	uint64_t                 abstime;
	cpu_data_t *             cdp;
	struct arm_saved_state * regs;
	unsigned int             user_mode;
	uintptr_t                pc;

	cdp = getCpuDatap();

	cdp->cpu_stat.timer_cnt++;
	SCHED_STATS_INC(timer_pop_count);

	assert(!ml_get_interrupts_enabled());

	abstime = mach_absolute_time();

	if (cdp->cpu_idle_pop != 0x0ULL) {
		if ((cdp->rtcPop - abstime) < cdp->cpu_idle_latency) {
			cdp->cpu_idle_pop = 0x0ULL;
			while (abstime < cdp->rtcPop) {
				abstime = mach_absolute_time();
			}
		} else {
			ClearIdlePop(FALSE);
		}
	}

	if ((regs = cdp->cpu_int_state)) {
		pc = get_saved_state_pc(regs);

#if __arm64__
		user_mode = PSR64_IS_USER(get_saved_state_cpsr(regs));
#else
		user_mode = (regs->cpsr & PSR_MODE_MASK) == PSR_USER_MODE;
#endif
	} else {
		pc = 0;
		user_mode = 0;
	}
	if (abstime >= cdp->rtcPop) {
		/* Log the interrupt service latency (-ve value expected by tool) */
		KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
		    MACHDBG_CODE(DBG_MACH_EXCP_DECI, 0) | DBG_FUNC_NONE,
		    -(abstime - cdp->rtcPop),
		    user_mode ? pc : VM_KERNEL_UNSLIDE(pc), user_mode, 0, 0);
	}

	/* call the generic etimer */
	timer_intr(user_mode, pc);
}

static int
deadline_to_decrementer(uint64_t deadline,
    uint64_t now)
{
	uint64_t delt;

	if (deadline <= now) {
		return DECREMENTER_MIN;
	} else {
		delt = deadline - now;

		return (delt >= (DECREMENTER_MAX + 1)) ? DECREMENTER_MAX : ((delt >= (DECREMENTER_MIN + 1)) ? (int)delt : DECREMENTER_MIN);
	}
}

/*
 *	Request a decrementer pop
 */
int
setPop(uint64_t time)
{
	int          delay_time;
	uint64_t     current_time;
	cpu_data_t * cdp;

	cdp = getCpuDatap();
	current_time = mach_absolute_time();

	delay_time = deadline_to_decrementer(time, current_time);
	cdp->rtcPop = delay_time + current_time;

	ml_set_decrementer((uint32_t) delay_time);

	return delay_time;
}

/*
 *	Request decrementer Idle Pop. Return true if set
 */
boolean_t
SetIdlePop(void)
{
	int          delay_time;
	uint64_t     time;
	uint64_t     current_time;
	cpu_data_t * cdp;

	cdp = getCpuDatap();
	current_time = mach_absolute_time();

	if (((cdp->rtcPop < current_time) ||
	    (cdp->rtcPop - current_time) < cdp->cpu_idle_latency)) {
		return FALSE;
	}

	time = cdp->rtcPop - cdp->cpu_idle_latency;

	delay_time = deadline_to_decrementer(time, current_time);
	cdp->cpu_idle_pop = delay_time + current_time;
	ml_set_decrementer((uint32_t) delay_time);

	return TRUE;
}

/*
 *	Clear decrementer Idle Pop
 */
void
ClearIdlePop(
	boolean_t wfi)
{
#if !__arm64__
#pragma unused(wfi)
#endif
	cpu_data_t * cdp;

	cdp = getCpuDatap();
	cdp->cpu_idle_pop = 0x0ULL;

#if __arm64__
	/*
	 * Don't update the HW timer if there's a pending
	 * interrupt (we can lose interrupt assertion);
	 * we want to take the interrupt right now and update
	 * the deadline from the handler).
	 *
	 * ARM64_TODO: consider this more carefully.
	 */
	if (!(wfi && ml_get_timer_pending()))
#endif
	{
		setPop(cdp->rtcPop);
	}
}

void
absolutetime_to_microtime(uint64_t       abstime,
    clock_sec_t *  secs,
    clock_usec_t * microsecs)
{
	uint64_t t64;

	*secs = t64 = abstime / rtclock_sec_divisor;
	abstime -= (t64 * rtclock_sec_divisor);

	*microsecs = (uint32_t)(abstime / rtclock_usec_divisor);
}

void
absolutetime_to_nanoseconds(uint64_t   abstime,
    uint64_t * result)
{
	uint64_t        t64;

	*result = (t64 = abstime / rtclock_sec_divisor) * NSEC_PER_SEC;
	abstime -= (t64 * rtclock_sec_divisor);
	*result += (abstime * NSEC_PER_SEC) / rtclock_sec_divisor;
}

void
nanoseconds_to_absolutetime(uint64_t   nanosecs,
    uint64_t * result)
{
	uint64_t        t64;

	*result = (t64 = nanosecs / NSEC_PER_SEC) * rtclock_sec_divisor;
	nanosecs -= (t64 * NSEC_PER_SEC);
	*result += (nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC;
}

void
nanotime_to_absolutetime(clock_sec_t  secs,
    clock_nsec_t nanosecs,
    uint64_t *   result)
{
	*result = ((uint64_t) secs * rtclock_sec_divisor) +
	    ((uint64_t) nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC;
}

void
clock_interval_to_absolutetime_interval(uint32_t   interval,
    uint32_t   scale_factor,
    uint64_t * result)
{
	uint64_t nanosecs = (uint64_t) interval * scale_factor;
	uint64_t t64;

	*result = (t64 = nanosecs / NSEC_PER_SEC) * rtclock_sec_divisor;
	nanosecs -= (t64 * NSEC_PER_SEC);
	*result += (nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC;
}

void
machine_delay_until(uint64_t interval,
    uint64_t deadline)
{
#pragma unused(interval)
	uint64_t now;

	do {
#if     __ARM_ENABLE_WFE_
		__builtin_arm_wfe();
#endif /* __ARM_ENABLE_WFE_ */

		now = mach_absolute_time();
	} while (now < deadline);
}
Commit	Line	Data
5ba3f43e A	1	/*
	2	* Copyright (c) 2007 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28	/*
	29	* @OSF_COPYRIGHT@
	30	*/
	31	/*
	32	* @APPLE_FREE_COPYRIGHT@
	33	*/
	34	/*
	35	* File: arm/rtclock.c
	36	* Purpose: Routines for handling the machine dependent
	37	* real-time clock.
	38	*/
	39
	40	#include <mach/mach_types.h>
	41
	42	#include <kern/clock.h>
	43	#include <kern/thread.h>
	44	#include <kern/macro_help.h>
	45	#include <kern/spl.h>
	46	#include <kern/timer_queue.h>
	47
	48	#include <kern/host_notify.h>
	49
	50	#include <machine/commpage.h>
	51	#include <machine/machine_routines.h>
f427ee49	52	#include <machine/config.h>
5ba3f43e A	53	#include <arm/exception.h>
	54	#include <arm/cpu_data_internal.h>
	55	#if __arm64__
	56	#include <arm64/proc_reg.h>
	57	#elif __arm__
	58	#include <arm/proc_reg.h>
	59	#else
	60	#error Unsupported arch
	61	#endif
	62	#include <arm/rtclock.h>
	63
	64	#include <IOKit/IOPlatformExpert.h>
	65	#include <libkern/OSAtomic.h>
	66
	67	#include <sys/kdebug.h>
	68
	69	#define MAX_TIMEBASE_TRIES 10
	70
	71	int rtclock_init(void);
	72
	73	static int
	74	deadline_to_decrementer(uint64_t deadline,
0a7de745	75	uint64_t now);
5ba3f43e A	76	static void
	77	timebase_callback(struct timebase_freq_t * freq);
	78
	79	#if DEVELOPMENT \|\| DEBUG
0a7de745	80	uint32_t absolute_time_validation = 0;
5ba3f43e A	81	#endif
	82
	83	/*
	84	* Configure the real-time clock device at boot
	85	*/
	86	void
	87	rtclock_early_init(void)
	88	{
	89	PE_register_timebase_callback(timebase_callback);
	90	#if DEVELOPMENT \|\| DEBUG
	91	uint32_t tmp_mv = 1;
0a7de745	92
f427ee49	93	#if defined(APPLE_ARM64_ARCH_FAMILY)
0a7de745	94	/* Enable MAT validation on A0 hardware by default. */
f427ee49 A	95	absolute_time_validation = ml_get_topology_info()->chip_revision == CPU_VERSION_A0;
f427ee49 A	96	#endif
0a7de745	97
5ba3f43e A	98	if (kern_feature_override(KF_MATV_OVRD)) {
	99	absolute_time_validation = 0;
	100	}
	101	if (PE_parse_boot_argn("timebase_validation", &tmp_mv, sizeof(tmp_mv))) {
0a7de745	102	absolute_time_validation = tmp_mv;
5ba3f43e A	103	}
	104	#endif
	105	}
	106
	107	static void
	108	timebase_callback(struct timebase_freq_t * freq)
	109	{
	110	unsigned long numer, denom;
	111	uint64_t t64_1, t64_2;
	112	uint32_t divisor;
	113
	114	if (freq->timebase_den < 1 \|\| freq->timebase_den > 4 \|\|
0a7de745	115	freq->timebase_num < freq->timebase_den) {
5ba3f43e	116	panic("rtclock timebase_callback: invalid constant %ld / %ld",
0a7de745 A	117	freq->timebase_num, freq->timebase_den);
0a7de745 A	118	}
5ba3f43e A	119
	120	denom = freq->timebase_num;
	121	numer = freq->timebase_den * NSEC_PER_SEC;
	122	// reduce by the greatest common denominator to minimize overflow
	123	if (numer > denom) {
	124	t64_1 = numer;
	125	t64_2 = denom;
	126	} else {
	127	t64_1 = denom;
	128	t64_2 = numer;
	129	}
	130	while (t64_2 != 0) {
	131	uint64_t temp = t64_2;
	132	t64_2 = t64_1 % t64_2;
	133	t64_1 = temp;
	134	}
	135	numer /= t64_1;
	136	denom /= t64_1;
	137
	138	rtclock_timebase_const.numer = (uint32_t)numer;
	139	rtclock_timebase_const.denom = (uint32_t)denom;
	140	divisor = (uint32_t)(freq->timebase_num / freq->timebase_den);
	141
	142	rtclock_sec_divisor = divisor;
	143	rtclock_usec_divisor = divisor / USEC_PER_SEC;
	144	}
	145
	146	/*
	147	* Initialize the system clock device for the current cpu
	148	*/
	149	int
	150	rtclock_init(void)
	151	{
	152	uint64_t abstime;
	153	cpu_data_t * cdp;
	154
	155	clock_timebase_init();
c3c9b80d A	156
	157	if (cpu_number() == master_cpu) {
	158	ml_init_lock_timeout();
	159	}
5ba3f43e A	160
	161	cdp = getCpuDatap();
	162
	163	abstime = mach_absolute_time();
0a7de745 A	164	cdp->rtcPop = EndOfAllTime; /* Init Pop time */
0a7de745 A	165	timer_resync_deadlines(); /* Start the timers going */
5ba3f43e	166
0a7de745	167	return 1;
5ba3f43e A	168	}
	169
	170	uint64_t
	171	mach_absolute_time(void)
	172	{
	173	#if DEVELOPMENT \|\| DEBUG
	174	if (__improbable(absolute_time_validation == 1)) {
	175	static volatile uint64_t s_last_absolute_time = 0;
	176	uint64_t new_absolute_time, old_absolute_time;
	177	int attempts = 0;
	178
	179	/* ARM 64: We need a dsb here to ensure that the load of s_last_absolute_time
	180	* completes before the timebase read. Were the load to complete after the
	181	* timebase read, there would be a window for another CPU to update
	182	* s_last_absolute_time and leave us in an inconsistent state. Consider the
	183	* following interleaving:
	184	*
	185	* Let s_last_absolute_time = t0
	186	* CPU0: Read timebase at t1
	187	* CPU1: Read timebase at t2
	188	* CPU1: Update s_last_absolute_time to t2
	189	* CPU0: Load completes
	190	* CPU0: Update s_last_absolute_time to t1
	191	*
	192	* This would cause the assertion to fail even though time did not go
	193	* backwards. Thus, we use a dsb to guarantee completion of the load before
	194	* the timebase read.
	195	*/
	196	do {
	197	attempts++;
	198	old_absolute_time = s_last_absolute_time;
	199
	200	#if __arm64__
0a7de745	201	__asm__ volatile ("dsb ld" ::: "memory");
5ba3f43e A	202	#else
	203	OSSynchronizeIO(); // See osfmk/arm64/rtclock.c
	204	#endif
	205
	206	new_absolute_time = ml_get_timebase();
	207	} while (attempts < MAX_TIMEBASE_TRIES && !OSCompareAndSwap64(old_absolute_time, new_absolute_time, &s_last_absolute_time));
	208
	209	if (attempts < MAX_TIMEBASE_TRIES && old_absolute_time > new_absolute_time) {
	210	panic("mach_absolute_time returning non-monotonically increasing value 0x%llx (old value 0x%llx\n)\n",
	211	new_absolute_time, old_absolute_time);
	212	}
	213	return new_absolute_time;
	214	} else {
	215	return ml_get_timebase();
	216	}
	217	#else
	218	return ml_get_timebase();
	219	#endif
	220	}
	221
	222	uint64_t
	223	mach_approximate_time(void)
	224	{
	225	#if __ARM_TIME__ \|\| __ARM_TIME_TIMEBASE_ONLY__ \|\| __arm64__
	226	/* Hardware supports a fast timestamp, so grab it without asserting monotonicity */
	227	return ml_get_timebase();
	228	#else
	229	processor_t processor;
	230	uint64_t approx_time;
	231
	232	disable_preemption();
	233	processor = current_processor();
	234	approx_time = processor->last_dispatch;
	235	enable_preemption();
	236
	237	return approx_time;
	238	#endif
	239	}
	240
	241	void
	242	clock_get_system_microtime(clock_sec_t * secs,
0a7de745	243	clock_usec_t * microsecs)
5ba3f43e A	244	{
	245	absolutetime_to_microtime(mach_absolute_time(), secs, microsecs);
	246	}
	247
	248	void
	249	clock_get_system_nanotime(clock_sec_t * secs,
0a7de745	250	clock_nsec_t * nanosecs)
5ba3f43e A	251	{
	252	uint64_t abstime;
	253	uint64_t t64;
	254
	255	abstime = mach_absolute_time();
	256	*secs = (t64 = abstime / rtclock_sec_divisor);
	257	abstime -= (t64 * rtclock_sec_divisor);
	258
	259	nanosecs = (clock_nsec_t)((abstime NSEC_PER_SEC) / rtclock_sec_divisor);
	260	}
	261
	262	void
	263	clock_gettimeofday_set_commpage(uint64_t abstime,
0a7de745 A	264	uint64_t sec,
	265	uint64_t frac,
	266	uint64_t scale,
	267	uint64_t tick_per_sec)
5ba3f43e A	268	{
	269	commpage_set_timestamp(abstime, sec, frac, scale, tick_per_sec);
	270	}
	271
	272	void
	273	clock_timebase_info(mach_timebase_info_t info)
	274	{
	275	*info = rtclock_timebase_const;
	276	}
	277
	278	/*
	279	* Real-time clock device interrupt.
	280	*/
	281	void
	282	rtclock_intr(__unused unsigned int is_user_context)
	283	{
	284	uint64_t abstime;
	285	cpu_data_t * cdp;
	286	struct arm_saved_state * regs;
	287	unsigned int user_mode;
	288	uintptr_t pc;
	289
	290	cdp = getCpuDatap();
	291
	292	cdp->cpu_stat.timer_cnt++;
f427ee49	293	SCHED_STATS_INC(timer_pop_count);
5ba3f43e A	294
	295	assert(!ml_get_interrupts_enabled());
	296
	297	abstime = mach_absolute_time();
	298
	299	if (cdp->cpu_idle_pop != 0x0ULL) {
0a7de745	300	if ((cdp->rtcPop - abstime) < cdp->cpu_idle_latency) {
5ba3f43e	301	cdp->cpu_idle_pop = 0x0ULL;
0a7de745	302	while (abstime < cdp->rtcPop) {
5ba3f43e	303	abstime = mach_absolute_time();
0a7de745	304	}
5ba3f43e A	305	} else {
	306	ClearIdlePop(FALSE);
	307	}
	308	}
	309
	310	if ((regs = cdp->cpu_int_state)) {
	311	pc = get_saved_state_pc(regs);
	312
	313	#if __arm64__
	314	user_mode = PSR64_IS_USER(get_saved_state_cpsr(regs));
	315	#else
	316	user_mode = (regs->cpsr & PSR_MODE_MASK) == PSR_USER_MODE;
	317	#endif
	318	} else {
	319	pc = 0;
	320	user_mode = 0;
	321	}
	322	if (abstime >= cdp->rtcPop) {
	323	/* Log the interrupt service latency (-ve value expected by tool) */
	324	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
0a7de745 A	325	MACHDBG_CODE(DBG_MACH_EXCP_DECI, 0) \| DBG_FUNC_NONE,
	326	-(abstime - cdp->rtcPop),
	327	user_mode ? pc : VM_KERNEL_UNSLIDE(pc), user_mode, 0, 0);
5ba3f43e A	328	}
	329
	330	/* call the generic etimer */
	331	timer_intr(user_mode, pc);
	332	}
	333
	334	static int
	335	deadline_to_decrementer(uint64_t deadline,
0a7de745	336	uint64_t now)
5ba3f43e A	337	{
	338	uint64_t delt;
	339
0a7de745	340	if (deadline <= now) {
5ba3f43e	341	return DECREMENTER_MIN;
0a7de745	342	} else {
5ba3f43e A	343	delt = deadline - now;
	344
	345	return (delt >= (DECREMENTER_MAX + 1)) ? DECREMENTER_MAX : ((delt >= (DECREMENTER_MIN + 1)) ? (int)delt : DECREMENTER_MIN);
	346	}
	347	}
	348
	349	/*
	350	* Request a decrementer pop
	351	*/
	352	int
	353	setPop(uint64_t time)
	354	{
	355	int delay_time;
	356	uint64_t current_time;
	357	cpu_data_t * cdp;
	358
	359	cdp = getCpuDatap();
	360	current_time = mach_absolute_time();
	361
	362	delay_time = deadline_to_decrementer(time, current_time);
	363	cdp->rtcPop = delay_time + current_time;
	364
	365	ml_set_decrementer((uint32_t) delay_time);
	366
0a7de745	367	return delay_time;
5ba3f43e A	368	}
	369
	370	/*
	371	* Request decrementer Idle Pop. Return true if set
	372	*/
	373	boolean_t
	374	SetIdlePop(void)
	375	{
	376	int delay_time;
	377	uint64_t time;
	378	uint64_t current_time;
	379	cpu_data_t * cdp;
	380
	381	cdp = getCpuDatap();
	382	current_time = mach_absolute_time();
	383
	384	if (((cdp->rtcPop < current_time) \|\|
0a7de745	385	(cdp->rtcPop - current_time) < cdp->cpu_idle_latency)) {
5ba3f43e	386	return FALSE;
0a7de745	387	}
5ba3f43e A	388
	389	time = cdp->rtcPop - cdp->cpu_idle_latency;
	390
	391	delay_time = deadline_to_decrementer(time, current_time);
	392	cdp->cpu_idle_pop = delay_time + current_time;
	393	ml_set_decrementer((uint32_t) delay_time);
	394
	395	return TRUE;
	396	}
	397
	398	/*
	399	* Clear decrementer Idle Pop
	400	*/
	401	void
	402	ClearIdlePop(
0a7de745	403	boolean_t wfi)
5ba3f43e A	404	{
	405	#if !__arm64__
	406	#pragma unused(wfi)
	407	#endif
	408	cpu_data_t * cdp;
	409
	410	cdp = getCpuDatap();
	411	cdp->cpu_idle_pop = 0x0ULL;
	412
	413	#if __arm64__
	414	/*
	415	* Don't update the HW timer if there's a pending
	416	* interrupt (we can lose interrupt assertion);
	417	* we want to take the interrupt right now and update
	418	* the deadline from the handler).
	419	*
	420	* ARM64_TODO: consider this more carefully.
	421	*/
	422	if (!(wfi && ml_get_timer_pending()))
	423	#endif
	424	{
	425	setPop(cdp->rtcPop);
	426	}
	427	}
	428
	429	void
	430	absolutetime_to_microtime(uint64_t abstime,
0a7de745 A	431	clock_sec_t * secs,
0a7de745 A	432	clock_usec_t * microsecs)
5ba3f43e A	433	{
	434	uint64_t t64;
	435
	436	*secs = t64 = abstime / rtclock_sec_divisor;
	437	abstime -= (t64 * rtclock_sec_divisor);
	438
	439	*microsecs = (uint32_t)(abstime / rtclock_usec_divisor);
	440	}
	441
	442	void
	443	absolutetime_to_nanoseconds(uint64_t abstime,
0a7de745	444	uint64_t * result)
5ba3f43e A	445	{
	446	uint64_t t64;
	447
	448	result = (t64 = abstime / rtclock_sec_divisor) NSEC_PER_SEC;
	449	abstime -= (t64 * rtclock_sec_divisor);
	450	result += (abstime NSEC_PER_SEC) / rtclock_sec_divisor;
	451	}
	452
	453	void
	454	nanoseconds_to_absolutetime(uint64_t nanosecs,
0a7de745	455	uint64_t * result)
5ba3f43e A	456	{
	457	uint64_t t64;
	458
	459	result = (t64 = nanosecs / NSEC_PER_SEC) rtclock_sec_divisor;
	460	nanosecs -= (t64 * NSEC_PER_SEC);
	461	result += (nanosecs rtclock_sec_divisor) / NSEC_PER_SEC;
	462	}
	463
	464	void
	465	nanotime_to_absolutetime(clock_sec_t secs,
0a7de745 A	466	clock_nsec_t nanosecs,
0a7de745 A	467	uint64_t * result)
5ba3f43e A	468	{
5ba3f43e A	469	result = ((uint64_t) secs rtclock_sec_divisor) +
0a7de745	470	((uint64_t) nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC;
5ba3f43e A	471	}
	472
	473	void
	474	clock_interval_to_absolutetime_interval(uint32_t interval,
0a7de745 A	475	uint32_t scale_factor,
0a7de745 A	476	uint64_t * result)
5ba3f43e A	477	{
	478	uint64_t nanosecs = (uint64_t) interval * scale_factor;
	479	uint64_t t64;
	480
	481	result = (t64 = nanosecs / NSEC_PER_SEC) rtclock_sec_divisor;
	482	nanosecs -= (t64 * NSEC_PER_SEC);
	483	result += (nanosecs rtclock_sec_divisor) / NSEC_PER_SEC;
	484	}
	485
	486	void
	487	machine_delay_until(uint64_t interval,
0a7de745	488	uint64_t deadline)
5ba3f43e A	489	{
	490	#pragma unused(interval)
	491	uint64_t now;
	492
	493	do {
0a7de745	494	#if __ARM_ENABLE_WFE_
f427ee49	495	__builtin_arm_wfe();
5ba3f43e A	496	#endif /* __ARM_ENABLE_WFE_ */
	497
	498	now = mach_absolute_time();
	499	} while (now < deadline);
	500	}