[apple/xnu.git] / libsyscall / wrappers / mach_continuous_time.c

/*
 * Copyright (c) 2015 Apple Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */
#include <stddef.h>
#include <stdbool.h>
#include <sys/types.h>
#include <machine/cpu_capabilities.h>
#include <mach/mach_time.h>

__attribute__((visibility("hidden")))
uint64_t
_mach_continuous_time_base(void)
{
#if !defined(__x86_64__) && !defined(__arm64__)
	// Deal with the lack of 64-bit loads on arm32 (see mach_approximate_time.s)
	while(1) {
		volatile uint64_t *base_ptr = (volatile uint64_t*)_COMM_PAGE_CONT_TIMEBASE;
		uint64_t read1, read2;
		read1 = *base_ptr;
#if defined(__arm__)
		__asm__ volatile("dsb sy" ::: "memory");
#elif defined(__i386__)
		__asm__ volatile("lfence" ::: "memory");
#else
#error "unsupported arch"
#endif
		read2 = *base_ptr;

		if(__builtin_expect((read1 == read2), 1))
			return read1;
	}
#else // 64-bit
	return *(volatile uint64_t*)_COMM_PAGE_CONT_TIMEBASE;
#endif // 64-bit
}

__attribute__((visibility("hidden")))
kern_return_t
_mach_continuous_hwclock(uint64_t *cont_time __unused)
{
#if defined(__arm64__)
	uint8_t cont_hwclock = *((uint8_t*)_COMM_PAGE_CONT_HWCLOCK);
	uint64_t timebase;
	if (cont_hwclock) {
		__asm__ volatile("isb\n" "mrs %0, CNTPCT_EL0" : "=r"(timebase));
		*cont_time = timebase;
		return KERN_SUCCESS;
	}
#endif
	return KERN_NOT_SUPPORTED;
}

__attribute__((visibility("hidden")))
kern_return_t
_mach_continuous_time(uint64_t* absolute_time, uint64_t* cont_time)
{
	volatile uint64_t *base_ptr = (volatile uint64_t*)_COMM_PAGE_CONT_TIMEBASE;
	volatile uint64_t read1, read2;
	volatile uint64_t absolute;

	do {
        read1 = *base_ptr;
        absolute = mach_absolute_time();
#if	defined(__arm__) || defined(__arm64__)
            /*
             * mach_absolute_time() contains an instruction barrier which will
             * prevent the speculation of read2 above this point, so we don't
             * need another barrier here.
             */
#endif
		read2 = *base_ptr;
	} while (__builtin_expect((read1 != read2), 0));

	if (absolute_time) *absolute_time = absolute;
	if (cont_time) *cont_time = absolute + read1;

	return KERN_SUCCESS;
}

uint64_t
mach_continuous_time(void)
{
	uint64_t cont_time;
	if (_mach_continuous_hwclock(&cont_time) != KERN_SUCCESS)
		_mach_continuous_time(NULL, &cont_time);
	return cont_time;
}

uint64_t
mach_continuous_approximate_time(void)
{
	/*
	 * No retry loop here because if we use a slightly too old timebase that's
	 * okay, we are approximate time anyway.
	*/
	volatile register uint64_t time_base = _mach_continuous_time_base();
	return time_base + mach_approximate_time();
}
Commit	Line	Data
39037602 A	1	/*
	2	* Copyright (c) 2015 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. Please obtain a copy of the License at
	10	* http://www.opensource.apple.com/apsl/ and read it before using this
	11	* file.
	12	*
	13	* The Original Code and all software distributed under the License are
	14	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	15	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	16	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	17	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	18	* Please see the License for the specific language governing rights and
	19	* limitations under the License.
	20	*
	21	* @APPLE_LICENSE_HEADER_END@
	22	*/
	23	#include <stddef.h>
	24	#include <stdbool.h>
	25	#include <sys/types.h>
	26	#include <machine/cpu_capabilities.h>
	27	#include <mach/mach_time.h>
	28
	29	__attribute__((visibility("hidden")))
	30	uint64_t
	31	_mach_continuous_time_base(void)
	32	{
	33	#if !defined(__x86_64__) && !defined(__arm64__)
	34	// Deal with the lack of 64-bit loads on arm32 (see mach_approximate_time.s)
	35	while(1) {
	36	volatile uint64_t base_ptr = (volatile uint64_t)_COMM_PAGE_CONT_TIMEBASE;
	37	uint64_t read1, read2;
	38	read1 = *base_ptr;
5ba3f43e A	39	#if defined(__arm__)
	40	__asm__ volatile("dsb sy" ::: "memory");
	41	#elif defined(__i386__)
39037602 A	42	__asm__ volatile("lfence" ::: "memory");
	43	#else
	44	#error "unsupported arch"
	45	#endif
	46	read2 = *base_ptr;
	47
	48	if(__builtin_expect((read1 == read2), 1))
	49	return read1;
	50	}
	51	#else // 64-bit
	52	return (volatile uint64_t)_COMM_PAGE_CONT_TIMEBASE;
	53	#endif // 64-bit
	54	}
	55
5ba3f43e A	56	__attribute__((visibility("hidden")))
	57	kern_return_t
	58	_mach_continuous_hwclock(uint64_t *cont_time __unused)
	59	{
	60	#if defined(__arm64__)
	61	uint8_t cont_hwclock = ((uint8_t)_COMM_PAGE_CONT_HWCLOCK);
	62	uint64_t timebase;
	63	if (cont_hwclock) {
	64	__asm__ volatile("isb\n" "mrs %0, CNTPCT_EL0" : "=r"(timebase));
	65	*cont_time = timebase;
	66	return KERN_SUCCESS;
	67	}
	68	#endif
	69	return KERN_NOT_SUPPORTED;
	70	}
39037602 A	71
	72	__attribute__((visibility("hidden")))
	73	kern_return_t
	74	_mach_continuous_time(uint64_t* absolute_time, uint64_t* cont_time)
	75	{
	76	volatile uint64_t base_ptr = (volatile uint64_t)_COMM_PAGE_CONT_TIMEBASE;
	77	volatile uint64_t read1, read2;
	78	volatile uint64_t absolute;
	79
	80	do {
	81	read1 = *base_ptr;
	82	absolute = mach_absolute_time();
5ba3f43e A	83	#if defined(__arm__) \|\| defined(__arm64__)
	84	/*
	85	* mach_absolute_time() contains an instruction barrier which will
	86	* prevent the speculation of read2 above this point, so we don't
	87	* need another barrier here.
	88	*/
	89	#endif
39037602 A	90	read2 = *base_ptr;
	91	} while (__builtin_expect((read1 != read2), 0));
	92
	93	if (absolute_time) *absolute_time = absolute;
	94	if (cont_time) *cont_time = absolute + read1;
	95
	96	return KERN_SUCCESS;
	97	}
	98
	99	uint64_t
	100	mach_continuous_time(void)
	101	{
	102	uint64_t cont_time;
5ba3f43e A	103	if (_mach_continuous_hwclock(&cont_time) != KERN_SUCCESS)
5ba3f43e A	104	_mach_continuous_time(NULL, &cont_time);
39037602 A	105	return cont_time;
	106	}
	107
	108	uint64_t
	109	mach_continuous_approximate_time(void)
	110	{
5ba3f43e A	111	/*
	112	* No retry loop here because if we use a slightly too old timebase that's
	113	* okay, we are approximate time anyway.
	114	*/
	115	volatile register uint64_t time_base = _mach_continuous_time_base();
	116	return time_base + mach_approximate_time();
39037602	117	}