[apple/xnu.git] / osfmk / arm / commpage / commpage.c

/*
 * Copyright (c) 2007 Apple Inc. All rights reserved.
 * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
/*
 * @OSF_COPYRIGHT@
 */
/*
 * @APPLE_FREE_COPYRIGHT@
 */
/*
 *	File:		arm/commpage/commpage.c
 *	Purpose:	Set up and export a RO/RW page
 */
#include <libkern/section_keywords.h>
#include <mach/mach_types.h>
#include <mach/machine.h>
#include <mach/vm_map.h>
#include <machine/cpu_capabilities.h>
#include <machine/commpage.h>
#include <machine/pmap.h>
#include <vm/vm_kern.h>
#include <vm/vm_map.h>
#include <vm/vm_protos.h>
#include <ipc/ipc_port.h>
#include <arm/cpuid.h>		/* for cpuid_info() & cache_info() */
#include <arm/rtclock.h>
#include <libkern/OSAtomic.h>
#include <stdatomic.h>

#include <sys/kdebug.h>

#if CONFIG_ATM
#include <atm/atm_internal.h>
#endif

static void commpage_init_cpu_capabilities( void );
static int commpage_cpus( void );

SECURITY_READ_ONLY_LATE(vm_address_t)	commPagePtr=0;
SECURITY_READ_ONLY_LATE(vm_address_t)	sharedpage_rw_addr = 0;
SECURITY_READ_ONLY_LATE(uint32_t)	_cpu_capabilities = 0;

/* For sysctl access from BSD side */
extern int	gARMv81Atomics;
extern int	gARMv8Crc32;

void
commpage_populate(
	void)
{
	uint16_t	c2;
	int cpufamily;

	sharedpage_rw_addr = pmap_create_sharedpage();
	commPagePtr = (vm_address_t)_COMM_PAGE_BASE_ADDRESS;

	*((uint16_t*)(_COMM_PAGE_VERSION+_COMM_PAGE_RW_OFFSET)) = (uint16_t) _COMM_PAGE_THIS_VERSION;

	commpage_init_cpu_capabilities();
	commpage_set_timestamp(0, 0, 0, 0, 0);

	if (_cpu_capabilities & kCache32)
		c2 = 32;
	else if (_cpu_capabilities & kCache64)
		c2 = 64;
	else if (_cpu_capabilities & kCache128)
		c2 = 128;
	else
		c2 = 0;

	*((uint16_t*)(_COMM_PAGE_CACHE_LINESIZE+_COMM_PAGE_RW_OFFSET)) = c2;
	*((uint32_t*)(_COMM_PAGE_SPIN_COUNT+_COMM_PAGE_RW_OFFSET)) = 1;

	commpage_update_active_cpus();
	cpufamily = cpuid_get_cpufamily();

	/* machine_info valid after ml_get_max_cpus() */
	*((uint8_t*)(_COMM_PAGE_PHYSICAL_CPUS+_COMM_PAGE_RW_OFFSET)) = (uint8_t) machine_info.physical_cpu_max;
	*((uint8_t*)(_COMM_PAGE_LOGICAL_CPUS+_COMM_PAGE_RW_OFFSET))= (uint8_t) machine_info.logical_cpu_max;
	*((uint64_t*)(_COMM_PAGE_MEMORY_SIZE+_COMM_PAGE_RW_OFFSET)) = machine_info.max_mem;
	*((uint32_t*)(_COMM_PAGE_CPUFAMILY+_COMM_PAGE_RW_OFFSET)) = (uint32_t)cpufamily;
	*((uint32_t*)(_COMM_PAGE_DEV_FIRM+_COMM_PAGE_RW_OFFSET)) = (uint32_t)PE_i_can_has_debugger(NULL);
	*((uint8_t*)(_COMM_PAGE_USER_TIMEBASE+_COMM_PAGE_RW_OFFSET)) = user_timebase_allowed();
	*((uint8_t*)(_COMM_PAGE_CONT_HWCLOCK+_COMM_PAGE_RW_OFFSET)) = user_cont_hwclock_allowed();
	*((uint8_t*)(_COMM_PAGE_KERNEL_PAGE_SHIFT+_COMM_PAGE_RW_OFFSET)) = (uint8_t) page_shift;

#if __arm64__
	*((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_32+_COMM_PAGE_RW_OFFSET)) = (uint8_t) page_shift_user32;
	*((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_64+_COMM_PAGE_RW_OFFSET)) = (uint8_t) SIXTEENK_PAGE_SHIFT;
#elif (__ARM_ARCH_7K__ >= 2) && defined(PLATFORM_WatchOS)
	/* enforce 16KB alignment for watch targets with new ABI */
	*((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_32+_COMM_PAGE_RW_OFFSET)) = (uint8_t) SIXTEENK_PAGE_SHIFT;
	*((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_64+_COMM_PAGE_RW_OFFSET)) = (uint8_t) SIXTEENK_PAGE_SHIFT;
#else /* __arm64__ */
	*((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_32+_COMM_PAGE_RW_OFFSET)) = (uint8_t) PAGE_SHIFT;
	*((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_64+_COMM_PAGE_RW_OFFSET)) = (uint8_t) PAGE_SHIFT;
#endif /* __arm64__ */

	commpage_update_timebase();
	commpage_update_mach_continuous_time(0);

	clock_sec_t secs;
	clock_usec_t microsecs;
	clock_get_boottime_microtime(&secs, &microsecs);
	commpage_update_boottime(secs * USEC_PER_SEC + microsecs);

	/* 
	 * set commpage approximate time to zero for initialization. 
	 * scheduler shall populate correct value before running user thread
	 */
	*((uint64_t *)(_COMM_PAGE_APPROX_TIME+ _COMM_PAGE_RW_OFFSET)) = 0;
#ifdef CONFIG_MACH_APPROXIMATE_TIME
	*((uint8_t *)(_COMM_PAGE_APPROX_TIME_SUPPORTED+_COMM_PAGE_RW_OFFSET)) = 1;
#else
	*((uint8_t *)(_COMM_PAGE_APPROX_TIME_SUPPORTED+_COMM_PAGE_RW_OFFSET)) = 0;
#endif

	commpage_update_kdebug_state();

#if CONFIG_ATM
	commpage_update_atm_diagnostic_config(atm_get_diagnostic_config());
#endif

}

struct mu {
	uint64_t m;				// magic number
	int32_t a;				// add indicator
	int32_t s;				// shift amount
};

void
commpage_set_timestamp(
	uint64_t	tbr, 
	uint64_t	secs,
	uint64_t	frac,
	uint64_t	scale,
	uint64_t	tick_per_sec)
{
	new_commpage_timeofday_data_t *commpage_timeofday_datap;

	if (commPagePtr == 0)
		return;

	commpage_timeofday_datap =  (new_commpage_timeofday_data_t *)(_COMM_PAGE_NEWTIMEOFDAY_DATA+_COMM_PAGE_RW_OFFSET);

	commpage_timeofday_datap->TimeStamp_tick = 0x0ULL;

#if	(__ARM_ARCH__ >= 7)
	__asm__ volatile("dmb ish");
#endif
	commpage_timeofday_datap->TimeStamp_sec = secs;
	commpage_timeofday_datap->TimeStamp_frac = frac;
	commpage_timeofday_datap->Ticks_scale = scale;
	commpage_timeofday_datap->Ticks_per_sec = tick_per_sec;

#if	(__ARM_ARCH__ >= 7)
	__asm__ volatile("dmb ish");
#endif
	commpage_timeofday_datap->TimeStamp_tick = tbr;
}

/*
 * Update _COMM_PAGE_MEMORY_PRESSURE.  Called periodically from vm's compute_memory_pressure()
 */

void
commpage_set_memory_pressure(
    unsigned int    pressure )
{
	if (commPagePtr == 0)
		return;
	*((uint32_t *)(_COMM_PAGE_MEMORY_PRESSURE+_COMM_PAGE_RW_OFFSET)) = pressure;
}

/*
 * Update _COMM_PAGE_SPIN_COUNT.  We might want to reduce when running on a battery, etc.
 */

void
commpage_set_spin_count(
        unsigned int    count )
{
        if (count == 0)     /* we test for 0 after decrement, not before */
            count = 1;

	if (commPagePtr == 0)
		return;
	*((uint32_t *)(_COMM_PAGE_SPIN_COUNT+_COMM_PAGE_RW_OFFSET)) = count;
}

/*
 * Determine number of CPUs on this system.
 */
static int
commpage_cpus( void )
{
	int cpus;

	cpus = ml_get_max_cpus();	// NB: this call can block

	if (cpus == 0)
		panic("commpage cpus==0");
	if (cpus > 0xFF)
		cpus = 0xFF;

	return cpus;
}

vm_address_t
_get_commpage_priv_address(void)
{
	return sharedpage_rw_addr;
}

/*
 * Initialize _cpu_capabilities vector
 */
static void
commpage_init_cpu_capabilities( void )
{
	uint32_t bits;
	int cpus;
	ml_cpu_info_t cpu_info;

	bits = 0;
	ml_cpu_get_info(&cpu_info);

	switch (cpu_info.cache_line_size) {
		case 128:
			bits |= kCache128;
			break;
		case 64:
			bits |= kCache64;
			break;
		case 32:
			bits |= kCache32;
			break;
		default:
			break;
	}
	cpus = commpage_cpus();

	if (cpus == 1)
		bits |= kUP;

	bits |= (cpus << kNumCPUsShift);

	bits |= kFastThreadLocalStorage;        // TPIDRURO for TLS

#if	__ARM_VFP__
	bits |= kHasVfp;
	arm_mvfp_info_t *mvfp_info = arm_mvfp_info();
	if (mvfp_info->neon)
		bits |= kHasNeon;
	if (mvfp_info->neon_hpfp)
		bits |= kHasNeonHPFP;
	if (mvfp_info->neon_fp16)
		bits |= kHasNeonFP16;
#endif
#if defined(__arm64__)
	bits |= kHasFMA;
#endif
#if	__ARM_ENABLE_WFE_
#ifdef __arm64__
	if (arm64_wfe_allowed()) {
		bits |= kHasEvent;
	}
#else
	bits |= kHasEvent;
#endif
#endif
#if __ARM_V8_CRYPTO_EXTENSIONS__ 
	bits |= kHasARMv8Crypto;
#endif
#ifdef __arm64__
	uint64_t isar0 = __builtin_arm_rsr64("ID_AA64ISAR0_EL1");
	if ((isar0 & ID_AA64ISAR0_EL1_ATOMIC_MASK) == ID_AA64ISAR0_EL1_ATOMIC_8_1) {
		bits |= kHasARMv81Atomics;
		gARMv81Atomics = 1;
	}
	if ((isar0 & ID_AA64ISAR0_EL1_CRC32_MASK) == ID_AA64ISAR0_EL1_CRC32_EN) {
		bits |= kHasARMv8Crc32;
		gARMv8Crc32 = 1;
	}
#endif
	_cpu_capabilities = bits;

	*((uint32_t *)(_COMM_PAGE_CPU_CAPABILITIES+_COMM_PAGE_RW_OFFSET)) = _cpu_capabilities;
}

/*
 * Updated every time a logical CPU goes offline/online
 */
void
commpage_update_active_cpus(void)
{
        if (!commPagePtr)
                return;
	*((uint8_t *)(_COMM_PAGE_ACTIVE_CPUS+_COMM_PAGE_RW_OFFSET)) = processor_avail_count;
}

/*
 * Update the commpage bits for mach_absolute_time and mach_continuous_time (for userspace)
 */
void
commpage_update_timebase(void)
{
	if (commPagePtr) {
		*((uint64_t*)(_COMM_PAGE_TIMEBASE_OFFSET+_COMM_PAGE_RW_OFFSET)) = rtclock_base_abstime;
	}
}

/*
 * Update the commpage with current kdebug state. This currently has bits for
 * global trace state, and typefilter enablement. It is likely additional state
 * will be tracked in the future.
 *
 * INVARIANT: This value will always be 0 if global tracing is disabled. This
 * allows simple guard tests of "if (*_COMM_PAGE_KDEBUG_ENABLE) { ... }"
 */
void
commpage_update_kdebug_state(void)
{
	if (commPagePtr)
		*((volatile uint32_t*)(_COMM_PAGE_KDEBUG_ENABLE+_COMM_PAGE_RW_OFFSET)) = kdebug_commpage_state();
}

/* Ditto for atm_diagnostic_config */
void
commpage_update_atm_diagnostic_config(uint32_t diagnostic_config)
{
	if (commPagePtr)
		*((volatile uint32_t*)(_COMM_PAGE_ATM_DIAGNOSTIC_CONFIG+_COMM_PAGE_RW_OFFSET)) = diagnostic_config;
}

/*
 * Update the commpage data with the state of multiuser mode for
 * this device. Allowing various services in userspace to avoid
 * IPC in the (more common) non-multiuser environment.
 */
void
commpage_update_multiuser_config(uint32_t multiuser_config)
{
	if (commPagePtr)
		*((volatile uint32_t *)(_COMM_PAGE_MULTIUSER_CONFIG+_COMM_PAGE_RW_OFFSET)) = multiuser_config;
}

/*
 * update the commpage data for 
 * last known value of mach_absolute_time()
 */

void
commpage_update_mach_approximate_time(uint64_t abstime)
{
#ifdef CONFIG_MACH_APPROXIMATE_TIME
	uintptr_t approx_time_base = (uintptr_t)(_COMM_PAGE_APPROX_TIME + _COMM_PAGE_RW_OFFSET);
	uint64_t saved_data;

	if (commPagePtr) {
		saved_data = atomic_load_explicit((_Atomic uint64_t *)approx_time_base,
			memory_order_relaxed);
		if (saved_data < abstime) {
			/* ignoring the success/fail return value assuming that
			 * if the value has been updated since we last read it,
			 * "someone" has a newer timestamp than us and ours is
			 * now invalid. */
			atomic_compare_exchange_strong_explicit((_Atomic uint64_t *)approx_time_base, 
				&saved_data, abstime, memory_order_relaxed, memory_order_relaxed);
		}
	}
#else
#pragma unused (abstime)
#endif
}

/*
 * update the commpage data's total system sleep time for 
 * userspace call to mach_continuous_time()
 */
void
commpage_update_mach_continuous_time(uint64_t sleeptime)
{
	if (commPagePtr) {
#ifdef __arm64__
		*((uint64_t *)(_COMM_PAGE_CONT_TIMEBASE + _COMM_PAGE_RW_OFFSET)) = sleeptime;
#else
		uint64_t *c_time_base = (uint64_t *)(_COMM_PAGE_CONT_TIMEBASE + _COMM_PAGE_RW_OFFSET);
		uint64_t old;
		do {
			old = *c_time_base;
		} while(!OSCompareAndSwap64(old, sleeptime, c_time_base));
#endif /* __arm64__ */
	}
}

/*
 * update the commpage's value for the boot time
 */
void
commpage_update_boottime(uint64_t value)
{
	if (commPagePtr) {
#ifdef __arm64__
		*((uint64_t *)(_COMM_PAGE_BOOTTIME_USEC + _COMM_PAGE_RW_OFFSET)) = value;
#else
		uint64_t *cp = (uint64_t *)(_COMM_PAGE_BOOTTIME_USEC + _COMM_PAGE_RW_OFFSET);
		uint64_t old_value;
		do {
			old_value = *cp;
		} while (!OSCompareAndSwap64(old_value, value, cp));
#endif /* __arm64__ */
	}
}


/*
 * After this counter has incremented, all running CPUs are guaranteed to
 * have quiesced, i.e. executed serially dependent memory barriers.
 * This is only tracked for CPUs running in userspace, therefore only useful
 * outside the kernel.
 *
 * Note that you can't know which side of those barriers your read was from,
 * so you have to observe 2 increments in order to ensure that you saw a
 * serially dependent barrier chain across all running CPUs.
 */
uint64_t
commpage_increment_cpu_quiescent_counter(void)
{
	if (!commPagePtr)
		return 0;

	uint64_t old_gen;

	_Atomic uint64_t *sched_gen = (_Atomic uint64_t *)(_COMM_PAGE_CPU_QUIESCENT_COUNTER +
	                                                   _COMM_PAGE_RW_OFFSET);
	/*
	 * On 32bit architectures, double-wide atomic load or stores are a CAS,
	 * so the atomic increment is the most efficient way to increment the
	 * counter.
	 *
	 * On 64bit architectures however, because the update is synchronized by
	 * the cpu mask, relaxed loads and stores is more efficient.
	 */
#if __LP64__
	old_gen = atomic_load_explicit(sched_gen, memory_order_relaxed);
	atomic_store_explicit(sched_gen, old_gen + 1, memory_order_relaxed);
#else
	old_gen = atomic_fetch_add_explicit(sched_gen, 1, memory_order_relaxed);
#endif
	return old_gen;
}
Commit	Line	Data
5ba3f43e A	1	/*
	2	* Copyright (c) 2007 Apple Inc. All rights reserved.
	3	* Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
	4	*
	5	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	6	*
	7	* This file contains Original Code and/or Modifications of Original Code
	8	* as defined in and that are subject to the Apple Public Source License
	9	* Version 2.0 (the 'License'). You may not use this file except in
	10	* compliance with the License. The rights granted to you under the License
	11	* may not be used to create, or enable the creation or redistribution of,
	12	* unlawful or unlicensed copies of an Apple operating system, or to
	13	* circumvent, violate, or enable the circumvention or violation of, any
	14	* terms of an Apple operating system software license agreement.
	15	*
	16	* Please obtain a copy of the License at
	17	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	18	*
	19	* The Original Code and all software distributed under the License are
	20	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	21	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	22	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	23	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	24	* Please see the License for the specific language governing rights and
	25	* limitations under the License.
	26	*
	27	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	28	*/
	29	/*
	30	* @OSF_COPYRIGHT@
	31	*/
	32	/*
	33	* @APPLE_FREE_COPYRIGHT@
	34	*/
	35	/*
	36	* File: arm/commpage/commpage.c
	37	* Purpose: Set up and export a RO/RW page
	38	*/
d9a64523	39	#include <libkern/section_keywords.h>
5ba3f43e A	40	#include <mach/mach_types.h>
	41	#include <mach/machine.h>
	42	#include <mach/vm_map.h>
	43	#include <machine/cpu_capabilities.h>
	44	#include <machine/commpage.h>
	45	#include <machine/pmap.h>
	46	#include <vm/vm_kern.h>
	47	#include <vm/vm_map.h>
	48	#include <vm/vm_protos.h>
	49	#include <ipc/ipc_port.h>
	50	#include <arm/cpuid.h> /* for cpuid_info() & cache_info() */
	51	#include <arm/rtclock.h>
	52	#include <libkern/OSAtomic.h>
	53	#include <stdatomic.h>
	54
	55	#include <sys/kdebug.h>
	56
	57	#if CONFIG_ATM
	58	#include <atm/atm_internal.h>
	59	#endif
	60
	61	static void commpage_init_cpu_capabilities( void );
	62	static int commpage_cpus( void );
	63
d9a64523 A	64	SECURITY_READ_ONLY_LATE(vm_address_t) commPagePtr=0;
	65	SECURITY_READ_ONLY_LATE(vm_address_t) sharedpage_rw_addr = 0;
	66	SECURITY_READ_ONLY_LATE(uint32_t) _cpu_capabilities = 0;
5ba3f43e	67
d9a64523 A	68	/* For sysctl access from BSD side */
	69	extern int gARMv81Atomics;
	70	extern int gARMv8Crc32;
5ba3f43e A	71
	72	void
	73	commpage_populate(
	74	void)
	75	{
	76	uint16_t c2;
	77	int cpufamily;
	78
	79	sharedpage_rw_addr = pmap_create_sharedpage();
	80	commPagePtr = (vm_address_t)_COMM_PAGE_BASE_ADDRESS;
	81
	82	((uint16_t)(_COMM_PAGE_VERSION+_COMM_PAGE_RW_OFFSET)) = (uint16_t) _COMM_PAGE_THIS_VERSION;
	83
	84	commpage_init_cpu_capabilities();
	85	commpage_set_timestamp(0, 0, 0, 0, 0);
	86
	87	if (_cpu_capabilities & kCache32)
	88	c2 = 32;
	89	else if (_cpu_capabilities & kCache64)
	90	c2 = 64;
	91	else if (_cpu_capabilities & kCache128)
	92	c2 = 128;
	93	else
	94	c2 = 0;
	95
	96	((uint16_t)(_COMM_PAGE_CACHE_LINESIZE+_COMM_PAGE_RW_OFFSET)) = c2;
	97	((uint32_t)(_COMM_PAGE_SPIN_COUNT+_COMM_PAGE_RW_OFFSET)) = 1;
	98
	99	commpage_update_active_cpus();
	100	cpufamily = cpuid_get_cpufamily();
	101
	102	/* machine_info valid after ml_get_max_cpus() */
	103	((uint8_t)(_COMM_PAGE_PHYSICAL_CPUS+_COMM_PAGE_RW_OFFSET)) = (uint8_t) machine_info.physical_cpu_max;
	104	((uint8_t)(_COMM_PAGE_LOGICAL_CPUS+_COMM_PAGE_RW_OFFSET))= (uint8_t) machine_info.logical_cpu_max;
	105	((uint64_t)(_COMM_PAGE_MEMORY_SIZE+_COMM_PAGE_RW_OFFSET)) = machine_info.max_mem;
	106	((uint32_t)(_COMM_PAGE_CPUFAMILY+_COMM_PAGE_RW_OFFSET)) = (uint32_t)cpufamily;
	107	((uint32_t)(_COMM_PAGE_DEV_FIRM+_COMM_PAGE_RW_OFFSET)) = (uint32_t)PE_i_can_has_debugger(NULL);
	108	((uint8_t)(_COMM_PAGE_USER_TIMEBASE+_COMM_PAGE_RW_OFFSET)) = user_timebase_allowed();
	109	((uint8_t)(_COMM_PAGE_CONT_HWCLOCK+_COMM_PAGE_RW_OFFSET)) = user_cont_hwclock_allowed();
	110	((uint8_t)(_COMM_PAGE_KERNEL_PAGE_SHIFT+_COMM_PAGE_RW_OFFSET)) = (uint8_t) page_shift;
	111
	112	#if __arm64__
	113	((uint8_t)(_COMM_PAGE_USER_PAGE_SHIFT_32+_COMM_PAGE_RW_OFFSET)) = (uint8_t) page_shift_user32;
	114	((uint8_t)(_COMM_PAGE_USER_PAGE_SHIFT_64+_COMM_PAGE_RW_OFFSET)) = (uint8_t) SIXTEENK_PAGE_SHIFT;
	115	#elif (__ARM_ARCH_7K__ >= 2) && defined(PLATFORM_WatchOS)
	116	/* enforce 16KB alignment for watch targets with new ABI */
	117	((uint8_t)(_COMM_PAGE_USER_PAGE_SHIFT_32+_COMM_PAGE_RW_OFFSET)) = (uint8_t) SIXTEENK_PAGE_SHIFT;
	118	((uint8_t)(_COMM_PAGE_USER_PAGE_SHIFT_64+_COMM_PAGE_RW_OFFSET)) = (uint8_t) SIXTEENK_PAGE_SHIFT;
	119	#else /* __arm64__ */
	120	((uint8_t)(_COMM_PAGE_USER_PAGE_SHIFT_32+_COMM_PAGE_RW_OFFSET)) = (uint8_t) PAGE_SHIFT;
	121	((uint8_t)(_COMM_PAGE_USER_PAGE_SHIFT_64+_COMM_PAGE_RW_OFFSET)) = (uint8_t) PAGE_SHIFT;
	122	#endif /* __arm64__ */
	123
	124	commpage_update_timebase();
	125	commpage_update_mach_continuous_time(0);
	126
	127	clock_sec_t secs;
	128	clock_usec_t microsecs;
	129	clock_get_boottime_microtime(&secs, &microsecs);
	130	commpage_update_boottime(secs * USEC_PER_SEC + microsecs);
	131
	132	/*
	133	* set commpage approximate time to zero for initialization.
	134	* scheduler shall populate correct value before running user thread
135	*/
136	((uint64_t )(_COMM_PAGE_APPROX_TIME+ _COMM_PAGE_RW_OFFSET)) = 0;
137	#ifdef CONFIG_MACH_APPROXIMATE_TIME
138	((uint8_t )(_COMM_PAGE_APPROX_TIME_SUPPORTED+_COMM_PAGE_RW_OFFSET)) = 1;
139	#else
140	((uint8_t )(_COMM_PAGE_APPROX_TIME_SUPPORTED+_COMM_PAGE_RW_OFFSET)) = 0;
141	#endif
142
143	commpage_update_kdebug_state();
144
145	#if CONFIG_ATM
146	commpage_update_atm_diagnostic_config(atm_get_diagnostic_config());
147	#endif
148
149	}
150
151	struct mu {
152	uint64_t m; // magic number
153	int32_t a; // add indicator
154	int32_t s; // shift amount
155	};
156
157	void
158	commpage_set_timestamp(
159	uint64_t tbr,
160	uint64_t secs,
161	uint64_t frac,
162	uint64_t scale,
163	uint64_t tick_per_sec)
164	{
165	new_commpage_timeofday_data_t *commpage_timeofday_datap;
166
167	if (commPagePtr == 0)
168	return;
169
170	commpage_timeofday_datap = (new_commpage_timeofday_data_t *)(_COMM_PAGE_NEWTIMEOFDAY_DATA+_COMM_PAGE_RW_OFFSET);
171
172	commpage_timeofday_datap->TimeStamp_tick = 0x0ULL;
173
174	#if (__ARM_ARCH__ >= 7)
175	__asm__ volatile("dmb ish");
176	#endif
177	commpage_timeofday_datap->TimeStamp_sec = secs;
178	commpage_timeofday_datap->TimeStamp_frac = frac;
179	commpage_timeofday_datap->Ticks_scale = scale;
180	commpage_timeofday_datap->Ticks_per_sec = tick_per_sec;
181
182	#if (__ARM_ARCH__ >= 7)
183	__asm__ volatile("dmb ish");
184	#endif
185	commpage_timeofday_datap->TimeStamp_tick = tbr;
186	}
187
188	/*
189	* Update _COMM_PAGE_MEMORY_PRESSURE. Called periodically from vm's compute_memory_pressure()
190	*/
191
192	void
193	commpage_set_memory_pressure(
194	unsigned int pressure )
195	{
196	if (commPagePtr == 0)
197	return;
198	((uint32_t )(_COMM_PAGE_MEMORY_PRESSURE+_COMM_PAGE_RW_OFFSET)) = pressure;
199	}
200
201	/*
202	* Update _COMM_PAGE_SPIN_COUNT. We might want to reduce when running on a battery, etc.
203	*/
204
205	void
206	commpage_set_spin_count(
207	unsigned int count )
208	{
209	if (count == 0) /* we test for 0 after decrement, not before */
210	count = 1;
211
212	if (commPagePtr == 0)
213	return;
214	((uint32_t )(_COMM_PAGE_SPIN_COUNT+_COMM_PAGE_RW_OFFSET)) = count;
215	}
216
217	/*
218	* Determine number of CPUs on this system.
219	*/
220	static int
221	commpage_cpus( void )
222	{
223	int cpus;
224
225	cpus = ml_get_max_cpus(); // NB: this call can block
226
227	if (cpus == 0)
228	panic("commpage cpus==0");
229	if (cpus > 0xFF)
230	cpus = 0xFF;
231
232	return cpus;
233	}
234
d9a64523 A	235	vm_address_t
	236	_get_commpage_priv_address(void)
	237	{
	238	return sharedpage_rw_addr;
	239	}
	240
5ba3f43e A	241	/*
	242	* Initialize _cpu_capabilities vector
	243	*/
	244	static void
	245	commpage_init_cpu_capabilities( void )
	246	{
	247	uint32_t bits;
	248	int cpus;
	249	ml_cpu_info_t cpu_info;
	250
	251	bits = 0;
	252	ml_cpu_get_info(&cpu_info);
	253
	254	switch (cpu_info.cache_line_size) {
	255	case 128:
	256	bits \|= kCache128;
	257	break;
	258	case 64:
	259	bits \|= kCache64;
	260	break;
	261	case 32:
	262	bits \|= kCache32;
	263	break;
	264	default:
	265	break;
	266	}
	267	cpus = commpage_cpus();
	268
	269	if (cpus == 1)
	270	bits \|= kUP;
	271
	272	bits \|= (cpus << kNumCPUsShift);
	273
	274	bits \|= kFastThreadLocalStorage; // TPIDRURO for TLS
5c9f4661	275
5ba3f43e A	276	#if __ARM_VFP__
5ba3f43e A	277	bits \|= kHasVfp;
5c9f4661 A	278	arm_mvfp_info_t *mvfp_info = arm_mvfp_info();
	279	if (mvfp_info->neon)
	280	bits \|= kHasNeon;
	281	if (mvfp_info->neon_hpfp)
5ba3f43e	282	bits \|= kHasNeonHPFP;
d9a64523 A	283	if (mvfp_info->neon_fp16)
d9a64523 A	284	bits \|= kHasNeonFP16;
5ba3f43e A	285	#endif
	286	#if defined(__arm64__)
	287	bits \|= kHasFMA;
	288	#endif
	289	#if __ARM_ENABLE_WFE_
	290	#ifdef __arm64__
	291	if (arm64_wfe_allowed()) {
	292	bits \|= kHasEvent;
	293	}
	294	#else
	295	bits \|= kHasEvent;
	296	#endif
	297	#endif
	298	#if __ARM_V8_CRYPTO_EXTENSIONS__
	299	bits \|= kHasARMv8Crypto;
	300	#endif
	301	#ifdef __arm64__
d9a64523 A	302	uint64_t isar0 = __builtin_arm_rsr64("ID_AA64ISAR0_EL1");
d9a64523 A	303	if ((isar0 & ID_AA64ISAR0_EL1_ATOMIC_MASK) == ID_AA64ISAR0_EL1_ATOMIC_8_1) {
5ba3f43e A	304	bits \|= kHasARMv81Atomics;
	305	gARMv81Atomics = 1;
	306	}
d9a64523 A	307	if ((isar0 & ID_AA64ISAR0_EL1_CRC32_MASK) == ID_AA64ISAR0_EL1_CRC32_EN) {
	308	bits \|= kHasARMv8Crc32;
	309	gARMv8Crc32 = 1;
	310	}
5ba3f43e A	311	#endif
	312	_cpu_capabilities = bits;
	313
	314	((uint32_t )(_COMM_PAGE_CPU_CAPABILITIES+_COMM_PAGE_RW_OFFSET)) = _cpu_capabilities;
	315	}
	316
	317	/*
	318	* Updated every time a logical CPU goes offline/online
	319	*/
	320	void
	321	commpage_update_active_cpus(void)
	322	{
	323	if (!commPagePtr)
	324	return;
	325	((uint8_t )(_COMM_PAGE_ACTIVE_CPUS+_COMM_PAGE_RW_OFFSET)) = processor_avail_count;
	326	}
	327
	328	/*
	329	* Update the commpage bits for mach_absolute_time and mach_continuous_time (for userspace)
	330	*/
	331	void
	332	commpage_update_timebase(void)
	333	{
	334	if (commPagePtr) {
	335	((uint64_t)(_COMM_PAGE_TIMEBASE_OFFSET+_COMM_PAGE_RW_OFFSET)) = rtclock_base_abstime;
	336	}
	337	}
	338
	339	/*
	340	* Update the commpage with current kdebug state. This currently has bits for
	341	* global trace state, and typefilter enablement. It is likely additional state
	342	* will be tracked in the future.
	343	*
	344	* INVARIANT: This value will always be 0 if global tracing is disabled. This
	345	* allows simple guard tests of "if (*_COMM_PAGE_KDEBUG_ENABLE) { ... }"
	346	*/
	347	void
	348	commpage_update_kdebug_state(void)
	349	{
	350	if (commPagePtr)
	351	((volatile uint32_t)(_COMM_PAGE_KDEBUG_ENABLE+_COMM_PAGE_RW_OFFSET)) = kdebug_commpage_state();
	352	}
	353
	354	/* Ditto for atm_diagnostic_config */
	355	void
	356	commpage_update_atm_diagnostic_config(uint32_t diagnostic_config)
	357	{
	358	if (commPagePtr)
	359	((volatile uint32_t)(_COMM_PAGE_ATM_DIAGNOSTIC_CONFIG+_COMM_PAGE_RW_OFFSET)) = diagnostic_config;
	360	}
	361
	362	/*
	363	* Update the commpage data with the state of multiuser mode for
	364	* this device. Allowing various services in userspace to avoid
	365	* IPC in the (more common) non-multiuser environment.
	366	*/
	367	void
	368	commpage_update_multiuser_config(uint32_t multiuser_config)
	369	{
	370	if (commPagePtr)
	371	((volatile uint32_t )(_COMM_PAGE_MULTIUSER_CONFIG+_COMM_PAGE_RW_OFFSET)) = multiuser_config;
	372	}
	373
	374	/*
375	* update the commpage data for
376	* last known value of mach_absolute_time()
377	*/
378
379	void
380	commpage_update_mach_approximate_time(uint64_t abstime)
381	{
382	#ifdef CONFIG_MACH_APPROXIMATE_TIME
383	uintptr_t approx_time_base = (uintptr_t)(_COMM_PAGE_APPROX_TIME + _COMM_PAGE_RW_OFFSET);
384	uint64_t saved_data;
385
386	if (commPagePtr) {
387	saved_data = atomic_load_explicit((_Atomic uint64_t *)approx_time_base,
388	memory_order_relaxed);
389	if (saved_data < abstime) {
390	/* ignoring the success/fail return value assuming that
391	* if the value has been updated since we last read it,
392	* "someone" has a newer timestamp than us and ours is
393	* now invalid. */
394	atomic_compare_exchange_strong_explicit((_Atomic uint64_t *)approx_time_base,
395	&saved_data, abstime, memory_order_relaxed, memory_order_relaxed);
396	}
397	}
398	#else
399	#pragma unused (abstime)
400	#endif
401	}
402
403	/*
404	* update the commpage data's total system sleep time for
405	* userspace call to mach_continuous_time()
406	*/
407	void
408	commpage_update_mach_continuous_time(uint64_t sleeptime)
409	{
410	if (commPagePtr) {
411	#ifdef __arm64__
412	((uint64_t )(_COMM_PAGE_CONT_TIMEBASE + _COMM_PAGE_RW_OFFSET)) = sleeptime;
413	#else
414	uint64_t c_time_base = (uint64_t )(_COMM_PAGE_CONT_TIMEBASE + _COMM_PAGE_RW_OFFSET);
415	uint64_t old;
416	do {
417	old = *c_time_base;
418	} while(!OSCompareAndSwap64(old, sleeptime, c_time_base));
419	#endif /* __arm64__ */
420	}
421	}
422
423	/*
424	* update the commpage's value for the boot time
425	*/
426	void
427	commpage_update_boottime(uint64_t value)
428	{
429	if (commPagePtr) {
430	#ifdef __arm64__
431	((uint64_t )(_COMM_PAGE_BOOTTIME_USEC + _COMM_PAGE_RW_OFFSET)) = value;
432	#else
433	uint64_t cp = (uint64_t )(_COMM_PAGE_BOOTTIME_USEC + _COMM_PAGE_RW_OFFSET);
434	uint64_t old_value;
435	do {
436	old_value = *cp;
437	} while (!OSCompareAndSwap64(old_value, value, cp));
438	#endif /* __arm64__ */
439	}
440	}
d26ffc64	441
d9a64523	442
d26ffc64	443	/*
d9a64523 A	444	* After this counter has incremented, all running CPUs are guaranteed to
	445	* have quiesced, i.e. executed serially dependent memory barriers.
	446	* This is only tracked for CPUs running in userspace, therefore only useful
	447	* outside the kernel.
	448	*
	449	* Note that you can't know which side of those barriers your read was from,
	450	* so you have to observe 2 increments in order to ensure that you saw a
	451	* serially dependent barrier chain across all running CPUs.
d26ffc64	452	*/
d9a64523 A	453	uint64_t
	454	commpage_increment_cpu_quiescent_counter(void)
	455	{
	456	if (!commPagePtr)
	457	return 0;
	458
	459	uint64_t old_gen;
	460
	461	_Atomic uint64_t sched_gen = (_Atomic uint64_t )(_COMM_PAGE_CPU_QUIESCENT_COUNTER +
	462	_COMM_PAGE_RW_OFFSET);
	463	/*
	464	* On 32bit architectures, double-wide atomic load or stores are a CAS,
	465	* so the atomic increment is the most efficient way to increment the
	466	* counter.
	467	*
	468	* On 64bit architectures however, because the update is synchronized by
	469	* the cpu mask, relaxed loads and stores is more efficient.
	470	*/
	471	#if __LP64__
	472	old_gen = atomic_load_explicit(sched_gen, memory_order_relaxed);
	473	atomic_store_explicit(sched_gen, old_gen + 1, memory_order_relaxed);
d26ffc64	474	#else
d9a64523 A	475	old_gen = atomic_fetch_add_explicit(sched_gen, 1, memory_order_relaxed);
	476	#endif
	477	return old_gen;
d26ffc64	478	}
d9a64523	479