[apple/xnu.git] / osfmk / arm64 / tlb.h

/*
 * Copyright (c) 2019 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#pragma once

#include <arm64/proc_reg.h>
#include <machine/atomic.h>

#define tlbi_addr(x) ((((x) >> 12) & TLBI_ADDR_MASK) << TLBI_ADDR_SHIFT)
#define tlbi_asid(x) (((uintptr_t)(x) & TLBI_ASID_MASK) << TLBI_ASID_SHIFT)

#if __ARM_KERNEL_PROTECT__
/*
 * __ARM_KERNEL_PROTECT__ adds two complications to TLB management:
 *
 * 1. As each pmap has two ASIDs, every TLB operation that targets an ASID must
 *   target both ASIDs for the pmap that owns the target ASID.
 *
 * 2. Any TLB operation targeting the kernel_pmap ASID (ASID 0) must target all
 *   ASIDs (as kernel_pmap mappings may be referenced while using an ASID that
 *   belongs to another pmap).  We expect these routines to be called with the
 *   EL0 ASID for the target; not the EL1 ASID.
 */
#endif /* __ARM_KERNEL_PROTECT__ */

static inline void
sync_tlb_flush(void)
{
	__builtin_arm_dsb(DSB_ISH);
	__builtin_arm_isb(ISB_SY);
}

static inline void
sync_tlb_flush_local(void)
{
	__builtin_arm_dsb(DSB_NSH);
	__builtin_arm_isb(ISB_SY);
}

// flush_mmu_tlb: full TLB flush on all cores
static inline void
flush_mmu_tlb_async(void)
{
	asm volatile ("tlbi vmalle1is");
}

static inline void
flush_mmu_tlb(void)
{
	flush_mmu_tlb_async();
	sync_tlb_flush();
}

// flush_core_tlb: full TLB flush on local core only
static inline void
flush_core_tlb_async(void)
{
	asm volatile ("tlbi vmalle1");
}

static inline void
flush_core_tlb(void)
{
	flush_core_tlb_async();
	sync_tlb_flush();
}

// flush_mmu_tlb_allentries_async: flush entries that map VA range, all ASIDS, all cores
// start and end are in units of 4K pages.
static inline void
flush_mmu_tlb_allentries_async(uint64_t start, uint64_t end, uint64_t pmap_page_size)
{
#if __ARM_16K_PG__
	if (pmap_page_size == 16384) {
		start = start & ~0x3ULL;

		/*
		 * The code below is not necessarily correct.  From an overview of
		 * the client code, the expected contract for TLB flushes is that
		 * we will expand from an "address, length" pair to "start address,
		 * end address" in the course of a TLB flush.  This suggests that
		 * a flush for "X, X+4" is actually only asking for a flush of a
		 * single 16KB page.  At the same time, we'd like to be prepared
		 * for bad inputs (X, X+3), so add 3 and then truncate the 4KB page
		 * number to a 16KB page boundary.  This should deal correctly with
		 * unaligned inputs.
		 *
		 * If our expecations about client behavior are wrong however, this
		 * will lead to occasional TLB corruption on platforms with 16KB
		 * pages.
		 */
		end = (end + 0x3ULL) & ~0x3ULL;
	}
#endif // __ARM_16K_PG__
	for (; start < end; start += (pmap_page_size / 4096)) {
		asm volatile ("tlbi vaae1is, %0" : : "r"(start));
	}
}

static inline void
flush_mmu_tlb_allentries(uint64_t start, uint64_t end, uint64_t pmap_page_size)
{
	flush_mmu_tlb_allentries_async(start, end, pmap_page_size);
	sync_tlb_flush();
}

// flush_mmu_tlb_entry: flush TLB entries that map a VA and ASID, all cores
// Will also flush global entries that match the VA
static inline void
flush_mmu_tlb_entry_async(uint64_t val)
{
#if __ARM_KERNEL_PROTECT__
	uint64_t asid = val >> TLBI_ASID_SHIFT;
	if (asid == 0) {
		asm volatile ("tlbi vaae1is, %0" : : "r"(val));
		return;
	}
	val = val & ~(1ULL << TLBI_ASID_SHIFT);
	asm volatile ("tlbi vae1is, %0" : : "r"(val));
	val = val | (1ULL << TLBI_ASID_SHIFT);
#endif /* __ARM_KERNEL_PROTECT__ */
	asm volatile ("tlbi vae1is, %0" : : "r"(val));
}

static inline void
flush_mmu_tlb_entry(uint64_t val)
{
	flush_mmu_tlb_entry_async(val);
	sync_tlb_flush();
}

// flush_mmu_tlb_entries: flush TLB entries that map a VA range and ASID, all cores
// start and end must have the ASID in the high 16 bits, with the VA in units of 4K in the lowest bits
// Will also flush global entries that match the VA range
static inline void
flush_mmu_tlb_entries_async(uint64_t start, uint64_t end, uint64_t pmap_page_size)
{
#if __ARM_16K_PG__
	if (pmap_page_size == 16384) {
		start = start & ~0x3ULL;

		/*
		 * The code below is not necessarily correct.  From an overview of
		 * the client code, the expected contract for TLB flushes is that
		 * we will expand from an "address, length" pair to "start address,
		 * end address" in the course of a TLB flush.  This suggests that
		 * a flush for "X, X+4" is actually only asking for a flush of a
		 * single 16KB page.  At the same time, we'd like to be prepared
		 * for bad inputs (X, X+3), so add 3 and then truncate the 4KB page
		 * number to a 16KB page boundary.  This should deal correctly with
		 * unaligned inputs.
		 *
		 * If our expecations about client behavior are wrong however, this
		 * will lead to occasional TLB corruption on platforms with 16KB
		 * pages.
		 */
		end = (end + 0x3ULL) & ~0x3ULL;
	}
#endif // __ARM_16K_PG__
#if __ARM_KERNEL_PROTECT__
	uint64_t asid = start >> TLBI_ASID_SHIFT;
	/*
	 * If we are flushing ASID 0, this is a kernel operation.  With this
	 * ASID scheme, this means we should flush all ASIDs.
	 */
	if (asid == 0) {
		for (; start < end; start += (pmap_page_size / 4096)) {
			asm volatile ("tlbi vaae1is, %0" : : "r"(start));
		}
		return;
	}
	start = start | (1ULL << TLBI_ASID_SHIFT);
	end = end | (1ULL << TLBI_ASID_SHIFT);
	for (; start < end; start += (pmap_page_size / 4096)) {
		start = start & ~(1ULL << TLBI_ASID_SHIFT);
		asm volatile ("tlbi vae1is, %0" : : "r"(start));
		start = start | (1ULL << TLBI_ASID_SHIFT);
		asm volatile ("tlbi vae1is, %0" : : "r"(start));
	}
#else
	for (; start < end; start += (pmap_page_size / 4096)) {
		asm volatile ("tlbi vae1is, %0" : : "r"(start));
	}
#endif /* __ARM_KERNEL_PROTECT__ */
}

static inline void
flush_mmu_tlb_entries(uint64_t start, uint64_t end, uint64_t pmap_page_size)
{
	flush_mmu_tlb_entries_async(start, end, pmap_page_size);
	sync_tlb_flush();
}

// flush_mmu_tlb_asid: flush all entries that match an ASID, on all cores
// ASID must be in high 16 bits of argument
// Will not flush global entries
static inline void
flush_mmu_tlb_asid_async(uint64_t val)
{
#if __ARM_KERNEL_PROTECT__
	/*
	 * If we are flushing ASID 0, this is a kernel operation.  With this
	 * ASID scheme, this means we should flush all ASIDs.
	 */
	uint64_t asid = val >> TLBI_ASID_SHIFT;
	if (asid == 0) {
		asm volatile ("tlbi vmalle1is");
		return;
	}
	val = val & ~(1ULL << TLBI_ASID_SHIFT);
	asm volatile ("tlbi aside1is, %0" : : "r"(val));
	val = val | (1ULL << TLBI_ASID_SHIFT);
#endif /* __ARM_KERNEL_PROTECT__ */
	asm volatile ("tlbi aside1is, %0" : : "r"(val));
}

static inline void
flush_mmu_tlb_asid(uint64_t val)
{
	flush_mmu_tlb_asid_async(val);
	sync_tlb_flush();
}

// flush_core_tlb_asid: flush all entries that match an ASID, local core only
// ASID must be in high 16 bits of argument
// Will not flush global entries
static inline void
flush_core_tlb_asid_async(uint64_t val)
{
#if __ARM_KERNEL_PROTECT__
	/*
	 * If we are flushing ASID 0, this is a kernel operation.  With this
	 * ASID scheme, this means we should flush all ASIDs.
	 */
	uint64_t asid = val >> TLBI_ASID_SHIFT;
	if (asid == 0) {
		asm volatile ("tlbi vmalle1");
		return;
	}
	val = val & ~(1ULL << TLBI_ASID_SHIFT);
	asm volatile ("tlbi aside1, %0" : : "r"(val));
	val = val | (1ULL << TLBI_ASID_SHIFT);
#endif /* __ARM_KERNEL_PROTECT__ */
	asm volatile ("tlbi aside1, %0" : : "r"(val));
}

static inline void
flush_core_tlb_asid(uint64_t val)
{
	flush_core_tlb_asid_async(val);
	sync_tlb_flush();
}

#if __ARM_RANGE_TLBI__
#if __ARM_KERNEL_PROTECT__
	#error __ARM_RANGE_TLBI__ + __ARM_KERNEL_PROTECT__ is not currently supported
#endif

#define ARM64_TLB_RANGE_PAGES (1ULL << 21)
#define rtlbi_addr(x, shift) (((x) >> (shift)) & RTLBI_ADDR_MASK)
#define rtlbi_scale(x) ((uint64_t)(x) << RTLBI_SCALE_SHIFT)
#define rtlbi_num(x) ((uint64_t)(x) << RTLBI_NUM_SHIFT)

/**
 * Given the number of pages to invalidate, generate the correct parameter to
 * pass to any of the TLBI by range methods.
 */
static inline uint64_t
generate_rtlbi_param(ppnum_t npages, uint32_t asid, vm_offset_t va, uint64_t pmap_page_shift)
{
	/**
	 * Per the armv8.4 RTLBI extension spec, the range encoded in the rtlbi register operand is defined by:
	 * BaseADDR <= VA < BaseADDR+((NUM+1)*2^(5*SCALE+1) * Translation_Granule_Size)
	 */
	unsigned order = (unsigned)(sizeof(npages) * 8) - (unsigned)__builtin_clz(npages - 1) - 1;
	unsigned scale = ((order ? order : 1) - 1) / 5;
	unsigned granule = 1 << ((5 * scale) + 1);
	unsigned num = (((npages + granule - 1) & ~(granule - 1)) / granule) - 1;
	return tlbi_asid(asid) | RTLBI_TG(pmap_page_shift) | rtlbi_scale(scale) | rtlbi_num(num) | rtlbi_addr(va, pmap_page_shift);
}

// flush_mmu_tlb_range: flush TLB entries that map a VA range using a single instruction
// The argument should be encoded according to generate_rtlbi_param().
// Follows the same ASID matching behavior as flush_mmu_tlb_entries()
static inline void
flush_mmu_tlb_range_async(uint64_t val)
{
	asm volatile ("tlbi rvae1is, %0" : : "r"(val));
}

static inline void
flush_mmu_tlb_range(uint64_t val)
{
	flush_mmu_tlb_range_async(val);
	sync_tlb_flush();
}

// flush_mmu_tlb_allrange: flush TLB entries that map a VA range using a single instruction
// The argument should be encoded according to generate_rtlbi_param().
// Follows the same ASID matching behavior as flush_mmu_tlb_allentries()
static inline void
flush_mmu_tlb_allrange_async(uint64_t val)
{
	asm volatile ("tlbi rvaae1is, %0" : : "r"(val));
}

static inline void
flush_mmu_tlb_allrange(uint64_t val)
{
	flush_mmu_tlb_allrange_async(val);
	sync_tlb_flush();
}

// flush_core_tlb_allrange: flush TLB entries that map a VA range using a single instruction, local core only
// The argument should be encoded according to generate_rtlbi_param().
// Follows the same ASID matching behavior as flush_mmu_tlb_allentries()
static inline void
flush_core_tlb_allrange_async(uint64_t val)
{
	asm volatile ("tlbi rvaae1, %0" : : "r"(val));
}

static inline void
flush_core_tlb_allrange(uint64_t val)
{
	flush_core_tlb_allrange_async(val);
	sync_tlb_flush();
}

#endif // __ARM_RANGE_TLBI__
Commit	Line	Data
cb323159 A	1	/*
	2	* Copyright (c) 2019 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	#pragma once
	30
	31	#include <arm64/proc_reg.h>
	32	#include <machine/atomic.h>
	33
	34	#define tlbi_addr(x) ((((x) >> 12) & TLBI_ADDR_MASK) << TLBI_ADDR_SHIFT)
	35	#define tlbi_asid(x) (((uintptr_t)(x) & TLBI_ASID_MASK) << TLBI_ASID_SHIFT)
	36
	37	#if __ARM_KERNEL_PROTECT__
	38	/*
	39	* __ARM_KERNEL_PROTECT__ adds two complications to TLB management:
	40	*
	41	* 1. As each pmap has two ASIDs, every TLB operation that targets an ASID must
	42	* target both ASIDs for the pmap that owns the target ASID.
	43	*
	44	* 2. Any TLB operation targeting the kernel_pmap ASID (ASID 0) must target all
	45	* ASIDs (as kernel_pmap mappings may be referenced while using an ASID that
	46	* belongs to another pmap). We expect these routines to be called with the
	47	* EL0 ASID for the target; not the EL1 ASID.
	48	*/
	49	#endif /* __ARM_KERNEL_PROTECT__ */
	50
	51	static inline void
	52	sync_tlb_flush(void)
	53	{
	54	__builtin_arm_dsb(DSB_ISH);
	55	__builtin_arm_isb(ISB_SY);
	56	}
	57
2a1bd2d3 A	58	static inline void
	59	sync_tlb_flush_local(void)
	60	{
	61	__builtin_arm_dsb(DSB_NSH);
	62	__builtin_arm_isb(ISB_SY);
	63	}
	64
cb323159 A	65	// flush_mmu_tlb: full TLB flush on all cores
	66	static inline void
	67	flush_mmu_tlb_async(void)
	68	{
	69	asm volatile ("tlbi vmalle1is");
	70	}
	71
	72	static inline void
	73	flush_mmu_tlb(void)
	74	{
	75	flush_mmu_tlb_async();
	76	sync_tlb_flush();
	77	}
	78
	79	// flush_core_tlb: full TLB flush on local core only
	80	static inline void
	81	flush_core_tlb_async(void)
	82	{
	83	asm volatile ("tlbi vmalle1");
	84	}
	85
	86	static inline void
	87	flush_core_tlb(void)
	88	{
	89	flush_core_tlb_async();
	90	sync_tlb_flush();
	91	}
	92
	93	// flush_mmu_tlb_allentries_async: flush entries that map VA range, all ASIDS, all cores
	94	// start and end are in units of 4K pages.
	95	static inline void
f427ee49	96	flush_mmu_tlb_allentries_async(uint64_t start, uint64_t end, uint64_t pmap_page_size)
cb323159 A	97	{
cb323159 A	98	#if __ARM_16K_PG__
f427ee49 A	99	if (pmap_page_size == 16384) {
f427ee49 A	100	start = start & ~0x3ULL;
cb323159	101
f427ee49 A	102	/*
	103	* The code below is not necessarily correct. From an overview of
	104	* the client code, the expected contract for TLB flushes is that
	105	* we will expand from an "address, length" pair to "start address,
	106	* end address" in the course of a TLB flush. This suggests that
	107	* a flush for "X, X+4" is actually only asking for a flush of a
	108	* single 16KB page. At the same time, we'd like to be prepared
	109	* for bad inputs (X, X+3), so add 3 and then truncate the 4KB page
	110	* number to a 16KB page boundary. This should deal correctly with
	111	* unaligned inputs.
	112	*
	113	* If our expecations about client behavior are wrong however, this
	114	* will lead to occasional TLB corruption on platforms with 16KB
	115	* pages.
	116	*/
	117	end = (end + 0x3ULL) & ~0x3ULL;
	118	}
cb323159	119	#endif // __ARM_16K_PG__
f427ee49	120	for (; start < end; start += (pmap_page_size / 4096)) {
cb323159 A	121	asm volatile ("tlbi vaae1is, %0" : : "r"(start));
	122	}
	123	}
	124
	125	static inline void
f427ee49	126	flush_mmu_tlb_allentries(uint64_t start, uint64_t end, uint64_t pmap_page_size)
cb323159	127	{
f427ee49	128	flush_mmu_tlb_allentries_async(start, end, pmap_page_size);
cb323159 A	129	sync_tlb_flush();
	130	}
	131
	132	// flush_mmu_tlb_entry: flush TLB entries that map a VA and ASID, all cores
	133	// Will also flush global entries that match the VA
	134	static inline void
	135	flush_mmu_tlb_entry_async(uint64_t val)
	136	{
	137	#if __ARM_KERNEL_PROTECT__
	138	uint64_t asid = val >> TLBI_ASID_SHIFT;
	139	if (asid == 0) {
	140	asm volatile ("tlbi vaae1is, %0" : : "r"(val));
	141	return;
	142	}
	143	val = val & ~(1ULL << TLBI_ASID_SHIFT);
	144	asm volatile ("tlbi vae1is, %0" : : "r"(val));
	145	val = val \| (1ULL << TLBI_ASID_SHIFT);
	146	#endif /* __ARM_KERNEL_PROTECT__ */
	147	asm volatile ("tlbi vae1is, %0" : : "r"(val));
	148	}
	149
	150	static inline void
	151	flush_mmu_tlb_entry(uint64_t val)
	152	{
	153	flush_mmu_tlb_entry_async(val);
	154	sync_tlb_flush();
	155	}
	156
	157	// flush_mmu_tlb_entries: flush TLB entries that map a VA range and ASID, all cores
	158	// start and end must have the ASID in the high 16 bits, with the VA in units of 4K in the lowest bits
	159	// Will also flush global entries that match the VA range
	160	static inline void
f427ee49	161	flush_mmu_tlb_entries_async(uint64_t start, uint64_t end, uint64_t pmap_page_size)
cb323159 A	162	{
cb323159 A	163	#if __ARM_16K_PG__
f427ee49 A	164	if (pmap_page_size == 16384) {
f427ee49 A	165	start = start & ~0x3ULL;
cb323159	166
f427ee49 A	167	/*
	168	* The code below is not necessarily correct. From an overview of
	169	* the client code, the expected contract for TLB flushes is that
	170	* we will expand from an "address, length" pair to "start address,
	171	* end address" in the course of a TLB flush. This suggests that
	172	* a flush for "X, X+4" is actually only asking for a flush of a
	173	* single 16KB page. At the same time, we'd like to be prepared
	174	* for bad inputs (X, X+3), so add 3 and then truncate the 4KB page
	175	* number to a 16KB page boundary. This should deal correctly with
	176	* unaligned inputs.
	177	*
	178	* If our expecations about client behavior are wrong however, this
	179	* will lead to occasional TLB corruption on platforms with 16KB
	180	* pages.
	181	*/
	182	end = (end + 0x3ULL) & ~0x3ULL;
	183	}
cb323159 A	184	#endif // __ARM_16K_PG__
	185	#if __ARM_KERNEL_PROTECT__
	186	uint64_t asid = start >> TLBI_ASID_SHIFT;
	187	/*
	188	* If we are flushing ASID 0, this is a kernel operation. With this
	189	* ASID scheme, this means we should flush all ASIDs.
	190	*/
	191	if (asid == 0) {
f427ee49	192	for (; start < end; start += (pmap_page_size / 4096)) {
cb323159 A	193	asm volatile ("tlbi vaae1is, %0" : : "r"(start));
	194	}
	195	return;
	196	}
	197	start = start \| (1ULL << TLBI_ASID_SHIFT);
	198	end = end \| (1ULL << TLBI_ASID_SHIFT);
f427ee49	199	for (; start < end; start += (pmap_page_size / 4096)) {
cb323159 A	200	start = start & ~(1ULL << TLBI_ASID_SHIFT);
	201	asm volatile ("tlbi vae1is, %0" : : "r"(start));
	202	start = start \| (1ULL << TLBI_ASID_SHIFT);
	203	asm volatile ("tlbi vae1is, %0" : : "r"(start));
	204	}
	205	#else
f427ee49	206	for (; start < end; start += (pmap_page_size / 4096)) {
cb323159 A	207	asm volatile ("tlbi vae1is, %0" : : "r"(start));
	208	}
	209	#endif /* __ARM_KERNEL_PROTECT__ */
	210	}
	211
	212	static inline void
f427ee49	213	flush_mmu_tlb_entries(uint64_t start, uint64_t end, uint64_t pmap_page_size)
cb323159	214	{
f427ee49	215	flush_mmu_tlb_entries_async(start, end, pmap_page_size);
cb323159 A	216	sync_tlb_flush();
	217	}
	218
	219	// flush_mmu_tlb_asid: flush all entries that match an ASID, on all cores
	220	// ASID must be in high 16 bits of argument
	221	// Will not flush global entries
	222	static inline void
	223	flush_mmu_tlb_asid_async(uint64_t val)
	224	{
	225	#if __ARM_KERNEL_PROTECT__
	226	/*
	227	* If we are flushing ASID 0, this is a kernel operation. With this
	228	* ASID scheme, this means we should flush all ASIDs.
	229	*/
	230	uint64_t asid = val >> TLBI_ASID_SHIFT;
	231	if (asid == 0) {
	232	asm volatile ("tlbi vmalle1is");
	233	return;
	234	}
	235	val = val & ~(1ULL << TLBI_ASID_SHIFT);
	236	asm volatile ("tlbi aside1is, %0" : : "r"(val));
	237	val = val \| (1ULL << TLBI_ASID_SHIFT);
	238	#endif /* __ARM_KERNEL_PROTECT__ */
	239	asm volatile ("tlbi aside1is, %0" : : "r"(val));
	240	}
	241
	242	static inline void
	243	flush_mmu_tlb_asid(uint64_t val)
	244	{
	245	flush_mmu_tlb_asid_async(val);
	246	sync_tlb_flush();
	247	}
	248
	249	// flush_core_tlb_asid: flush all entries that match an ASID, local core only
	250	// ASID must be in high 16 bits of argument
	251	// Will not flush global entries
	252	static inline void
	253	flush_core_tlb_asid_async(uint64_t val)
	254	{
	255	#if __ARM_KERNEL_PROTECT__
	256	/*
	257	* If we are flushing ASID 0, this is a kernel operation. With this
	258	* ASID scheme, this means we should flush all ASIDs.
	259	*/
	260	uint64_t asid = val >> TLBI_ASID_SHIFT;
	261	if (asid == 0) {
	262	asm volatile ("tlbi vmalle1");
	263	return;
	264	}
	265	val = val & ~(1ULL << TLBI_ASID_SHIFT);
	266	asm volatile ("tlbi aside1, %0" : : "r"(val));
	267	val = val \| (1ULL << TLBI_ASID_SHIFT);
	268	#endif /* __ARM_KERNEL_PROTECT__ */
	269	asm volatile ("tlbi aside1, %0" : : "r"(val));
	270	}
	271
	272	static inline void
	273	flush_core_tlb_asid(uint64_t val)
	274	{
	275	flush_core_tlb_asid_async(val);
	276	sync_tlb_flush();
	277	}
	278
	279	#if __ARM_RANGE_TLBI__
280	#if __ARM_KERNEL_PROTECT__
281	#error __ARM_RANGE_TLBI__ + __ARM_KERNEL_PROTECT__ is not currently supported
282	#endif
283
f427ee49 A	284	#define ARM64_TLB_RANGE_PAGES (1ULL << 21)
f427ee49 A	285	#define rtlbi_addr(x, shift) (((x) >> (shift)) & RTLBI_ADDR_MASK)
cb323159 A	286	#define rtlbi_scale(x) ((uint64_t)(x) << RTLBI_SCALE_SHIFT)
	287	#define rtlbi_num(x) ((uint64_t)(x) << RTLBI_NUM_SHIFT)
	288
	289	/**
	290	* Given the number of pages to invalidate, generate the correct parameter to
	291	* pass to any of the TLBI by range methods.
	292	*/
	293	static inline uint64_t
f427ee49	294	generate_rtlbi_param(ppnum_t npages, uint32_t asid, vm_offset_t va, uint64_t pmap_page_shift)
cb323159 A	295	{
	296	/**
	297	* Per the armv8.4 RTLBI extension spec, the range encoded in the rtlbi register operand is defined by:
	298	* BaseADDR <= VA < BaseADDR+((NUM+1)2^(5SCALE+1) * Translation_Granule_Size)
	299	*/
2a1bd2d3	300	unsigned order = (unsigned)(sizeof(npages) * 8) - (unsigned)__builtin_clz(npages - 1) - 1;
cb323159 A	301	unsigned scale = ((order ? order : 1) - 1) / 5;
	302	unsigned granule = 1 << ((5 * scale) + 1);
	303	unsigned num = (((npages + granule - 1) & ~(granule - 1)) / granule) - 1;
f427ee49	304	return tlbi_asid(asid) \| RTLBI_TG(pmap_page_shift) \| rtlbi_scale(scale) \| rtlbi_num(num) \| rtlbi_addr(va, pmap_page_shift);
cb323159 A	305	}
	306
	307	// flush_mmu_tlb_range: flush TLB entries that map a VA range using a single instruction
	308	// The argument should be encoded according to generate_rtlbi_param().
	309	// Follows the same ASID matching behavior as flush_mmu_tlb_entries()
	310	static inline void
	311	flush_mmu_tlb_range_async(uint64_t val)
	312	{
	313	asm volatile ("tlbi rvae1is, %0" : : "r"(val));
	314	}
	315
	316	static inline void
	317	flush_mmu_tlb_range(uint64_t val)
	318	{
	319	flush_mmu_tlb_range_async(val);
	320	sync_tlb_flush();
	321	}
	322
	323	// flush_mmu_tlb_allrange: flush TLB entries that map a VA range using a single instruction
	324	// The argument should be encoded according to generate_rtlbi_param().
	325	// Follows the same ASID matching behavior as flush_mmu_tlb_allentries()
	326	static inline void
	327	flush_mmu_tlb_allrange_async(uint64_t val)
	328	{
	329	asm volatile ("tlbi rvaae1is, %0" : : "r"(val));
	330	}
	331
	332	static inline void
	333	flush_mmu_tlb_allrange(uint64_t val)
	334	{
	335	flush_mmu_tlb_allrange_async(val);
	336	sync_tlb_flush();
	337	}
	338
f427ee49 A	339	// flush_core_tlb_allrange: flush TLB entries that map a VA range using a single instruction, local core only
	340	// The argument should be encoded according to generate_rtlbi_param().
	341	// Follows the same ASID matching behavior as flush_mmu_tlb_allentries()
	342	static inline void
	343	flush_core_tlb_allrange_async(uint64_t val)
	344	{
	345	asm volatile ("tlbi rvaae1, %0" : : "r"(val));
	346	}
	347
	348	static inline void
	349	flush_core_tlb_allrange(uint64_t val)
	350	{
	351	flush_core_tlb_allrange_async(val);
	352	sync_tlb_flush();
	353	}
	354
cb323159 A	355	#endif // __ARM_RANGE_TLBI__
	356
	357