__builtin_arm_isb(ISB_SY);
}
+static inline void
+sync_tlb_flush_local(void)
+{
+ __builtin_arm_dsb(DSB_NSH);
+ __builtin_arm_isb(ISB_SY);
+}
+
// flush_mmu_tlb: full TLB flush on all cores
static inline void
flush_mmu_tlb_async(void)
// flush_mmu_tlb_allentries_async: flush entries that map VA range, all ASIDS, all cores
// start and end are in units of 4K pages.
static inline void
-flush_mmu_tlb_allentries_async(uint64_t start, uint64_t end)
+flush_mmu_tlb_allentries_async(uint64_t start, uint64_t end, uint64_t pmap_page_size)
{
#if __ARM_16K_PG__
- start = start & ~0x3ULL;
+ if (pmap_page_size == 16384) {
+ start = start & ~0x3ULL;
- /*
- * The code below is not necessarily correct. From an overview of
- * the client code, the expected contract for TLB flushes is that
- * we will expand from an "address, length" pair to "start address,
- * end address" in the course of a TLB flush. This suggests that
- * a flush for "X, X+4" is actually only asking for a flush of a
- * single 16KB page. At the same time, we'd like to be prepared
- * for bad inputs (X, X+3), so add 3 and then truncate the 4KB page
- * number to a 16KB page boundary. This should deal correctly with
- * unaligned inputs.
- *
- * If our expecations about client behavior are wrong however, this
- * will lead to occasional TLB corruption on platforms with 16KB
- * pages.
- */
- end = (end + 0x3ULL) & ~0x3ULL;
+ /*
+ * The code below is not necessarily correct. From an overview of
+ * the client code, the expected contract for TLB flushes is that
+ * we will expand from an "address, length" pair to "start address,
+ * end address" in the course of a TLB flush. This suggests that
+ * a flush for "X, X+4" is actually only asking for a flush of a
+ * single 16KB page. At the same time, we'd like to be prepared
+ * for bad inputs (X, X+3), so add 3 and then truncate the 4KB page
+ * number to a 16KB page boundary. This should deal correctly with
+ * unaligned inputs.
+ *
+ * If our expecations about client behavior are wrong however, this
+ * will lead to occasional TLB corruption on platforms with 16KB
+ * pages.
+ */
+ end = (end + 0x3ULL) & ~0x3ULL;
+ }
#endif // __ARM_16K_PG__
- for (; start < end; start += (ARM_PGBYTES / 4096)) {
+ for (; start < end; start += (pmap_page_size / 4096)) {
asm volatile ("tlbi vaae1is, %0" : : "r"(start));
}
}
static inline void
-flush_mmu_tlb_allentries(uint64_t start, uint64_t end)
+flush_mmu_tlb_allentries(uint64_t start, uint64_t end, uint64_t pmap_page_size)
{
- flush_mmu_tlb_allentries_async(start, end);
+ flush_mmu_tlb_allentries_async(start, end, pmap_page_size);
sync_tlb_flush();
}
// start and end must have the ASID in the high 16 bits, with the VA in units of 4K in the lowest bits
// Will also flush global entries that match the VA range
static inline void
-flush_mmu_tlb_entries_async(uint64_t start, uint64_t end)
+flush_mmu_tlb_entries_async(uint64_t start, uint64_t end, uint64_t pmap_page_size)
{
#if __ARM_16K_PG__
- start = start & ~0x3ULL;
+ if (pmap_page_size == 16384) {
+ start = start & ~0x3ULL;
- /*
- * The code below is not necessarily correct. From an overview of
- * the client code, the expected contract for TLB flushes is that
- * we will expand from an "address, length" pair to "start address,
- * end address" in the course of a TLB flush. This suggests that
- * a flush for "X, X+4" is actually only asking for a flush of a
- * single 16KB page. At the same time, we'd like to be prepared
- * for bad inputs (X, X+3), so add 3 and then truncate the 4KB page
- * number to a 16KB page boundary. This should deal correctly with
- * unaligned inputs.
- *
- * If our expecations about client behavior are wrong however, this
- * will lead to occasional TLB corruption on platforms with 16KB
- * pages.
- */
- end = (end + 0x3ULL) & ~0x3ULL;
+ /*
+ * The code below is not necessarily correct. From an overview of
+ * the client code, the expected contract for TLB flushes is that
+ * we will expand from an "address, length" pair to "start address,
+ * end address" in the course of a TLB flush. This suggests that
+ * a flush for "X, X+4" is actually only asking for a flush of a
+ * single 16KB page. At the same time, we'd like to be prepared
+ * for bad inputs (X, X+3), so add 3 and then truncate the 4KB page
+ * number to a 16KB page boundary. This should deal correctly with
+ * unaligned inputs.
+ *
+ * If our expecations about client behavior are wrong however, this
+ * will lead to occasional TLB corruption on platforms with 16KB
+ * pages.
+ */
+ end = (end + 0x3ULL) & ~0x3ULL;
+ }
#endif // __ARM_16K_PG__
#if __ARM_KERNEL_PROTECT__
uint64_t asid = start >> TLBI_ASID_SHIFT;
* ASID scheme, this means we should flush all ASIDs.
*/
if (asid == 0) {
- for (; start < end; start += (ARM_PGBYTES / 4096)) {
+ for (; start < end; start += (pmap_page_size / 4096)) {
asm volatile ("tlbi vaae1is, %0" : : "r"(start));
}
return;
}
start = start | (1ULL << TLBI_ASID_SHIFT);
end = end | (1ULL << TLBI_ASID_SHIFT);
- for (; start < end; start += (ARM_PGBYTES / 4096)) {
+ for (; start < end; start += (pmap_page_size / 4096)) {
start = start & ~(1ULL << TLBI_ASID_SHIFT);
asm volatile ("tlbi vae1is, %0" : : "r"(start));
start = start | (1ULL << TLBI_ASID_SHIFT);
asm volatile ("tlbi vae1is, %0" : : "r"(start));
}
#else
- for (; start < end; start += (ARM_PGBYTES / 4096)) {
+ for (; start < end; start += (pmap_page_size / 4096)) {
asm volatile ("tlbi vae1is, %0" : : "r"(start));
}
#endif /* __ARM_KERNEL_PROTECT__ */
}
static inline void
-flush_mmu_tlb_entries(uint64_t start, uint64_t end)
+flush_mmu_tlb_entries(uint64_t start, uint64_t end, uint64_t pmap_page_size)
{
- flush_mmu_tlb_entries_async(start, end);
+ flush_mmu_tlb_entries_async(start, end, pmap_page_size);
sync_tlb_flush();
}
#error __ARM_RANGE_TLBI__ + __ARM_KERNEL_PROTECT__ is not currently supported
#endif
-#define ARM64_16K_TLB_RANGE_PAGES (1ULL << 21)
-#define rtlbi_addr(x) (((x) >> RTLBI_ADDR_SHIFT) & RTLBI_ADDR_MASK)
+#define ARM64_TLB_RANGE_PAGES (1ULL << 21)
+#define rtlbi_addr(x, shift) (((x) >> (shift)) & RTLBI_ADDR_MASK)
#define rtlbi_scale(x) ((uint64_t)(x) << RTLBI_SCALE_SHIFT)
#define rtlbi_num(x) ((uint64_t)(x) << RTLBI_NUM_SHIFT)
* pass to any of the TLBI by range methods.
*/
static inline uint64_t
-generate_rtlbi_param(ppnum_t npages, uint32_t asid, vm_offset_t va)
+generate_rtlbi_param(ppnum_t npages, uint32_t asid, vm_offset_t va, uint64_t pmap_page_shift)
{
/**
* Per the armv8.4 RTLBI extension spec, the range encoded in the rtlbi register operand is defined by:
* BaseADDR <= VA < BaseADDR+((NUM+1)*2^(5*SCALE+1) * Translation_Granule_Size)
*/
- unsigned order = (sizeof(npages) * 8) - __builtin_clz(npages - 1) - 1;
+ unsigned order = (unsigned)(sizeof(npages) * 8) - (unsigned)__builtin_clz(npages - 1) - 1;
unsigned scale = ((order ? order : 1) - 1) / 5;
unsigned granule = 1 << ((5 * scale) + 1);
unsigned num = (((npages + granule - 1) & ~(granule - 1)) / granule) - 1;
- return tlbi_asid(asid) | RTLBI_TG | rtlbi_scale(scale) | rtlbi_num(num) | rtlbi_addr(va);
+ return tlbi_asid(asid) | RTLBI_TG(pmap_page_shift) | rtlbi_scale(scale) | rtlbi_num(num) | rtlbi_addr(va, pmap_page_shift);
}
// flush_mmu_tlb_range: flush TLB entries that map a VA range using a single instruction
sync_tlb_flush();
}
+// flush_core_tlb_allrange: flush TLB entries that map a VA range using a single instruction, local core only
+// The argument should be encoded according to generate_rtlbi_param().
+// Follows the same ASID matching behavior as flush_mmu_tlb_allentries()
+static inline void
+flush_core_tlb_allrange_async(uint64_t val)
+{
+ asm volatile ("tlbi rvaae1, %0" : : "r"(val));
+}
+
+static inline void
+flush_core_tlb_allrange(uint64_t val)
+{
+ flush_core_tlb_allrange_async(val);
+ sync_tlb_flush();
+}
+
#endif // __ARM_RANGE_TLBI__
projects / apple / xnu.git / blobdiff