xnu-7195.101.1.tar.gz

[apple/xnu.git] / osfmk / arm64 / tlb.h

diff --git a/osfmk/arm64/tlb.h b/osfmk/arm64/tlb.h
index eb1face77e836b7665b26c8c87d75e1f2d64d975..aa3a73c8e53126623f5d829426d2ca22ac20a501 100644 (file)
--- a/osfmk/arm64/tlb.h
+++ b/osfmk/arm64/tlb.h
@@ -55,6 +55,13 @@ sync_tlb_flush(void)
        __builtin_arm_isb(ISB_SY);
 }
 
        __builtin_arm_isb(ISB_SY);
 }
 

+static inline void
+sync_tlb_flush_local(void)
+{
+       __builtin_arm_dsb(DSB_NSH);
+       __builtin_arm_isb(ISB_SY);
+}
+

 // flush_mmu_tlb: full TLB flush on all cores
 static inline void
 flush_mmu_tlb_async(void)
 // flush_mmu_tlb: full TLB flush on all cores
 static inline void
 flush_mmu_tlb_async(void)


@@ -86,37 +93,39 @@ flush_core_tlb(void)
 // flush_mmu_tlb_allentries_async: flush entries that map VA range, all ASIDS, all cores
 // start and end are in units of 4K pages.
 static inline void
 // flush_mmu_tlb_allentries_async: flush entries that map VA range, all ASIDS, all cores
 // start and end are in units of 4K pages.
 static inline void

-flush_mmu_tlb_allentries_async(uint64_t start, uint64_t end)
+flush_mmu_tlb_allentries_async(uint64_t start, uint64_t end, uint64_t pmap_page_size)

 {
 #if __ARM_16K_PG__
 {
 #if __ARM_16K_PG__

-       start = start & ~0x3ULL;
+       if (pmap_page_size == 16384) {
+               start = start & ~0x3ULL;

 
 

-       /*
-        * The code below is not necessarily correct.  From an overview of
-        * the client code, the expected contract for TLB flushes is that
-        * we will expand from an "address, length" pair to "start address,
-        * end address" in the course of a TLB flush.  This suggests that
-        * a flush for "X, X+4" is actually only asking for a flush of a
-        * single 16KB page.  At the same time, we'd like to be prepared
-        * for bad inputs (X, X+3), so add 3 and then truncate the 4KB page
-        * number to a 16KB page boundary.  This should deal correctly with
-        * unaligned inputs.
-        *
-        * If our expecations about client behavior are wrong however, this
-        * will lead to occasional TLB corruption on platforms with 16KB
-        * pages.
-        */
-       end = (end + 0x3ULL) & ~0x3ULL;
+               /*
+                * The code below is not necessarily correct.  From an overview of
+                * the client code, the expected contract for TLB flushes is that
+                * we will expand from an "address, length" pair to "start address,
+                * end address" in the course of a TLB flush.  This suggests that
+                * a flush for "X, X+4" is actually only asking for a flush of a
+                * single 16KB page.  At the same time, we'd like to be prepared
+                * for bad inputs (X, X+3), so add 3 and then truncate the 4KB page
+                * number to a 16KB page boundary.  This should deal correctly with
+                * unaligned inputs.
+                *
+                * If our expecations about client behavior are wrong however, this
+                * will lead to occasional TLB corruption on platforms with 16KB
+                * pages.
+                */
+               end = (end + 0x3ULL) & ~0x3ULL;
+       }

 #endif // __ARM_16K_PG__
 #endif // __ARM_16K_PG__

-       for (; start < end; start += (ARM_PGBYTES / 4096)) {
+       for (; start < end; start += (pmap_page_size / 4096)) {

                asm volatile ("tlbi vaae1is, %0" : : "r"(start));
        }
 }
 
 static inline void
                asm volatile ("tlbi vaae1is, %0" : : "r"(start));
        }
 }
 
 static inline void

-flush_mmu_tlb_allentries(uint64_t start, uint64_t end)
+flush_mmu_tlb_allentries(uint64_t start, uint64_t end, uint64_t pmap_page_size)

 {
 {

-       flush_mmu_tlb_allentries_async(start, end);
+       flush_mmu_tlb_allentries_async(start, end, pmap_page_size);

        sync_tlb_flush();
 }
 
        sync_tlb_flush();
 }
 


@@ -149,27 +158,29 @@ flush_mmu_tlb_entry(uint64_t val)
 // start and end must have the ASID in the high 16 bits, with the VA in units of 4K in the lowest bits
 // Will also flush global entries that match the VA range
 static inline void
 // start and end must have the ASID in the high 16 bits, with the VA in units of 4K in the lowest bits
 // Will also flush global entries that match the VA range
 static inline void

-flush_mmu_tlb_entries_async(uint64_t start, uint64_t end)
+flush_mmu_tlb_entries_async(uint64_t start, uint64_t end, uint64_t pmap_page_size)

 {
 #if __ARM_16K_PG__
 {
 #if __ARM_16K_PG__

-       start = start & ~0x3ULL;
+       if (pmap_page_size == 16384) {
+               start = start & ~0x3ULL;

 
 

-       /*
-        * The code below is not necessarily correct.  From an overview of
-        * the client code, the expected contract for TLB flushes is that
-        * we will expand from an "address, length" pair to "start address,
-        * end address" in the course of a TLB flush.  This suggests that
-        * a flush for "X, X+4" is actually only asking for a flush of a
-        * single 16KB page.  At the same time, we'd like to be prepared
-        * for bad inputs (X, X+3), so add 3 and then truncate the 4KB page
-        * number to a 16KB page boundary.  This should deal correctly with
-        * unaligned inputs.
-        *
-        * If our expecations about client behavior are wrong however, this
-        * will lead to occasional TLB corruption on platforms with 16KB
-        * pages.
-        */
-       end = (end + 0x3ULL) & ~0x3ULL;
+               /*
+                * The code below is not necessarily correct.  From an overview of
+                * the client code, the expected contract for TLB flushes is that
+                * we will expand from an "address, length" pair to "start address,
+                * end address" in the course of a TLB flush.  This suggests that
+                * a flush for "X, X+4" is actually only asking for a flush of a
+                * single 16KB page.  At the same time, we'd like to be prepared
+                * for bad inputs (X, X+3), so add 3 and then truncate the 4KB page
+                * number to a 16KB page boundary.  This should deal correctly with
+                * unaligned inputs.
+                *
+                * If our expecations about client behavior are wrong however, this
+                * will lead to occasional TLB corruption on platforms with 16KB
+                * pages.
+                */
+               end = (end + 0x3ULL) & ~0x3ULL;
+       }

 #endif // __ARM_16K_PG__
 #if __ARM_KERNEL_PROTECT__
        uint64_t asid = start >> TLBI_ASID_SHIFT;
 #endif // __ARM_16K_PG__
 #if __ARM_KERNEL_PROTECT__
        uint64_t asid = start >> TLBI_ASID_SHIFT;


@@ -178,30 +189,30 @@ flush_mmu_tlb_entries_async(uint64_t start, uint64_t end)
         * ASID scheme, this means we should flush all ASIDs.
         */
        if (asid == 0) {
         * ASID scheme, this means we should flush all ASIDs.
         */
        if (asid == 0) {

-               for (; start < end; start += (ARM_PGBYTES / 4096)) {
+               for (; start < end; start += (pmap_page_size / 4096)) {

                        asm volatile ("tlbi vaae1is, %0" : : "r"(start));
                }
                return;
        }
        start = start | (1ULL << TLBI_ASID_SHIFT);
        end = end | (1ULL << TLBI_ASID_SHIFT);
                        asm volatile ("tlbi vaae1is, %0" : : "r"(start));
                }
                return;
        }
        start = start | (1ULL << TLBI_ASID_SHIFT);
        end = end | (1ULL << TLBI_ASID_SHIFT);

-       for (; start < end; start += (ARM_PGBYTES / 4096)) {
+       for (; start < end; start += (pmap_page_size / 4096)) {

                start = start & ~(1ULL << TLBI_ASID_SHIFT);
                asm volatile ("tlbi vae1is, %0" : : "r"(start));
                start = start | (1ULL << TLBI_ASID_SHIFT);
                asm volatile ("tlbi vae1is, %0" : : "r"(start));
        }
 #else
                start = start & ~(1ULL << TLBI_ASID_SHIFT);
                asm volatile ("tlbi vae1is, %0" : : "r"(start));
                start = start | (1ULL << TLBI_ASID_SHIFT);
                asm volatile ("tlbi vae1is, %0" : : "r"(start));
        }
 #else

-       for (; start < end; start += (ARM_PGBYTES / 4096)) {
+       for (; start < end; start += (pmap_page_size / 4096)) {

                asm volatile ("tlbi vae1is, %0" : : "r"(start));
        }
 #endif /* __ARM_KERNEL_PROTECT__ */
 }
 
 static inline void
                asm volatile ("tlbi vae1is, %0" : : "r"(start));
        }
 #endif /* __ARM_KERNEL_PROTECT__ */
 }
 
 static inline void

-flush_mmu_tlb_entries(uint64_t start, uint64_t end)
+flush_mmu_tlb_entries(uint64_t start, uint64_t end, uint64_t pmap_page_size)

 {
 {

-       flush_mmu_tlb_entries_async(start, end);
+       flush_mmu_tlb_entries_async(start, end, pmap_page_size);

        sync_tlb_flush();
 }
 
        sync_tlb_flush();
 }
 


@@ -270,8 +281,8 @@ flush_core_tlb_asid(uint64_t val)
        #error __ARM_RANGE_TLBI__ + __ARM_KERNEL_PROTECT__ is not currently supported
 #endif
 
        #error __ARM_RANGE_TLBI__ + __ARM_KERNEL_PROTECT__ is not currently supported
 #endif
 

-#define ARM64_16K_TLB_RANGE_PAGES (1ULL << 21)
-#define rtlbi_addr(x) (((x) >> RTLBI_ADDR_SHIFT) & RTLBI_ADDR_MASK)
+#define ARM64_TLB_RANGE_PAGES (1ULL << 21)
+#define rtlbi_addr(x, shift) (((x) >> (shift)) & RTLBI_ADDR_MASK)

 #define rtlbi_scale(x) ((uint64_t)(x) << RTLBI_SCALE_SHIFT)
 #define rtlbi_num(x) ((uint64_t)(x) << RTLBI_NUM_SHIFT)
 
 #define rtlbi_scale(x) ((uint64_t)(x) << RTLBI_SCALE_SHIFT)
 #define rtlbi_num(x) ((uint64_t)(x) << RTLBI_NUM_SHIFT)
 


@@ -280,17 +291,17 @@ flush_core_tlb_asid(uint64_t val)
  * pass to any of the TLBI by range methods.
  */
 static inline uint64_t
  * pass to any of the TLBI by range methods.
  */
 static inline uint64_t

-generate_rtlbi_param(ppnum_t npages, uint32_t asid, vm_offset_t va)
+generate_rtlbi_param(ppnum_t npages, uint32_t asid, vm_offset_t va, uint64_t pmap_page_shift)

 {
        /**
         * Per the armv8.4 RTLBI extension spec, the range encoded in the rtlbi register operand is defined by:
         * BaseADDR <= VA < BaseADDR+((NUM+1)*2^(5*SCALE+1) * Translation_Granule_Size)
         */
 {
        /**
         * Per the armv8.4 RTLBI extension spec, the range encoded in the rtlbi register operand is defined by:
         * BaseADDR <= VA < BaseADDR+((NUM+1)*2^(5*SCALE+1) * Translation_Granule_Size)
         */

-       unsigned order = (sizeof(npages) * 8) - __builtin_clz(npages - 1) - 1;
+       unsigned order = (unsigned)(sizeof(npages) * 8) - (unsigned)__builtin_clz(npages - 1) - 1;

        unsigned scale = ((order ? order : 1) - 1) / 5;
        unsigned granule = 1 << ((5 * scale) + 1);
        unsigned num = (((npages + granule - 1) & ~(granule - 1)) / granule) - 1;
        unsigned scale = ((order ? order : 1) - 1) / 5;
        unsigned granule = 1 << ((5 * scale) + 1);
        unsigned num = (((npages + granule - 1) & ~(granule - 1)) / granule) - 1;

-       return tlbi_asid(asid) | RTLBI_TG | rtlbi_scale(scale) | rtlbi_num(num) | rtlbi_addr(va);
+       return tlbi_asid(asid) | RTLBI_TG(pmap_page_shift) | rtlbi_scale(scale) | rtlbi_num(num) | rtlbi_addr(va, pmap_page_shift);

 }
 
 // flush_mmu_tlb_range: flush TLB entries that map a VA range using a single instruction
 }
 
 // flush_mmu_tlb_range: flush TLB entries that map a VA range using a single instruction


@@ -325,6 +336,22 @@ flush_mmu_tlb_allrange(uint64_t val)
        sync_tlb_flush();
 }
 
        sync_tlb_flush();
 }
 

+// flush_core_tlb_allrange: flush TLB entries that map a VA range using a single instruction, local core only
+// The argument should be encoded according to generate_rtlbi_param().
+// Follows the same ASID matching behavior as flush_mmu_tlb_allentries()
+static inline void
+flush_core_tlb_allrange_async(uint64_t val)
+{
+       asm volatile ("tlbi rvaae1, %0" : : "r"(val));
+}
+
+static inline void
+flush_core_tlb_allrange(uint64_t val)
+{
+       flush_core_tlb_allrange_async(val);
+       sync_tlb_flush();
+}
+

 #endif // __ARM_RANGE_TLBI__
 
 
 #endif // __ARM_RANGE_TLBI__