osfmk/arm64/tlb.h

   1 /*
   2  * Copyright (c) 2019 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #pragma once
  30
  31 #include <arm64/proc_reg.h>
  32 #include <machine/atomic.h>
  33
  34 #define tlbi_addr(x) ((((x) >> 12) & TLBI_ADDR_MASK) << TLBI_ADDR_SHIFT)
  35 #define tlbi_asid(x) (((uintptr_t)(x) & TLBI_ASID_MASK) << TLBI_ASID_SHIFT)
  36
  37 #if __ARM_KERNEL_PROTECT__
  38 /*
  39  * __ARM_KERNEL_PROTECT__ adds two complications to TLB management:
  40  *
  41  * 1. As each pmap has two ASIDs, every TLB operation that targets an ASID must
  42  *   target both ASIDs for the pmap that owns the target ASID.
  43  *
  44  * 2. Any TLB operation targeting the kernel_pmap ASID (ASID 0) must target all
  45  *   ASIDs (as kernel_pmap mappings may be referenced while using an ASID that
  46  *   belongs to another pmap).  We expect these routines to be called with the
  47  *   EL0 ASID for the target; not the EL1 ASID.
  48  */
  49 #endif /* __ARM_KERNEL_PROTECT__ */
  50
  51 static inline void
  52 sync_tlb_flush(void)
  53 {
  54         __builtin_arm_dsb(DSB_ISH);
  55         __builtin_arm_isb(ISB_SY);
  56 }
  57
  58 // flush_mmu_tlb: full TLB flush on all cores
  59 static inline void
  60 flush_mmu_tlb_async(void)
  61 {
  62         asm volatile ("tlbi vmalle1is");
  63 }
  64
  65 static inline void
  66 flush_mmu_tlb(void)
  67 {
  68         flush_mmu_tlb_async();
  69         sync_tlb_flush();
  70 }
  71
  72 // flush_core_tlb: full TLB flush on local core only
  73 static inline void
  74 flush_core_tlb_async(void)
  75 {
  76         asm volatile ("tlbi vmalle1");
  77 }
  78
  79 static inline void
  80 flush_core_tlb(void)
  81 {
  82         flush_core_tlb_async();
  83         sync_tlb_flush();
  84 }
  85
  86 // flush_mmu_tlb_allentries_async: flush entries that map VA range, all ASIDS, all cores
  87 // start and end are in units of 4K pages.
  88 static inline void
  89 flush_mmu_tlb_allentries_async(uint64_t start, uint64_t end, uint64_t pmap_page_size)
  90 {
  91 #if __ARM_16K_PG__
  92         if (pmap_page_size == 16384) {
  93                 start = start & ~0x3ULL;
  94
  95                 /*
  96                  * The code below is not necessarily correct.  From an overview of
  97                  * the client code, the expected contract for TLB flushes is that
  98                  * we will expand from an "address, length" pair to "start address,
  99                  * end address" in the course of a TLB flush.  This suggests that
 100                  * a flush for "X, X+4" is actually only asking for a flush of a
 101                  * single 16KB page.  At the same time, we'd like to be prepared
 102                  * for bad inputs (X, X+3), so add 3 and then truncate the 4KB page
 103                  * number to a 16KB page boundary.  This should deal correctly with
 104                  * unaligned inputs.
 105                  *
 106                  * If our expecations about client behavior are wrong however, this
 107                  * will lead to occasional TLB corruption on platforms with 16KB
 108                  * pages.
 109                  */
 110                 end = (end + 0x3ULL) & ~0x3ULL;
 111         }
 112 #endif // __ARM_16K_PG__
 113         for (; start < end; start += (pmap_page_size / 4096)) {
 114                 asm volatile ("tlbi vaae1is, %0" : : "r"(start));
 115         }
 116 }
 117
 118 static inline void
 119 flush_mmu_tlb_allentries(uint64_t start, uint64_t end, uint64_t pmap_page_size)
 120 {
 121         flush_mmu_tlb_allentries_async(start, end, pmap_page_size);
 122         sync_tlb_flush();
 123 }
 124
 125 // flush_mmu_tlb_entry: flush TLB entries that map a VA and ASID, all cores
 126 // Will also flush global entries that match the VA
 127 static inline void
 128 flush_mmu_tlb_entry_async(uint64_t val)
 129 {
 130 #if __ARM_KERNEL_PROTECT__
 131         uint64_t asid = val >> TLBI_ASID_SHIFT;
 132         if (asid == 0) {
 133                 asm volatile ("tlbi vaae1is, %0" : : "r"(val));
 134                 return;
 135         }
 136         val = val & ~(1ULL << TLBI_ASID_SHIFT);
 137         asm volatile ("tlbi vae1is, %0" : : "r"(val));
 138         val = val | (1ULL << TLBI_ASID_SHIFT);
 139 #endif /* __ARM_KERNEL_PROTECT__ */
 140         asm volatile ("tlbi vae1is, %0" : : "r"(val));
 141 }
 142
 143 static inline void
 144 flush_mmu_tlb_entry(uint64_t val)
 145 {
 146         flush_mmu_tlb_entry_async(val);
 147         sync_tlb_flush();
 148 }
 149
 150 // flush_mmu_tlb_entries: flush TLB entries that map a VA range and ASID, all cores
 151 // start and end must have the ASID in the high 16 bits, with the VA in units of 4K in the lowest bits
 152 // Will also flush global entries that match the VA range
 153 static inline void
 154 flush_mmu_tlb_entries_async(uint64_t start, uint64_t end, uint64_t pmap_page_size)
 155 {
 156 #if __ARM_16K_PG__
 157         if (pmap_page_size == 16384) {
 158                 start = start & ~0x3ULL;
 159
 160                 /*
 161                  * The code below is not necessarily correct.  From an overview of
 162                  * the client code, the expected contract for TLB flushes is that
 163                  * we will expand from an "address, length" pair to "start address,
 164                  * end address" in the course of a TLB flush.  This suggests that
 165                  * a flush for "X, X+4" is actually only asking for a flush of a
 166                  * single 16KB page.  At the same time, we'd like to be prepared
 167                  * for bad inputs (X, X+3), so add 3 and then truncate the 4KB page
 168                  * number to a 16KB page boundary.  This should deal correctly with
 169                  * unaligned inputs.
 170                  *
 171                  * If our expecations about client behavior are wrong however, this
 172                  * will lead to occasional TLB corruption on platforms with 16KB
 173                  * pages.
 174                  */
 175                 end = (end + 0x3ULL) & ~0x3ULL;
 176         }
 177 #endif // __ARM_16K_PG__
 178 #if __ARM_KERNEL_PROTECT__
 179         uint64_t asid = start >> TLBI_ASID_SHIFT;
 180         /*
 181          * If we are flushing ASID 0, this is a kernel operation.  With this
 182          * ASID scheme, this means we should flush all ASIDs.
 183          */
 184         if (asid == 0) {
 185                 for (; start < end; start += (pmap_page_size / 4096)) {
 186                         asm volatile ("tlbi vaae1is, %0" : : "r"(start));
 187                 }
 188                 return;
 189         }
 190         start = start | (1ULL << TLBI_ASID_SHIFT);
 191         end = end | (1ULL << TLBI_ASID_SHIFT);
 192         for (; start < end; start += (pmap_page_size / 4096)) {
 193                 start = start & ~(1ULL << TLBI_ASID_SHIFT);
 194                 asm volatile ("tlbi vae1is, %0" : : "r"(start));
 195                 start = start | (1ULL << TLBI_ASID_SHIFT);
 196                 asm volatile ("tlbi vae1is, %0" : : "r"(start));
 197         }
 198 #else
 199         for (; start < end; start += (pmap_page_size / 4096)) {
 200                 asm volatile ("tlbi vae1is, %0" : : "r"(start));
 201         }
 202 #endif /* __ARM_KERNEL_PROTECT__ */
 203 }
 204
 205 static inline void
 206 flush_mmu_tlb_entries(uint64_t start, uint64_t end, uint64_t pmap_page_size)
 207 {
 208         flush_mmu_tlb_entries_async(start, end, pmap_page_size);
 209         sync_tlb_flush();
 210 }
 211
 212 // flush_mmu_tlb_asid: flush all entries that match an ASID, on all cores
 213 // ASID must be in high 16 bits of argument
 214 // Will not flush global entries
 215 static inline void
 216 flush_mmu_tlb_asid_async(uint64_t val)
 217 {
 218 #if __ARM_KERNEL_PROTECT__
 219         /*
 220          * If we are flushing ASID 0, this is a kernel operation.  With this
 221          * ASID scheme, this means we should flush all ASIDs.
 222          */
 223         uint64_t asid = val >> TLBI_ASID_SHIFT;
 224         if (asid == 0) {
 225                 asm volatile ("tlbi vmalle1is");
 226                 return;
 227         }
 228         val = val & ~(1ULL << TLBI_ASID_SHIFT);
 229         asm volatile ("tlbi aside1is, %0" : : "r"(val));
 230         val = val | (1ULL << TLBI_ASID_SHIFT);
 231 #endif /* __ARM_KERNEL_PROTECT__ */
 232         asm volatile ("tlbi aside1is, %0" : : "r"(val));
 233 }
 234
 235 static inline void
 236 flush_mmu_tlb_asid(uint64_t val)
 237 {
 238         flush_mmu_tlb_asid_async(val);
 239         sync_tlb_flush();
 240 }
 241
 242 // flush_core_tlb_asid: flush all entries that match an ASID, local core only
 243 // ASID must be in high 16 bits of argument
 244 // Will not flush global entries
 245 static inline void
 246 flush_core_tlb_asid_async(uint64_t val)
 247 {
 248 #if __ARM_KERNEL_PROTECT__
 249         /*
 250          * If we are flushing ASID 0, this is a kernel operation.  With this
 251          * ASID scheme, this means we should flush all ASIDs.
 252          */
 253         uint64_t asid = val >> TLBI_ASID_SHIFT;
 254         if (asid == 0) {
 255                 asm volatile ("tlbi vmalle1");
 256                 return;
 257         }
 258         val = val & ~(1ULL << TLBI_ASID_SHIFT);
 259         asm volatile ("tlbi aside1, %0" : : "r"(val));
 260         val = val | (1ULL << TLBI_ASID_SHIFT);
 261 #endif /* __ARM_KERNEL_PROTECT__ */
 262         asm volatile ("tlbi aside1, %0" : : "r"(val));
 263 }
 264
 265 static inline void
 266 flush_core_tlb_asid(uint64_t val)
 267 {
 268         flush_core_tlb_asid_async(val);
 269         sync_tlb_flush();
 270 }
 271
 272 #if __ARM_RANGE_TLBI__
 273 #if __ARM_KERNEL_PROTECT__
 274         #error __ARM_RANGE_TLBI__ + __ARM_KERNEL_PROTECT__ is not currently supported
 275 #endif
 276
 277 #define ARM64_TLB_RANGE_PAGES (1ULL << 21)
 278 #define rtlbi_addr(x, shift) (((x) >> (shift)) & RTLBI_ADDR_MASK)
 279 #define rtlbi_scale(x) ((uint64_t)(x) << RTLBI_SCALE_SHIFT)
 280 #define rtlbi_num(x) ((uint64_t)(x) << RTLBI_NUM_SHIFT)
 281
 282 /**
 283  * Given the number of pages to invalidate, generate the correct parameter to
 284  * pass to any of the TLBI by range methods.
 285  */
 286 static inline uint64_t
 287 generate_rtlbi_param(ppnum_t npages, uint32_t asid, vm_offset_t va, uint64_t pmap_page_shift)
 288 {
 289         /**
 290          * Per the armv8.4 RTLBI extension spec, the range encoded in the rtlbi register operand is defined by:
 291          * BaseADDR <= VA < BaseADDR+((NUM+1)*2^(5*SCALE+1) * Translation_Granule_Size)
 292          */
 293         unsigned order = (sizeof(npages) * 8) - __builtin_clz(npages - 1) - 1;
 294         unsigned scale = ((order ? order : 1) - 1) / 5;
 295         unsigned granule = 1 << ((5 * scale) + 1);
 296         unsigned num = (((npages + granule - 1) & ~(granule - 1)) / granule) - 1;
 297         return tlbi_asid(asid) | RTLBI_TG(pmap_page_shift) | rtlbi_scale(scale) | rtlbi_num(num) | rtlbi_addr(va, pmap_page_shift);
 298 }
 299
 300 // flush_mmu_tlb_range: flush TLB entries that map a VA range using a single instruction
 301 // The argument should be encoded according to generate_rtlbi_param().
 302 // Follows the same ASID matching behavior as flush_mmu_tlb_entries()
 303 static inline void
 304 flush_mmu_tlb_range_async(uint64_t val)
 305 {
 306         asm volatile ("tlbi rvae1is, %0" : : "r"(val));
 307 }
 308
 309 static inline void
 310 flush_mmu_tlb_range(uint64_t val)
 311 {
 312         flush_mmu_tlb_range_async(val);
 313         sync_tlb_flush();
 314 }
 315
 316 // flush_mmu_tlb_allrange: flush TLB entries that map a VA range using a single instruction
 317 // The argument should be encoded according to generate_rtlbi_param().
 318 // Follows the same ASID matching behavior as flush_mmu_tlb_allentries()
 319 static inline void
 320 flush_mmu_tlb_allrange_async(uint64_t val)
 321 {
 322         asm volatile ("tlbi rvaae1is, %0" : : "r"(val));
 323 }
 324
 325 static inline void
 326 flush_mmu_tlb_allrange(uint64_t val)
 327 {
 328         flush_mmu_tlb_allrange_async(val);
 329         sync_tlb_flush();
 330 }
 331
 332 // flush_core_tlb_allrange: flush TLB entries that map a VA range using a single instruction, local core only
 333 // The argument should be encoded according to generate_rtlbi_param().
 334 // Follows the same ASID matching behavior as flush_mmu_tlb_allentries()
 335 static inline void
 336 flush_core_tlb_allrange_async(uint64_t val)
 337 {
 338         asm volatile ("tlbi rvaae1, %0" : : "r"(val));
 339 }
 340
 341 static inline void
 342 flush_core_tlb_allrange(uint64_t val)
 343 {
 344         flush_core_tlb_allrange_async(val);
 345         sync_tlb_flush();
 346 }
 347
 348 #endif // __ARM_RANGE_TLBI__
 349
 350