osfmk/arm64/tlb.h

   1 /*
   2  * Copyright (c) 2019 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #pragma once
  30
  31 #include <arm64/proc_reg.h>
  32 #include <machine/atomic.h>
  33
  34 #define tlbi_addr(x) ((((x) >> 12) & TLBI_ADDR_MASK) << TLBI_ADDR_SHIFT)
  35 #define tlbi_asid(x) (((uintptr_t)(x) & TLBI_ASID_MASK) << TLBI_ASID_SHIFT)
  36
  37 #if __ARM_KERNEL_PROTECT__
  38 /*
  39  * __ARM_KERNEL_PROTECT__ adds two complications to TLB management:
  40  *
  41  * 1. As each pmap has two ASIDs, every TLB operation that targets an ASID must
  42  *   target both ASIDs for the pmap that owns the target ASID.
  43  *
  44  * 2. Any TLB operation targeting the kernel_pmap ASID (ASID 0) must target all
  45  *   ASIDs (as kernel_pmap mappings may be referenced while using an ASID that
  46  *   belongs to another pmap).  We expect these routines to be called with the
  47  *   EL0 ASID for the target; not the EL1 ASID.
  48  */
  49 #endif /* __ARM_KERNEL_PROTECT__ */
  50
  51 static inline void
  52 sync_tlb_flush(void)
  53 {
  54         __builtin_arm_dsb(DSB_ISH);
  55         __builtin_arm_isb(ISB_SY);
  56 }
  57
  58 // flush_mmu_tlb: full TLB flush on all cores
  59 static inline void
  60 flush_mmu_tlb_async(void)
  61 {
  62         asm volatile ("tlbi vmalle1is");
  63 }
  64
  65 static inline void
  66 flush_mmu_tlb(void)
  67 {
  68         flush_mmu_tlb_async();
  69         sync_tlb_flush();
  70 }
  71
  72 // flush_core_tlb: full TLB flush on local core only
  73 static inline void
  74 flush_core_tlb_async(void)
  75 {
  76         asm volatile ("tlbi vmalle1");
  77 }
  78
  79 static inline void
  80 flush_core_tlb(void)
  81 {
  82         flush_core_tlb_async();
  83         sync_tlb_flush();
  84 }
  85
  86 // flush_mmu_tlb_allentries_async: flush entries that map VA range, all ASIDS, all cores
  87 // start and end are in units of 4K pages.
  88 static inline void
  89 flush_mmu_tlb_allentries_async(uint64_t start, uint64_t end)
  90 {
  91 #if __ARM_16K_PG__
  92         start = start & ~0x3ULL;
  93
  94         /*
  95          * The code below is not necessarily correct.  From an overview of
  96          * the client code, the expected contract for TLB flushes is that
  97          * we will expand from an "address, length" pair to "start address,
  98          * end address" in the course of a TLB flush.  This suggests that
  99          * a flush for "X, X+4" is actually only asking for a flush of a
 100          * single 16KB page.  At the same time, we'd like to be prepared
 101          * for bad inputs (X, X+3), so add 3 and then truncate the 4KB page
 102          * number to a 16KB page boundary.  This should deal correctly with
 103          * unaligned inputs.
 104          *
 105          * If our expecations about client behavior are wrong however, this
 106          * will lead to occasional TLB corruption on platforms with 16KB
 107          * pages.
 108          */
 109         end = (end + 0x3ULL) & ~0x3ULL;
 110 #endif // __ARM_16K_PG__
 111         for (; start < end; start += (ARM_PGBYTES / 4096)) {
 112                 asm volatile ("tlbi vaae1is, %0" : : "r"(start));
 113         }
 114 }
 115
 116 static inline void
 117 flush_mmu_tlb_allentries(uint64_t start, uint64_t end)
 118 {
 119         flush_mmu_tlb_allentries_async(start, end);
 120         sync_tlb_flush();
 121 }
 122
 123 // flush_mmu_tlb_entry: flush TLB entries that map a VA and ASID, all cores
 124 // Will also flush global entries that match the VA
 125 static inline void
 126 flush_mmu_tlb_entry_async(uint64_t val)
 127 {
 128 #if __ARM_KERNEL_PROTECT__
 129         uint64_t asid = val >> TLBI_ASID_SHIFT;
 130         if (asid == 0) {
 131                 asm volatile ("tlbi vaae1is, %0" : : "r"(val));
 132                 return;
 133         }
 134         val = val & ~(1ULL << TLBI_ASID_SHIFT);
 135         asm volatile ("tlbi vae1is, %0" : : "r"(val));
 136         val = val | (1ULL << TLBI_ASID_SHIFT);
 137 #endif /* __ARM_KERNEL_PROTECT__ */
 138         asm volatile ("tlbi vae1is, %0" : : "r"(val));
 139 }
 140
 141 static inline void
 142 flush_mmu_tlb_entry(uint64_t val)
 143 {
 144         flush_mmu_tlb_entry_async(val);
 145         sync_tlb_flush();
 146 }
 147
 148 // flush_mmu_tlb_entries: flush TLB entries that map a VA range and ASID, all cores
 149 // start and end must have the ASID in the high 16 bits, with the VA in units of 4K in the lowest bits
 150 // Will also flush global entries that match the VA range
 151 static inline void
 152 flush_mmu_tlb_entries_async(uint64_t start, uint64_t end)
 153 {
 154 #if __ARM_16K_PG__
 155         start = start & ~0x3ULL;
 156
 157         /*
 158          * The code below is not necessarily correct.  From an overview of
 159          * the client code, the expected contract for TLB flushes is that
 160          * we will expand from an "address, length" pair to "start address,
 161          * end address" in the course of a TLB flush.  This suggests that
 162          * a flush for "X, X+4" is actually only asking for a flush of a
 163          * single 16KB page.  At the same time, we'd like to be prepared
 164          * for bad inputs (X, X+3), so add 3 and then truncate the 4KB page
 165          * number to a 16KB page boundary.  This should deal correctly with
 166          * unaligned inputs.
 167          *
 168          * If our expecations about client behavior are wrong however, this
 169          * will lead to occasional TLB corruption on platforms with 16KB
 170          * pages.
 171          */
 172         end = (end + 0x3ULL) & ~0x3ULL;
 173 #endif // __ARM_16K_PG__
 174 #if __ARM_KERNEL_PROTECT__
 175         uint64_t asid = start >> TLBI_ASID_SHIFT;
 176         /*
 177          * If we are flushing ASID 0, this is a kernel operation.  With this
 178          * ASID scheme, this means we should flush all ASIDs.
 179          */
 180         if (asid == 0) {
 181                 for (; start < end; start += (ARM_PGBYTES / 4096)) {
 182                         asm volatile ("tlbi vaae1is, %0" : : "r"(start));
 183                 }
 184                 return;
 185         }
 186         start = start | (1ULL << TLBI_ASID_SHIFT);
 187         end = end | (1ULL << TLBI_ASID_SHIFT);
 188         for (; start < end; start += (ARM_PGBYTES / 4096)) {
 189                 start = start & ~(1ULL << TLBI_ASID_SHIFT);
 190                 asm volatile ("tlbi vae1is, %0" : : "r"(start));
 191                 start = start | (1ULL << TLBI_ASID_SHIFT);
 192                 asm volatile ("tlbi vae1is, %0" : : "r"(start));
 193         }
 194 #else
 195         for (; start < end; start += (ARM_PGBYTES / 4096)) {
 196                 asm volatile ("tlbi vae1is, %0" : : "r"(start));
 197         }
 198 #endif /* __ARM_KERNEL_PROTECT__ */
 199 }
 200
 201 static inline void
 202 flush_mmu_tlb_entries(uint64_t start, uint64_t end)
 203 {
 204         flush_mmu_tlb_entries_async(start, end);
 205         sync_tlb_flush();
 206 }
 207
 208 // flush_mmu_tlb_asid: flush all entries that match an ASID, on all cores
 209 // ASID must be in high 16 bits of argument
 210 // Will not flush global entries
 211 static inline void
 212 flush_mmu_tlb_asid_async(uint64_t val)
 213 {
 214 #if __ARM_KERNEL_PROTECT__
 215         /*
 216          * If we are flushing ASID 0, this is a kernel operation.  With this
 217          * ASID scheme, this means we should flush all ASIDs.
 218          */
 219         uint64_t asid = val >> TLBI_ASID_SHIFT;
 220         if (asid == 0) {
 221                 asm volatile ("tlbi vmalle1is");
 222                 return;
 223         }
 224         val = val & ~(1ULL << TLBI_ASID_SHIFT);
 225         asm volatile ("tlbi aside1is, %0" : : "r"(val));
 226         val = val | (1ULL << TLBI_ASID_SHIFT);
 227 #endif /* __ARM_KERNEL_PROTECT__ */
 228         asm volatile ("tlbi aside1is, %0" : : "r"(val));
 229 }
 230
 231 static inline void
 232 flush_mmu_tlb_asid(uint64_t val)
 233 {
 234         flush_mmu_tlb_asid_async(val);
 235         sync_tlb_flush();
 236 }
 237
 238 // flush_core_tlb_asid: flush all entries that match an ASID, local core only
 239 // ASID must be in high 16 bits of argument
 240 // Will not flush global entries
 241 static inline void
 242 flush_core_tlb_asid_async(uint64_t val)
 243 {
 244 #if __ARM_KERNEL_PROTECT__
 245         /*
 246          * If we are flushing ASID 0, this is a kernel operation.  With this
 247          * ASID scheme, this means we should flush all ASIDs.
 248          */
 249         uint64_t asid = val >> TLBI_ASID_SHIFT;
 250         if (asid == 0) {
 251                 asm volatile ("tlbi vmalle1");
 252                 return;
 253         }
 254         val = val & ~(1ULL << TLBI_ASID_SHIFT);
 255         asm volatile ("tlbi aside1, %0" : : "r"(val));
 256         val = val | (1ULL << TLBI_ASID_SHIFT);
 257 #endif /* __ARM_KERNEL_PROTECT__ */
 258         asm volatile ("tlbi aside1, %0" : : "r"(val));
 259 }
 260
 261 static inline void
 262 flush_core_tlb_asid(uint64_t val)
 263 {
 264         flush_core_tlb_asid_async(val);
 265         sync_tlb_flush();
 266 }
 267
 268 #if __ARM_RANGE_TLBI__
 269 #if __ARM_KERNEL_PROTECT__
 270         #error __ARM_RANGE_TLBI__ + __ARM_KERNEL_PROTECT__ is not currently supported
 271 #endif
 272
 273 #define ARM64_16K_TLB_RANGE_PAGES (1ULL << 21)
 274 #define rtlbi_addr(x) (((x) >> RTLBI_ADDR_SHIFT) & RTLBI_ADDR_MASK)
 275 #define rtlbi_scale(x) ((uint64_t)(x) << RTLBI_SCALE_SHIFT)
 276 #define rtlbi_num(x) ((uint64_t)(x) << RTLBI_NUM_SHIFT)
 277
 278 /**
 279  * Given the number of pages to invalidate, generate the correct parameter to
 280  * pass to any of the TLBI by range methods.
 281  */
 282 static inline uint64_t
 283 generate_rtlbi_param(ppnum_t npages, uint32_t asid, vm_offset_t va)
 284 {
 285         /**
 286          * Per the armv8.4 RTLBI extension spec, the range encoded in the rtlbi register operand is defined by:
 287          * BaseADDR <= VA < BaseADDR+((NUM+1)*2^(5*SCALE+1) * Translation_Granule_Size)
 288          */
 289         unsigned order = (sizeof(npages) * 8) - __builtin_clz(npages - 1) - 1;
 290         unsigned scale = ((order ? order : 1) - 1) / 5;
 291         unsigned granule = 1 << ((5 * scale) + 1);
 292         unsigned num = (((npages + granule - 1) & ~(granule - 1)) / granule) - 1;
 293         return tlbi_asid(asid) | RTLBI_TG | rtlbi_scale(scale) | rtlbi_num(num) | rtlbi_addr(va);
 294 }
 295
 296 // flush_mmu_tlb_range: flush TLB entries that map a VA range using a single instruction
 297 // The argument should be encoded according to generate_rtlbi_param().
 298 // Follows the same ASID matching behavior as flush_mmu_tlb_entries()
 299 static inline void
 300 flush_mmu_tlb_range_async(uint64_t val)
 301 {
 302         asm volatile ("tlbi rvae1is, %0" : : "r"(val));
 303 }
 304
 305 static inline void
 306 flush_mmu_tlb_range(uint64_t val)
 307 {
 308         flush_mmu_tlb_range_async(val);
 309         sync_tlb_flush();
 310 }
 311
 312 // flush_mmu_tlb_allrange: flush TLB entries that map a VA range using a single instruction
 313 // The argument should be encoded according to generate_rtlbi_param().
 314 // Follows the same ASID matching behavior as flush_mmu_tlb_allentries()
 315 static inline void
 316 flush_mmu_tlb_allrange_async(uint64_t val)
 317 {
 318         asm volatile ("tlbi rvaae1is, %0" : : "r"(val));
 319 }
 320
 321 static inline void
 322 flush_mmu_tlb_allrange(uint64_t val)
 323 {
 324         flush_mmu_tlb_allrange_async(val);
 325         sync_tlb_flush();
 326 }
 327
 328 #endif // __ARM_RANGE_TLBI__
 329
 330