osfmk/arm64/proc_reg.h

   1 /*
   2  * Copyright (c) 2007-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * Processor registers for ARM64
  30  */
  31 #ifndef _ARM64_PROC_REG_H_
  32 #define _ARM64_PROC_REG_H_
  33
  34 #include <arm/proc_reg.h>
  35
  36 #if __ARM_KERNEL_PROTECT__
  37 /*
  38  * __ARM_KERNEL_PROTECT__ is a feature intended to guard against potential
  39  * architectural or microarchitectural vulnerabilities that could allow cores to
  40  * read/access EL1-only mappings while in EL0 mode.  This is achieved by
  41  * removing as many mappings as possible when the core transitions to EL0 mode
  42  * from EL1 mode, and restoring those mappings when the core transitions to EL1
  43  * mode from EL0 mode.
  44  *
  45  * At the moment, this is achieved through use of ASIDs and TCR_EL1.  TCR_EL1 is
  46  * used to map and unmap the ordinary kernel mappings, by contracting and
  47  * expanding translation zone size for TTBR1 when exiting and entering EL1,
  48  * respectively:
  49  *
  50  * Kernel EL0 Mappings: TTBR1 mappings that must remain mapped while the core is
  51  *   is in EL0.
  52  * Kernel EL1 Mappings: TTBR1 mappings that must be mapped while the core is in
  53  *   EL1.
  54  *
  55  * T1SZ_USER: T1SZ_BOOT + 1
  56  * TTBR1_EL1_BASE_BOOT: (2^64) - (2^(64 - T1SZ_BOOT)
  57  * TTBR1_EL1_BASE_USER: (2^64) - (2^(64 - T1SZ_USER)
  58  * TTBR1_EL1_MAX: (2^64) - 1
  59  *
  60  * When in EL1, we program TCR_EL1 (specifically, TCR_EL1.T1SZ) to give the
  61  * the following TTBR1 layout:
  62  *
  63  *  TTBR1_EL1_BASE_BOOT   TTBR1_EL1_BASE_USER   TTBR1_EL1_MAX
  64  * +---------------------------------------------------------+
  65  * | Kernel EL0 Mappings |        Kernel EL1 Mappings        |
  66  * +---------------------------------------------------------+
  67  *
  68  * And when in EL0, we program TCR_EL1 to give the following TTBR1 layout:
  69  *
  70  *  TTBR1_EL1_BASE_USER                         TTBR1_EL1_MAX
  71  * +---------------------------------------------------------+
  72  * |                   Kernel EL0 Mappings                   |
  73  * +---------------------------------------------------------+
  74  *
  75  * With the current implementation, both the EL0 and EL1 mappings for the kernel
  76  * use otherwise empty translation tables for mapping the exception vectors (so
  77  * that we do not need to TLB flush the exception vector address when switching
  78  * between EL0 and EL1).  The rationale here is that the TLBI would require a
  79  * DSB, and DSBs can be extremely expensive.
  80  *
  81  * Each pmap is given two ASIDs: (n & ~1) as an EL0 ASID, and (n | 1) as an EL1
  82  * ASID.  The core switches between ASIDs on EL transitions, so that the TLB
  83  * does not need to be fully invalidated on an EL transition.
  84  *
  85  * Most kernel mappings will be marked non-global in this configuration, as
  86  * global mappings would be visible to userspace unless we invalidate them on
  87  * eret.
  88  */
  89 #endif /* __ARM_KERNEL_PROTECT */
  90
  91 /*
  92  * 64-bit Program Status Register (PSR64)
  93  *
  94  *  31      27 23  22 21 20 19      10 9       5 4   0
  95  * +-+-+-+-+-----+---+--+--+----------+-+-+-+-+-+-----+
  96  * |N|Z|C|V|00000|PAN|SS|IL|0000000000|D|A|I|F|0|  M  |
  97  * +-+-+-+-+-+---+---+--+--+----------+-+-+-+-+-+-----+
  98  *
  99  * where:
 100  *   NZCV: Comparison flags
 101  *   PAN:  Privileged Access Never
 102  *   SS:   Single step
 103  *   IL:   Illegal state
 104  *   DAIF: Interrupt masks
 105  *   M:    Mode field
 106  */
 107
 108 #define PSR64_NZCV_SHIFT 28
 109 #define PSR64_NZCV_MASK  (1 << PSR64_NZCV_SHIFT)
 110
 111 #define PSR64_N_SHIFT    31
 112 #define PSR64_N          (1 << PSR64_N_SHIFT)
 113
 114 #define PSR64_Z_SHIFT    30
 115 #define PSR64_Z          (1 << PSR64_Z_SHIFT)
 116
 117 #define PSR64_C_SHIFT    29
 118 #define PSR64_C          (1 << PSR64_C_SHIFT)
 119
 120 #define PSR64_V_SHIFT    28
 121 #define PSR64_V          (1 << PSR64_V_SHIFT)
 122
 123 #define PSR64_PAN_SHIFT  22
 124 #define PSR64_PAN        (1 << PSR64_PAN_SHIFT)
 125
 126 #define PSR64_SS_SHIFT   21
 127 #define PSR64_SS         (1 << PSR64_SS_SHIFT)
 128
 129 #define PSR64_IL_SHIFT   20
 130 #define PSR64_IL         (1 << PSR64_IL_SHIFT)
 131
 132 /*
 133  * msr DAIF, Xn and mrs Xn, DAIF transfer into
 134  * and out of bits 9:6
 135  */
 136 #define DAIF_DEBUG_SHIFT      9
 137 #define DAIF_DEBUGF           (1 << DAIF_DEBUG_SHIFT)
 138
 139 #define DAIF_ASYNC_SHIFT      8
 140 #define DAIF_ASYNCF           (1 << DAIF_ASYNC_SHIFT)
 141
 142 #define DAIF_IRQF_SHIFT       7
 143 #define DAIF_IRQF             (1 << DAIF_IRQF_SHIFT)
 144
 145 #define DAIF_FIQF_SHIFT       6
 146 #define DAIF_FIQF             (1 << DAIF_FIQF_SHIFT)
 147
 148 #define DAIF_ALL              (DAIF_DEBUGF | DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF)
 149 #define DAIF_STANDARD_DISABLE (DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF)
 150
 151 #define SPSR_INTERRUPTS_ENABLED(x) (!(x & DAIF_FIQF))
 152
 153 /*
 154  * msr DAIFSet, Xn, and msr DAIFClr, Xn transfer
 155  * from bits 3:0.
 156  */
 157 #define DAIFSC_DEBUGF           (1 << 3)
 158 #define DAIFSC_ASYNCF           (1 << 2)
 159 #define DAIFSC_IRQF             (1 << 1)
 160 #define DAIFSC_FIQF             (1 << 0)
 161 #define DAIFSC_ALL              (DAIFSC_DEBUGF | DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF)
 162 #define DAIFSC_STANDARD_DISABLE (DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF)
 163
 164 /*
 165  * ARM64_TODO: unify with ARM?
 166  */
 167 #define PSR64_CF         0x20000000 /* Carry/Borrow/Extend */
 168
 169 #define PSR64_MODE_MASK         0x1F
 170
 171 #define PSR64_MODE_USER32_THUMB 0x20
 172
 173 #define PSR64_MODE_RW_SHIFT     4
 174 #define PSR64_MODE_RW_64        0
 175 #define PSR64_MODE_RW_32        (0x1 << PSR64_MODE_RW_SHIFT)
 176
 177 #define PSR64_MODE_EL_SHIFT     2
 178 #define PSR64_MODE_EL_MASK      (0x3 << PSR64_MODE_EL_SHIFT)
 179 #define PSR64_MODE_EL3          (0x3 << PSR64_MODE_EL_SHIFT)
 180 #define PSR64_MODE_EL2          (0x2 << PSR64_MODE_EL_SHIFT)
 181 #define PSR64_MODE_EL1          (0x1 << PSR64_MODE_EL_SHIFT)
 182 #define PSR64_MODE_EL0          0
 183
 184 #define PSR64_MODE_SPX          0x1
 185 #define PSR64_MODE_SP0          0
 186
 187 #define PSR64_USER32_DEFAULT    (PSR64_MODE_RW_32 | PSR64_MODE_EL0 | PSR64_MODE_SP0)
 188 #define PSR64_USER64_DEFAULT    (PSR64_MODE_RW_64 | PSR64_MODE_EL0 | PSR64_MODE_SP0)
 189 #define PSR64_KERNEL_STANDARD   (DAIF_STANDARD_DISABLE | PSR64_MODE_RW_64 | PSR64_MODE_EL1 | PSR64_MODE_SP0)
 190 #if __ARM_PAN_AVAILABLE__
 191 #define PSR64_KERNEL_DEFAULT    (PSR64_KERNEL_STANDARD | PSR64_PAN)
 192 #else
 193 #define PSR64_KERNEL_DEFAULT    PSR64_KERNEL_STANDARD
 194 #endif
 195
 196 #define PSR64_IS_KERNEL(x)      ((x & PSR64_MODE_EL_MASK) > PSR64_MODE_EL0)
 197 #define PSR64_IS_USER(x)        ((x & PSR64_MODE_EL_MASK) == PSR64_MODE_EL0)
 198
 199 #define PSR64_IS_USER32(x)      (PSR64_IS_USER(x) && (x & PSR64_MODE_RW_32))
 200 #define PSR64_IS_USER64(x)      (PSR64_IS_USER(x) && !(x & PSR64_MODE_RW_32))
 201
 202
 203
 204 /*
 205  * System Control Register (SCTLR)
 206  */
 207
 208 #define SCTLR_RESERVED     ((3ULL << 28) | (1ULL << 22) | (1ULL << 20) | (1ULL << 11))
 209 #if defined(HAS_APPLE_PAC)
 210
 211 // 31    PACIA_ENABLED AddPACIA and AuthIA functions enabled
 212 #define SCTLR_PACIA_ENABLED_SHIFT 31
 213 #define SCTLR_PACIA_ENABLED       (1ULL << SCTLR_PACIA_ENABLED_SHIFT)
 214 // 30    PACIB_ENABLED AddPACIB and AuthIB functions enabled
 215 #define SCTLR_PACIB_ENABLED       (1ULL << 30)
 216 // 29:28 RES1 11
 217 // 27    PACDA_ENABLED AddPACDA and AuthDA functions enabled
 218 #define SCTLR_PACDA_ENABLED       (1ULL << 27)
 219 // 13    PACDB_ENABLED  AddPACDB and AuthDB functions enabled
 220 #define SCTLR_PACDB_ENABLED       (1ULL << 13)
 221
 222 #define SCTLR_JOP_KEYS_ENABLED (SCTLR_PACIA_ENABLED | SCTLR_PACDA_ENABLED | SCTLR_PACDB_ENABLED)
 223 #endif /* defined(HAS_APPLE_PAC) */
 224
 225 // 26    UCI User Cache Instructions
 226 #define SCTLR_UCI_ENABLED         (1ULL << 26)
 227
 228 // 25    EE             Exception Endianness
 229 #define SCTLR_EE_BIG_ENDIAN       (1ULL << 25)
 230
 231 // 24    E0E            EL0 Endianness
 232 #define SCTLR_E0E_BIG_ENDIAN      (1ULL << 24)
 233
 234 // 23    SPAN           Set PAN
 235 #define SCTLR_PAN_UNCHANGED       (1ULL << 23)
 236
 237 // 22    RES1           1
 238 // 21    RES0           0
 239 // 20    RES1           1
 240
 241 // 19    WXN            Writeable implies eXecute Never
 242 #define SCTLR_WXN_ENABLED         (1ULL << 19)
 243
 244 // 18    nTWE           Not trap WFE from EL0
 245 #define SCTLR_nTWE_WFE_ENABLED    (1ULL << 18)
 246
 247 // 17    RES0           0
 248
 249 // 16    nTWI           Not trap WFI from EL0
 250 #define SCTRL_nTWI_WFI_ENABLED    (1ULL << 16)
 251
 252 // 15    UCT            User Cache Type register (CTR_EL0)
 253 #define SCTLR_UCT_ENABLED         (1ULL << 15)
 254
 255 // 14    DZE            User Data Cache Zero (DC ZVA)
 256 #define SCTLR_DZE_ENABLED         (1ULL << 14)
 257
 258 // 12    I              Instruction cache enable
 259 #define SCTLR_I_ENABLED           (1ULL << 12)
 260
 261 // 11    RES1           1
 262 // 10    RES0           0
 263
 264 // 9     UMA            User Mask Access
 265 #define SCTLR_UMA_ENABLED         (1ULL << 9)
 266
 267 // 8     SED            SETEND Disable
 268 #define SCTLR_SED_DISABLED        (1ULL << 8)
 269
 270 // 7     ITD            IT Disable
 271 #define SCTLR_ITD_DISABLED        (1ULL << 7)
 272
 273 // 6     RES0           0
 274
 275 // 5     CP15BEN        CP15 Barrier ENable
 276 #define SCTLR_CP15BEN_ENABLED     (1ULL << 5)
 277
 278 // 4     SA0            Stack Alignment check for EL0
 279 #define SCTLR_SA0_ENABLED         (1ULL << 4)
 280
 281 // 3     SA             Stack Alignment check
 282 #define SCTLR_SA_ENABLED          (1ULL << 3)
 283
 284 // 2     C              Cache enable
 285 #define SCTLR_C_ENABLED           (1ULL << 2)
 286
 287 // 1     A              Alignment check
 288 #define SCTLR_A_ENABLED           (1ULL << 1)
 289
 290 // 0     M              MMU enable
 291 #define SCTLR_M_ENABLED           (1ULL << 0)
 292
 293 #define SCTLR_EL1_DEFAULT \
 294         (SCTLR_RESERVED | SCTLR_UCI_ENABLED | SCTLR_nTWE_WFE_ENABLED | SCTLR_DZE_ENABLED | \
 295          SCTLR_I_ENABLED | SCTLR_SED_DISABLED | SCTLR_CP15BEN_ENABLED |                    \
 296          SCTLR_SA0_ENABLED | SCTLR_SA_ENABLED | SCTLR_C_ENABLED | SCTLR_M_ENABLED)
 297
 298 /*
 299  * Coprocessor Access Control Register (CPACR)
 300  *
 301  *  31  28  27  22 21  20 19                 0
 302  * +---+---+------+------+--------------------+
 303  * |000|TTA|000000| FPEN |00000000000000000000|
 304  * +---+---+------+------+--------------------+
 305  *
 306  * where:
 307  *   TTA:  Trace trap
 308  *   FPEN: Floating point enable
 309  */
 310 #define CPACR_TTA_SHIFT     28
 311 #define CPACR_TTA           (1 << CPACR_TTA_SHIFT)
 312
 313 #define CPACR_FPEN_SHIFT    20
 314 #define CPACR_FPEN_EL0_TRAP (0x1 << CPACR_FPEN_SHIFT)
 315 #define CPACR_FPEN_ENABLE   (0x3 << CPACR_FPEN_SHIFT)
 316
 317 /*
 318  *  FPSR: Floating Point Status Register
 319  *
 320  *  31 30 29 28 27 26                  7   6  4   3   2   1   0
 321  * +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+
 322  * | N| Z| C| V|QC|0000000000000000000|IDC|00|IXC|UFC|OFC|DZC|IOC|
 323  * +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+
 324  */
 325
 326 #define FPSR_N_SHIFT   31
 327 #define FPSR_Z_SHIFT   30
 328 #define FPSR_C_SHIFT   29
 329 #define FPSR_V_SHIFT   28
 330 #define FPSR_QC_SHIFT  27
 331 #define FPSR_IDC_SHIFT 7
 332 #define FPSR_IXC_SHIFT 4
 333 #define FPSR_UFC_SHIFT 3
 334 #define FPSR_OFC_SHIFT 2
 335 #define FPSR_DZC_SHIFT 1
 336 #define FPSR_IOC_SHIFT 0
 337 #define FPSR_N         (1 << FPSR_N_SHIFT)
 338 #define FPSR_Z         (1 << FPSR_Z_SHIFT)
 339 #define FPSR_C         (1 << FPSR_C_SHIFT)
 340 #define FPSR_V         (1 << FPSR_V_SHIFT)
 341 #define FPSR_QC        (1 << FPSR_QC_SHIFT)
 342 #define FPSR_IDC       (1 << FPSR_IDC_SHIFT)
 343 #define FPSR_IXC       (1 << FPSR_IXC_SHIFT)
 344 #define FPSR_UFC       (1 << FPSR_UFC_SHIFT)
 345 #define FPSR_OFC       (1 << FPSR_OFC_SHIFT)
 346 #define FPSR_DZC       (1 << FPSR_DZC_SHIFT)
 347 #define FPSR_IOC       (1 << FPSR_IOC_SHIFT)
 348
 349 /*
 350  * A mask for all for all of the bits that are not RAZ for FPSR; this
 351  * is primarily for converting between a 32-bit view of NEON state
 352  * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR).
 353  */
 354 #define FPSR_MASK \
 355         (FPSR_N | FPSR_Z | FPSR_C | FPSR_V | FPSR_QC | FPSR_IDC | FPSR_IXC | \
 356          FPSR_UFC | FPSR_OFC | FPSR_DZC | FPSR_IOC)
 357
 358 /*
 359  *  FPCR: Floating Point Control Register
 360  *
 361  *  31    26  25 24 23    21     19 18  15  14 12  11  10  9   8   7      0
 362  * +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+
 363  * |00000|AHP|DN|FZ|RMODE|STRIDE| 0|LEN|IDE|00|IXE|UFE|OFE|DZE|IOE|00000000|
 364  * +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+
 365  */
 366
 367 #define FPCR_AHP_SHIFT    26
 368 #define FPCR_DN_SHIFT     25
 369 #define FPCR_FZ_SHIFT     24
 370 #define FPCR_RMODE_SHIFT  22
 371 #define FPCR_STRIDE_SHIFT 20
 372 #define FPCR_LEN_SHIFT    16
 373 #define FPCR_IDE_SHIFT    15
 374 #define FPCR_IXE_SHIFT    12
 375 #define FPCR_UFE_SHIFT    11
 376 #define FPCR_OFE_SHIFT    10
 377 #define FPCR_DZE_SHIFT    9
 378 #define FPCR_IOE_SHIFT    8
 379 #define FPCR_AHP          (1 << FPCR_AHP_SHIFT)
 380 #define FPCR_DN           (1 << FPCR_DN_SHIFT)
 381 #define FPCR_FZ           (1 << FPCR_FZ_SHIFT)
 382 #define FPCR_RMODE        (0x3 << FPCR_RMODE_SHIFT)
 383 #define FPCR_STRIDE       (0x3 << FPCR_STRIDE_SHIFT)
 384 #define FPCR_LEN          (0x7 << FPCR_LEN_SHIFT)
 385 #define FPCR_IDE          (1 << FPCR_IDE_SHIFT)
 386 #define FPCR_IXE          (1 << FPCR_IXE_SHIFT)
 387 #define FPCR_UFE          (1 << FPCR_UFE_SHIFT)
 388 #define FPCR_OFE          (1 << FPCR_OFE_SHIFT)
 389 #define FPCR_DZE          (1 << FPCR_DZE_SHIFT)
 390 #define FPCR_IOE          (1 << FPCR_IOE_SHIFT)
 391 #define FPCR_DEFAULT      (FPCR_DN)
 392 #define FPCR_DEFAULT_32   (FPCR_DN|FPCR_FZ)
 393
 394 /*
 395  * A mask for all for all of the bits that are not RAZ for FPCR; this
 396  * is primarily for converting between a 32-bit view of NEON state
 397  * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR).
 398  */
 399 #define FPCR_MASK \
 400         (FPCR_AHP | FPCR_DN | FPCR_FZ | FPCR_RMODE | FPCR_STRIDE | FPCR_LEN | \
 401          FPCR_IDE | FPCR_IXE | FPCR_UFE | FPCR_OFE | FPCR_DZE | FPCR_IOE)
 402
 403 /*
 404  * Translation Control Register (TCR)
 405  *
 406  * Legacy:
 407  *
 408  *  63  39   38   37 36   34 32    30 29 28 27 26 25 24   23 22 21  16    14 13 12 11 10 9   8    7   5  0
 409  * +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+
 410  * | zero |TBI1|TBI0|AS|z| IPS |z|TG1| SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ |z|TG0| SH0 |ORGN0|IRGN0|EPD0|z|T0SZ|
 411  * +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+
 412  *
 413  * Current (with 16KB granule support):
 414  *
 415  *  63  39   38   37 36   34 32    30 29 28 27 26 25 24   23 22 21  16    14 13 12 11 10 9   8    7   5  0
 416  * +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+
 417  * | zero |TBI1|TBI0|AS|z| IPS | TG1 | SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ | TG0 | SH0 |ORGN0|IRGN0|EPD0|z|T0SZ|
 418  * +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+
 419  *
 420  * TBI1:  Top Byte Ignored for TTBR1 region
 421  * TBI0:  Top Byte Ignored for TTBR0 region
 422  * AS:    ASID Size
 423  * IPS:   Physical Address Size limit
 424  * TG1:   Granule Size for TTBR1 region
 425  * SH1:   Shareability for TTBR1 region
 426  * ORGN1: Outer Cacheability for TTBR1 region
 427  * IRGN1: Inner Cacheability for TTBR1 region
 428  * EPD1:  Translation table walk disable for TTBR1
 429  * A1:    ASID selection from TTBR1 enable
 430  * T1SZ:  Virtual address size for TTBR1
 431  * TG0:   Granule Size for TTBR0 region
 432  * SH0:   Shareability for TTBR0 region
 433  * ORGN0: Outer Cacheability for TTBR0 region
 434  * IRGN0: Inner Cacheability for TTBR0 region
 435  * T0SZ:  Virtual address size for TTBR0
 436  */
 437
 438 #define TCR_T0SZ_SHIFT          0ULL
 439 #define TCR_TSZ_BITS            6ULL
 440 #define TCR_TSZ_MASK            ((1ULL << TCR_TSZ_BITS) - 1ULL)
 441
 442 #define TCR_IRGN0_SHIFT         8ULL
 443 #define TCR_IRGN0_DISABLED      (0ULL << TCR_IRGN0_SHIFT)
 444 #define TCR_IRGN0_WRITEBACK     (1ULL << TCR_IRGN0_SHIFT)
 445 #define TCR_IRGN0_WRITETHRU     (2ULL << TCR_IRGN0_SHIFT)
 446 #define TCR_IRGN0_WRITEBACKNO   (3ULL << TCR_IRGN0_SHIFT)
 447
 448 #define TCR_ORGN0_SHIFT         10ULL
 449 #define TCR_ORGN0_DISABLED      (0ULL << TCR_ORGN0_SHIFT)
 450 #define TCR_ORGN0_WRITEBACK     (1ULL << TCR_ORGN0_SHIFT)
 451 #define TCR_ORGN0_WRITETHRU     (2ULL << TCR_ORGN0_SHIFT)
 452 #define TCR_ORGN0_WRITEBACKNO   (3ULL << TCR_ORGN0_SHIFT)
 453
 454 #define TCR_SH0_SHIFT           12ULL
 455 #define TCR_SH0_NONE            (0ULL << TCR_SH0_SHIFT)
 456 #define TCR_SH0_OUTER           (2ULL << TCR_SH0_SHIFT)
 457 #define TCR_SH0_INNER           (3ULL << TCR_SH0_SHIFT)
 458
 459 #define TCR_TG0_GRANULE_SHIFT   (14ULL)
 460
 461 #define TCR_TG0_GRANULE_4KB     (0ULL << TCR_TG0_GRANULE_SHIFT)
 462 #define TCR_TG0_GRANULE_64KB    (1ULL << TCR_TG0_GRANULE_SHIFT)
 463 #define TCR_TG0_GRANULE_16KB    (2ULL << TCR_TG0_GRANULE_SHIFT)
 464
 465 #if __ARM_16K_PG__
 466 #define TCR_TG0_GRANULE_SIZE    (TCR_TG0_GRANULE_16KB)
 467 #else
 468 #define TCR_TG0_GRANULE_SIZE    (TCR_TG0_GRANULE_4KB)
 469 #endif
 470
 471 #define TCR_T1SZ_SHIFT          16ULL
 472
 473 #define TCR_A1_ASID1            (1ULL << 22ULL)
 474 #define TCR_EPD1_TTBR1_DISABLED (1ULL << 23ULL)
 475
 476 #define TCR_IRGN1_SHIFT          24ULL
 477 #define TCR_IRGN1_DISABLED       (0ULL << TCR_IRGN1_SHIFT)
 478 #define TCR_IRGN1_WRITEBACK      (1ULL << TCR_IRGN1_SHIFT)
 479 #define TCR_IRGN1_WRITETHRU      (2ULL << TCR_IRGN1_SHIFT)
 480 #define TCR_IRGN1_WRITEBACKNO    (3ULL << TCR_IRGN1_SHIFT)
 481
 482 #define TCR_ORGN1_SHIFT          26ULL
 483 #define TCR_ORGN1_DISABLED       (0ULL << TCR_ORGN1_SHIFT)
 484 #define TCR_ORGN1_WRITEBACK      (1ULL << TCR_ORGN1_SHIFT)
 485 #define TCR_ORGN1_WRITETHRU      (2ULL << TCR_ORGN1_SHIFT)
 486 #define TCR_ORGN1_WRITEBACKNO    (3ULL << TCR_ORGN1_SHIFT)
 487
 488 #define TCR_SH1_SHIFT            28ULL
 489 #define TCR_SH1_NONE             (0ULL << TCR_SH1_SHIFT)
 490 #define TCR_SH1_OUTER            (2ULL << TCR_SH1_SHIFT)
 491 #define TCR_SH1_INNER            (3ULL << TCR_SH1_SHIFT)
 492
 493 #define TCR_TG1_GRANULE_SHIFT    30ULL
 494
 495 #define TCR_TG1_GRANULE_16KB     (1ULL << TCR_TG1_GRANULE_SHIFT)
 496 #define TCR_TG1_GRANULE_4KB      (2ULL << TCR_TG1_GRANULE_SHIFT)
 497 #define TCR_TG1_GRANULE_64KB     (3ULL << TCR_TG1_GRANULE_SHIFT)
 498
 499 #if __ARM_16K_PG__
 500 #define TCR_TG1_GRANULE_SIZE     (TCR_TG1_GRANULE_16KB)
 501 #else
 502 #define TCR_TG1_GRANULE_SIZE     (TCR_TG1_GRANULE_4KB)
 503 #endif
 504
 505 #define TCR_IPS_SHIFT            32ULL
 506 #define TCR_IPS_32BITS           (0ULL << TCR_IPS_SHIFT)
 507 #define TCR_IPS_36BITS           (1ULL << TCR_IPS_SHIFT)
 508 #define TCR_IPS_40BITS           (2ULL << TCR_IPS_SHIFT)
 509 #define TCR_IPS_42BITS           (3ULL << TCR_IPS_SHIFT)
 510 #define TCR_IPS_44BITS           (4ULL << TCR_IPS_SHIFT)
 511 #define TCR_IPS_48BITS           (5ULL << TCR_IPS_SHIFT)
 512
 513 #define TCR_AS_16BIT_ASID        (1ULL << 36)
 514 #define TCR_TBI0_TOPBYTE_IGNORED (1ULL << 37)
 515 #define TCR_TBI1_TOPBYTE_IGNORED (1ULL << 38)
 516 #define TCR_TBID0_TBI_DATA_ONLY  (1ULL << 51)
 517 #define TCR_TBID1_TBI_DATA_ONLY  (1ULL << 52)
 518
 519 #if defined(HAS_APPLE_PAC)
 520 #define TCR_TBID0_ENABLE         TCR_TBID0_TBI_DATA_ONLY
 521 #else
 522 #define TCR_TBID0_ENABLE         0
 523 #endif
 524
 525 /*
 526  * Multiprocessor Affinity Register (MPIDR_EL1)
 527  *
 528  * +64-----------------------------31+30+29-25+24+23-16+15-8+7--0+
 529  * |000000000000000000000000000000001| U|00000|MT| Aff2|Aff1|Aff0|
 530  * +---------------------------------+--+-----+--+-----+----+----+
 531  *
 532  * where
 533  *   U:    Uniprocessor
 534  *   MT:   Multi-threading at lowest affinity level
 535  *   Aff2: "1" - PCORE, "0" - ECORE
 536  *   Aff1: Cluster ID
 537  *   Aff0: CPU ID
 538  */
 539 #define MPIDR_AFF0_SHIFT 0
 540 #define MPIDR_AFF0_WIDTH 8
 541 #define MPIDR_AFF0_MASK  (((1 << MPIDR_AFF0_WIDTH) - 1) << MPIDR_AFF0_SHIFT)
 542 #define MPIDR_AFF1_SHIFT 8
 543 #define MPIDR_AFF1_WIDTH 8
 544 #define MPIDR_AFF1_MASK  (((1 << MPIDR_AFF1_WIDTH) - 1) << MPIDR_AFF1_SHIFT)
 545 #define MPIDR_AFF2_SHIFT 16
 546 #define MPIDR_AFF2_WIDTH 8
 547 #define MPIDR_AFF2_MASK  (((1 << MPIDR_AFF2_WIDTH) - 1) << MPIDR_AFF2_SHIFT)
 548
 549 /*
 550  * TXSZ indicates the size of the range a TTBR covers.  Currently,
 551  * we support the following:
 552  *
 553  * 4KB pages, full page L1: 39 bit range.
 554  * 4KB pages, sub-page L1: 38 bit range.
 555  * 16KB pages, full page L1: 47 bit range.
 556  * 16KB pages, sub-page L1: 39 bit range.
 557  * 16KB pages, two level page tables: 36 bit range.
 558  */
 559 #if __ARM_KERNEL_PROTECT__
 560 /*
 561  * If we are configured to use __ARM_KERNEL_PROTECT__, the first half of the
 562  * address space is used for the mappings that will remain in place when in EL0.
 563  * As a result, 1 bit less of address space is available to the rest of the
 564  * the kernel.
 565  */
 566 #endif /* __ARM_KERNEL_PROTECT__ */
 567 #ifdef __ARM_16K_PG__
 568 #if __ARM64_PMAP_SUBPAGE_L1__
 569 #define T0SZ_BOOT 25ULL
 570 #else /* !__ARM64_PMAP_SUBPAGE_L1__ */
 571 #define T0SZ_BOOT 17ULL
 572 #endif /* !__ARM64_PMAP_SUBPAGE_L1__ */
 573 #else /* __ARM_16K_PG__ */
 574 #if __ARM64_PMAP_SUBPAGE_L1__
 575 #define T0SZ_BOOT 26ULL
 576 #else /* __ARM64_PMAP_SUBPAGE_L1__ */
 577 #define T0SZ_BOOT 25ULL
 578 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */
 579 #endif /* __ARM_16K_PG__ */
 580
 581 #if defined(APPLE_ARM64_ARCH_FAMILY)
 582 /* T0SZ must be the same as T1SZ */
 583 #define T1SZ_BOOT T0SZ_BOOT
 584 #else /* defined(APPLE_ARM64_ARCH_FAMILY) */
 585 #ifdef __ARM_16K_PG__
 586 #if __ARM64_PMAP_SUBPAGE_L1__
 587 #define T1SZ_BOOT 25ULL
 588 #else /* !__ARM64_PMAP_SUBPAGE_L1__ */
 589 #define T1SZ_BOOT 17ULL
 590 #endif /* !__ARM64_PMAP_SUBPAGE_L1__ */
 591 #else /* __ARM_16K_PG__ */
 592 #if __ARM64_PMAP_SUBPAGE_L1__
 593 #define T1SZ_BOOT 26ULL
 594 #else /* __ARM64_PMAP_SUBPAGE_L1__ */
 595 #define T1SZ_BOOT 25ULL
 596 #endif /*__ARM64_PMAP_SUBPAGE_L1__*/
 597 #endif /* __ARM_16K_PG__ */
 598 #endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
 599
 600 #if __ARM_42BIT_PA_SPACE__
 601 #define TCR_IPS_VALUE TCR_IPS_42BITS
 602 #else /* !__ARM_42BIT_PA_SPACE__ */
 603 #define TCR_IPS_VALUE TCR_IPS_40BITS
 604 #endif /* !__ARM_42BIT_PA_SPACE__ */
 605
 606 #define TCR_EL1_BASE \
 607         (TCR_IPS_VALUE | TCR_SH0_OUTER | TCR_ORGN0_WRITEBACK |         \
 608          TCR_IRGN0_WRITEBACK | (T0SZ_BOOT << TCR_T0SZ_SHIFT) |          \
 609          (TCR_TG0_GRANULE_SIZE) | TCR_SH1_OUTER | TCR_ORGN1_WRITEBACK | \
 610          TCR_IRGN1_WRITEBACK | (TCR_TG1_GRANULE_SIZE) |                 \
 611          TCR_TBI0_TOPBYTE_IGNORED | (TCR_TBID0_ENABLE))
 612
 613 #if __ARM_KERNEL_PROTECT__
 614 #define TCR_EL1_BOOT (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT))
 615 #define T1SZ_USER (T1SZ_BOOT + 1)
 616 #define TCR_EL1_USER (TCR_EL1_BASE | (T1SZ_USER << TCR_T1SZ_SHIFT))
 617 #else
 618 #define TCR_EL1_BOOT (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT))
 619 #endif /* __ARM_KERNEL_PROTECT__ */
 620
 621
 622 /*
 623  * Translation Table Base Register (TTBR)
 624  *
 625  *  63    48 47               x x-1  0
 626  * +--------+------------------+------+
 627  * |  ASID  |   Base Address   | zero |
 628  * +--------+------------------+------+
 629  *
 630  */
 631 #define TTBR_ASID_SHIFT 48
 632 #define TTBR_ASID_MASK  0xffff000000000000
 633
 634 #define TTBR_BADDR_MASK 0x0000ffffffffffff
 635
 636 /*
 637  * Memory Attribute Indirection Register
 638  *
 639  *  63   56 55   48 47   40 39   32 31   24 23   16 15    8 7     0
 640  * +-------+-------+-------+-------+-------+-------+-------+-------+
 641  * | Attr7 | Attr6 | Attr5 | Attr4 | Attr3 | Attr2 | Attr1 | Attr0 |
 642  * +-------+-------+-------+-------+-------+-------+-------+-------+
 643  *
 644  */
 645
 646 #define MAIR_ATTR_SHIFT(x)          (8*(x))
 647
 648 /* Strongly ordered or device memory attributes */
 649 #define MAIR_OUTER_STRONGLY_ORDERED 0x0
 650 #define MAIR_OUTER_DEVICE           0x0
 651
 652 #define MAIR_INNER_STRONGLY_ORDERED 0x0
 653 #define MAIR_INNER_DEVICE           0x4
 654
 655 /* Normal memory attributes */
 656 #define MAIR_OUTER_NON_CACHEABLE    0x40
 657 #define MAIR_OUTER_WRITE_THROUGH    0x80
 658 #define MAIR_OUTER_WRITE_BACK       0xc0
 659
 660 #define MAIR_INNER_NON_CACHEABLE    0x4
 661 #define MAIR_INNER_WRITE_THROUGH    0x8
 662 #define MAIR_INNER_WRITE_BACK       0xc
 663
 664 /* Allocate policy for cacheable memory */
 665 #define MAIR_OUTER_WRITE_ALLOCATE   0x10
 666 #define MAIR_OUTER_READ_ALLOCATE    0x20
 667
 668 #define MAIR_INNER_WRITE_ALLOCATE   0x1
 669 #define MAIR_INNER_READ_ALLOCATE    0x2
 670
 671 /* Memory Atribute Encoding */
 672
 673 /*
 674  * Device memory types:
 675  * G (gathering): multiple reads/writes can be combined
 676  * R (reordering): reads or writes may reach device out of program order
 677  * E (early-acknowledge): writes may return immediately (e.g. PCIe posted writes)
 678  */
 679 #define MAIR_DISABLE                   0x00 /* Device Memory, nGnRnE (strongly ordered) */
 680 #define MAIR_POSTED                    0x04 /* Device Memory, nGnRE (strongly ordered, posted writes) */
 681 #define MAIR_POSTED_REORDERED          0x08 /* Device Memory, nGRE (reorderable, posted writes) */
 682 #define MAIR_POSTED_COMBINED_REORDERED 0x0C /* Device Memory, GRE (reorderable, gathered writes, posted writes) */
 683 #define MAIR_WRITECOMB                 0x44 /* Normal Memory, Outer Non-Cacheable, Inner Non-Cacheable */
 684 #define MAIR_WRITETHRU                 0xBB /* Normal Memory, Outer Write-through, Inner Write-through */
 685 #define MAIR_WRITEBACK                 0xFF /* Normal Memory, Outer Write-back, Inner Write-back */
 686 #define MAIR_INNERWRITEBACK            0x4F /* Normal Memory, Outer Non-Cacheable, Inner Write-back */
 687
 688
 689 /*
 690  * ARM 4-level Page Table support - 2*1024TB (2^48) of address space
 691  */
 692
 693
 694 /*
 695  *  Memory Attribute Index
 696  */
 697 #define CACHE_ATTRINDX_WRITEBACK                 0x0 /* cache enabled, buffer enabled  (normal memory) */
 698 #define CACHE_ATTRINDX_WRITECOMB                 0x1 /* no cache, buffered writes (normal memory) */
 699 #define CACHE_ATTRINDX_WRITETHRU                 0x2 /* cache enabled, buffer disabled (normal memory) */
 700 #define CACHE_ATTRINDX_DISABLE                   0x3 /* no cache, no buffer (device memory) */
 701 #define CACHE_ATTRINDX_INNERWRITEBACK            0x4 /* inner cache enabled, buffer enabled, write allocate (normal memory) */
 702 #define CACHE_ATTRINDX_POSTED                    0x5 /* no cache, no buffer, posted writes (device memory) */
 703 #define CACHE_ATTRINDX_POSTED_REORDERED          0x6 /* no cache, reorderable access, posted writes (device memory) */
 704 #define CACHE_ATTRINDX_POSTED_COMBINED_REORDERED 0x7 /* no cache, write gathering, reorderable access, posted writes (device memory) */
 705 #define CACHE_ATTRINDX_DEFAULT                   CACHE_ATTRINDX_WRITEBACK
 706
 707
 708 /*
 709  * Access protection bit values (TTEs and PTEs), stage 1
 710  *
 711  * Bit 1 controls access type (1=RO, 0=RW), bit 0 controls user (1=access, 0=no access)
 712  */
 713 #define AP_RWNA 0x0 /* priv=read-write, user=no-access */
 714 #define AP_RWRW 0x1 /* priv=read-write, user=read-write */
 715 #define AP_RONA 0x2 /* priv=read-only, user=no-access */
 716 #define AP_RORO 0x3 /* priv=read-only, user=read-only */
 717 #define AP_MASK 0x3 /* mask to find ap bits */
 718
 719 /*
 720  * Shareability attributes
 721  */
 722 #define SH_NONE         0x0 /* Non shareable  */
 723 #define SH_NONE         0x0 /* Device shareable */
 724 #define SH_DEVICE       0x2 /* Normal memory Inner non shareable - Outer non shareable */
 725 #define SH_OUTER_MEMORY 0x2 /* Normal memory Inner shareable - Outer shareable */
 726 #define SH_INNER_MEMORY 0x3 /* Normal memory Inner shareable - Outer non shareable */
 727
 728
 729 /*
 730  * ARM Page Granule
 731  */
 732 #ifdef __ARM_16K_PG__
 733 #define ARM_PGSHIFT 14
 734 #else
 735 #define ARM_PGSHIFT 12
 736 #endif
 737 #define ARM_PGBYTES (1 << ARM_PGSHIFT)
 738 #define ARM_PGMASK  (ARM_PGBYTES-1)
 739
 740 /*
 741  *  L0 Translation table
 742  *
 743  *  4KB granule size:
 744  *    Each translation table is 4KB
 745  *    512 64-bit entries of 512GB (2^39) of address space.
 746  *    Covers 256TB (2^48) of address space.
 747  *
 748  *  16KB granule size:
 749  *    Each translation table is 16KB
 750  *    2 64-bit entries of 128TB (2^47) of address space.
 751  *    Covers 256TB (2^48) of address space.
 752  */
 753
 754 /* 16K L0 */
 755 #define ARM_16K_TT_L0_SIZE       0x0000800000000000ULL /* size of area covered by a tte */
 756 #define ARM_16K_TT_L0_OFFMASK    0x00007fffffffffffULL /* offset within an L0 entry */
 757 #define ARM_16K_TT_L0_SHIFT      47                    /* page descriptor shift */
 758 #define ARM_16K_TT_L0_INDEX_MASK 0x0000800000000000ULL /* mask for getting index in L0 table from virtual address */
 759
 760 /* 4K L0 */
 761 #define ARM_4K_TT_L0_SIZE       0x0000008000000000ULL /* size of area covered by a tte */
 762 #define ARM_4K_TT_L0_OFFMASK    0x0000007fffffffffULL /* offset within an L0 entry */
 763 #define ARM_4K_TT_L0_SHIFT      39                    /* page descriptor shift */
 764 #define ARM_4K_TT_L0_INDEX_MASK 0x0000ff8000000000ULL /* mask for getting index in L0 table from virtual address */
 765
 766 /*
 767  *  L1 Translation table
 768  *
 769  *  4KB granule size:
 770  *    Each translation table is 4KB
 771  *    512 64-bit entries of 1GB (2^30) of address space.
 772  *    Covers 512GB (2^39) of address space.
 773  *
 774  *  16KB granule size:
 775  *    Each translation table is 16KB
 776  *    2048 64-bit entries of 64GB (2^36) of address space.
 777  *    Covers 128TB (2^47) of address space.
 778  */
 779
 780 /* 16K L1 */
 781 #define ARM_16K_TT_L1_SIZE       0x0000001000000000ULL /* size of area covered by a tte */
 782 #define ARM_16K_TT_L1_OFFMASK    0x0000000fffffffffULL /* offset within an L1 entry */
 783 #define ARM_16K_TT_L1_SHIFT      36                    /* page descriptor shift */
 784 #ifdef __ARM64_PMAP_SUBPAGE_L1__
 785 /* This config supports 512GB per TTBR. */
 786 #define ARM_16K_TT_L1_INDEX_MASK 0x0000007000000000ULL /* mask for getting index into L1 table from virtual address */
 787 #else /* __ARM64_PMAP_SUBPAGE_L1__ */
 788 #define ARM_16K_TT_L1_INDEX_MASK 0x00007ff000000000ULL /* mask for getting index into L1 table from virtual address */
 789 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */
 790
 791 /* 4K L1 */
 792 #define ARM_4K_TT_L1_SIZE       0x0000000040000000ULL /* size of area covered by a tte */
 793 #define ARM_4K_TT_L1_OFFMASK    0x000000003fffffffULL /* offset within an L1 entry */
 794 #define ARM_4K_TT_L1_SHIFT      30                    /* page descriptor shift */
 795 #ifdef __ARM64_PMAP_SUBPAGE_L1__
 796 /* This config supports 256GB per TTBR. */
 797 #define ARM_4K_TT_L1_INDEX_MASK 0x0000003fc0000000ULL /* mask for getting index into L1 table from virtual address */
 798 #else /* __ARM64_PMAP_SUBPAGE_L1__ */
 799 #define ARM_4K_TT_L1_INDEX_MASK 0x0000007fc0000000ULL /* mask for getting index into L1 table from virtual address */
 800 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */
 801
 802 /* some sugar for getting pointers to page tables and entries */
 803
 804 #define L1_TABLE_INDEX(va) (((va) & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT)
 805 #define L2_TABLE_INDEX(va) (((va) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT)
 806 #define L3_TABLE_INDEX(va) (((va) & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT)
 807
 808 #define L2_TABLE_VA(tte)  ((tt_entry_t*) phystokv((*(tte)) & ARM_TTE_TABLE_MASK))
 809 #define L3_TABLE_VA(tte2) ((pt_entry_t*) phystokv((*(tte2)) & ARM_TTE_TABLE_MASK))
 810
 811 /*
 812  *  L2 Translation table
 813  *
 814  *  4KB granule size:
 815  *    Each translation table is 4KB
 816  *    512 64-bit entries of 2MB (2^21) of address space.
 817  *    Covers 1GB (2^30) of address space.
 818  *
 819  *  16KB granule size:
 820  *    Each translation table is 16KB
 821  *    2048 64-bit entries of 32MB (2^25) of address space.
 822  *    Covers 64GB (2^36) of address space.
 823  */
 824
 825 /* 16K L2 */
 826 #define ARM_16K_TT_L2_SIZE       0x0000000002000000ULL /* size of area covered by a tte */
 827 #define ARM_16K_TT_L2_OFFMASK    0x0000000001ffffffULL /* offset within an L2 entry */
 828 #define ARM_16K_TT_L2_SHIFT      25                    /* page descriptor shift */
 829 #define ARM_16K_TT_L2_INDEX_MASK 0x0000000ffe000000ULL /* mask for getting index in L2 table from virtual address */
 830
 831 /* 4K L2 */
 832 #define ARM_4K_TT_L2_SIZE       0x0000000000200000ULL /* size of area covered by a tte */
 833 #define ARM_4K_TT_L2_OFFMASK    0x00000000001fffffULL /* offset within an L2 entry */
 834 #define ARM_4K_TT_L2_SHIFT      21                    /* page descriptor shift */
 835 #define ARM_4K_TT_L2_INDEX_MASK 0x000000003fe00000ULL /* mask for getting index in L2 table from virtual address */
 836
 837 /*
 838  *  L3 Translation table
 839  *
 840  *  4KB granule size:
 841  *    Each translation table is 4KB
 842  *    512 64-bit entries of 4KB (2^12) of address space.
 843  *    Covers 2MB (2^21) of address space.
 844  *
 845  *  16KB granule size:
 846  *    Each translation table is 16KB
 847  *    2048 64-bit entries of 16KB (2^14) of address space.
 848  *    Covers 32MB (2^25) of address space.
 849  */
 850
 851 /* 16K L3 */
 852 #define ARM_16K_TT_L3_SIZE       0x0000000000004000ULL /* size of area covered by a tte */
 853 #define ARM_16K_TT_L3_OFFMASK    0x0000000000003fffULL /* offset within L3 PTE */
 854 #define ARM_16K_TT_L3_SHIFT      14                    /* page descriptor shift */
 855 #define ARM_16K_TT_L3_INDEX_MASK 0x0000000001ffc000ULL /* mask for page descriptor index */
 856
 857 /* 4K L3 */
 858 #define ARM_4K_TT_L3_SIZE       0x0000000000001000ULL /* size of area covered by a tte */
 859 #define ARM_4K_TT_L3_OFFMASK    0x0000000000000fffULL /* offset within L3 PTE */
 860 #define ARM_4K_TT_L3_SHIFT      12                    /* page descriptor shift */
 861 #define ARM_4K_TT_L3_INDEX_MASK 0x00000000001ff000ULL /* mask for page descriptor index */
 862
 863 #ifdef __ARM_16K_PG__
 864
 865 /* Native L0 defines */
 866 #define ARM_TT_L0_SIZE       ARM_16K_TT_L0_SIZE
 867 #define ARM_TT_L0_OFFMASK    ARM_16K_TT_L0_OFFMASK
 868 #define ARM_TT_L0_SHIFT      ARM_16K_TT_L0_SHIFT
 869 #define ARM_TT_L0_INDEX_MASK ARM_16K_TT_L0_INDEX_MASK
 870
 871 /* Native L1 defines */
 872 #define ARM_TT_L1_SIZE       ARM_16K_TT_L1_SIZE
 873 #define ARM_TT_L1_OFFMASK    ARM_16K_TT_L1_OFFMASK
 874 #define ARM_TT_L1_SHIFT      ARM_16K_TT_L1_SHIFT
 875 #define ARM_TT_L1_INDEX_MASK ARM_16K_TT_L1_INDEX_MASK
 876
 877 /* Native L2 defines */
 878 #define ARM_TT_L2_SIZE       ARM_16K_TT_L2_SIZE
 879 #define ARM_TT_L2_OFFMASK    ARM_16K_TT_L2_OFFMASK
 880 #define ARM_TT_L2_SHIFT      ARM_16K_TT_L2_SHIFT
 881 #define ARM_TT_L2_INDEX_MASK ARM_16K_TT_L2_INDEX_MASK
 882
 883 /* Native L3 defines */
 884 #define ARM_TT_L3_SIZE       ARM_16K_TT_L3_SIZE
 885 #define ARM_TT_L3_OFFMASK    ARM_16K_TT_L3_OFFMASK
 886 #define ARM_TT_L3_SHIFT      ARM_16K_TT_L3_SHIFT
 887 #define ARM_TT_L3_INDEX_MASK ARM_16K_TT_L3_INDEX_MASK
 888
 889 #else /* !__ARM_16K_PG__ */
 890
 891 /* Native L0 defines */
 892 #define ARM_TT_L0_SIZE       ARM_4K_TT_L0_SIZE
 893 #define ARM_TT_L0_OFFMASK    ARM_4K_TT_L0_OFFMASK
 894 #define ARM_TT_L0_SHIFT      ARM_4K_TT_L0_SHIFT
 895 #define ARM_TT_L0_INDEX_MASK ARM_4K_TT_L0_INDEX_MASK
 896
 897 /* Native L1 defines */
 898 #define ARM_TT_L1_SIZE       ARM_4K_TT_L1_SIZE
 899 #define ARM_TT_L1_OFFMASK    ARM_4K_TT_L1_OFFMASK
 900 #define ARM_TT_L1_SHIFT      ARM_4K_TT_L1_SHIFT
 901 #define ARM_TT_L1_INDEX_MASK ARM_4K_TT_L1_INDEX_MASK
 902
 903 /* Native L2 defines */
 904 #define ARM_TT_L2_SIZE       ARM_4K_TT_L2_SIZE
 905 #define ARM_TT_L2_OFFMASK    ARM_4K_TT_L2_OFFMASK
 906 #define ARM_TT_L2_SHIFT      ARM_4K_TT_L2_SHIFT
 907 #define ARM_TT_L2_INDEX_MASK ARM_4K_TT_L2_INDEX_MASK
 908
 909 /* Native L3 defines */
 910 #define ARM_TT_L3_SIZE       ARM_4K_TT_L3_SIZE
 911 #define ARM_TT_L3_OFFMASK    ARM_4K_TT_L3_OFFMASK
 912 #define ARM_TT_L3_SHIFT      ARM_4K_TT_L3_SHIFT
 913 #define ARM_TT_L3_INDEX_MASK ARM_4K_TT_L3_INDEX_MASK
 914
 915 #endif /* !__ARM_16K_PG__ */
 916
 917 /*
 918  * Convenience definitions for:
 919  *   ARM_TT_LEAF: The last level of the configured page table format.
 920  *   ARM_TT_TWIG: The second to last level of the configured page table format.
 921  *   ARM_TT_ROOT: The first level of the configured page table format.
 922  *
 923  *   My apologies to any botanists who may be reading this.
 924  */
 925 #define ARM_TT_LEAF_SIZE       ARM_TT_L3_SIZE
 926 #define ARM_TT_LEAF_OFFMASK    ARM_TT_L3_OFFMASK
 927 #define ARM_TT_LEAF_SHIFT      ARM_TT_L3_SHIFT
 928 #define ARM_TT_LEAF_INDEX_MASK ARM_TT_L3_INDEX_MASK
 929
 930 #define ARM_TT_TWIG_SIZE       ARM_TT_L2_SIZE
 931 #define ARM_TT_TWIG_OFFMASK    ARM_TT_L2_OFFMASK
 932 #define ARM_TT_TWIG_SHIFT      ARM_TT_L2_SHIFT
 933 #define ARM_TT_TWIG_INDEX_MASK ARM_TT_L2_INDEX_MASK
 934
 935 #define ARM_TT_ROOT_SIZE       ARM_TT_L1_SIZE
 936 #define ARM_TT_ROOT_OFFMASK    ARM_TT_L1_OFFMASK
 937 #define ARM_TT_ROOT_SHIFT      ARM_TT_L1_SHIFT
 938 #define ARM_TT_ROOT_INDEX_MASK ARM_TT_L1_INDEX_MASK
 939
 940 /*
 941  * 4KB granule size:
 942  *
 943  * Level 0 Translation Table Entry
 944  *
 945  *  63 62 61 60  59 58   52 51  48 47                  12 11    2 1 0
 946  * +--+-----+--+---+-------+------+----------------------+-------+-+-+
 947  * |NS|  AP |XN|PXN|ignored| zero | L1TableOutputAddress |ignored|1|V|
 948  * +--+-----+--+---+-------+------+----------------------+-------+-+-+
 949  *
 950  * Level 1 Translation Table Entry
 951  *
 952  *  63 62 61 60  59 58   52 51  48 47                  12 11    2 1 0
 953  * +--+-----+--+---+-------+------+----------------------+-------+-+-+
 954  * |NS|  AP |XN|PXN|ignored| zero | L2TableOutputAddress |ignored|1|V|
 955  * +--+-----+--+---+-------+------+----------------------+-------+-+-+
 956  *
 957  * Level 1 Translation Block Entry
 958  *
 959  *  63 59 58  55 54  53   52 51  48 47                  30 29  12 11 10 9  8 7  6  5 4     2 1 0
 960  * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
 961  * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:30] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V|
 962  * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
 963  *
 964  * Level 2 Translation Table Entry
 965  *
 966  *  63 62 61 60  59 58   52 51  48 47                  12 11    2 1 0
 967  * +--+-----+--+---+-------+------+----------------------+-------+-+-+
 968  * |NS|  AP |XN|PXN|ignored| zero | L3TableOutputAddress |ignored|1|V|
 969  * +--+-----+--+---+-------+------+----------------------+-------+-+-+
 970  *
 971  * Level 2 Translation Block Entry
 972  *
 973  *  63 59 58  55 54  53   52 51  48 47                  21 20  12 11 10 9  8 7  6  5 4     2 1 0
 974  * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
 975  * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:21] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V|
 976  * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
 977  *
 978  * 16KB granule size:
 979  *
 980  * Level 0 Translation Table Entry
 981  *
 982  *  63 62 61 60  59 58   52 51  48 47                  14 13    2 1 0
 983  * +--+-----+--+---+-------+------+----------------------+-------+-+-+
 984  * |NS|  AP |XN|PXN|ignored| zero | L1TableOutputAddress |ignored|1|V|
 985  * +--+-----+--+---+-------+------+----------------------+-------+-+-+
 986  *
 987  * Level 1 Translation Table Entry
 988  *
 989  *  63 62 61 60  59 58   52 51  48 47                  14 13    2 1 0
 990  * +--+-----+--+---+-------+------+----------------------+-------+-+-+
 991  * |NS|  AP |XN|PXN|ignored| zero | L2TableOutputAddress |ignored|1|V|
 992  * +--+-----+--+---+-------+------+----------------------+-------+-+-+
 993  *
 994  * Level 2 Translation Table Entry
 995  *
 996  *  63 62 61 60  59 58   52 51  48 47                  14 13    2 1 0
 997  * +--+-----+--+---+-------+------+----------------------+-------+-+-+
 998  * |NS|  AP |XN|PXN|ignored| zero | L3TableOutputAddress |ignored|1|V|
 999  * +--+-----+--+---+-------+------+----------------------+-------+-+-+
1000  *
1001  * Level 2 Translation Block Entry
1002  *
1003  *  63 59 58  55 54  53   52 51  48 47                  25 24  12 11 10 9  8 7  6  5 4     2 1 0
1004  * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
1005  * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:25] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V|
1006  * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
1007  *
1008  * where:
1009  *   nG:      notGlobal bit
1010  *   SH:      Shareability field
1011  *   AP:      access protection
1012  *   XN:      eXecute Never bit
1013  *   PXN:     Privilege eXecute Never bit
1014  *   NS:      Non-Secure bit
1015  *   HINT:    16 entry continuguous output hint
1016  *   AttrIdx: Memory Attribute Index
1017  */
1018
1019 #define TTE_SHIFT                   3                              /* shift width of a tte (sizeof(tte) == (1 << TTE_SHIFT)) */
1020 #ifdef __ARM_16K_PG__
1021 #define TTE_PGENTRIES               (16384 >> TTE_SHIFT)           /* number of ttes per page */
1022 #else
1023 #define TTE_PGENTRIES               (4096 >> TTE_SHIFT)            /* number of ttes per page */
1024 #endif
1025
1026 #define ARM_TTE_MAX                 (TTE_PGENTRIES)
1027
1028 #define ARM_TTE_EMPTY               0x0000000000000000ULL          /* unasigned - invalid entry */
1029 #define ARM_TTE_TYPE_FAULT          0x0000000000000000ULL          /* unasigned - invalid entry */
1030
1031 #define ARM_TTE_VALID               0x0000000000000001ULL          /* valid entry */
1032
1033 #define ARM_TTE_TYPE_MASK           0x0000000000000002ULL          /* mask for extracting the type */
1034 #define ARM_TTE_TYPE_TABLE          0x0000000000000002ULL          /* page table type */
1035 #define ARM_TTE_TYPE_BLOCK          0x0000000000000000ULL          /* block entry type */
1036 #define ARM_TTE_TYPE_L3BLOCK        0x0000000000000002ULL
1037 #define ARM_TTE_TYPE_MASK           0x0000000000000002ULL          /* mask for extracting the type */
1038
1039 #ifdef __ARM_16K_PG__
1040 /*
1041  * Note that L0/L1 block entries are disallowed for the 16KB granule size; what
1042  * are we doing with these?
1043  */
1044 #define ARM_TTE_BLOCK_SHIFT         12                             /* entry shift for a 16KB L3 TTE entry */
1045 #define ARM_TTE_BLOCK_L0_SHIFT      ARM_TT_L0_SHIFT                /* block shift for 128TB section */
1046 #define ARM_TTE_BLOCK_L1_MASK       0x0000fff000000000ULL          /* mask to extract phys address from L1 block entry */
1047 #define ARM_TTE_BLOCK_L1_SHIFT      ARM_TT_L1_SHIFT                /* block shift for 64GB section */
1048 #define ARM_TTE_BLOCK_L2_MASK       0x0000fffffe000000ULL          /* mask to extract phys address from Level 2 Translation Block entry */
1049 #define ARM_TTE_BLOCK_L2_SHIFT      ARM_TT_L2_SHIFT                /* block shift for 32MB section */
1050 #else
1051 #define ARM_TTE_BLOCK_SHIFT         12                             /* entry shift for a 4KB L3 TTE entry */
1052 #define ARM_TTE_BLOCK_L0_SHIFT      ARM_TT_L0_SHIFT                /* block shift for 2048GB section */
1053 #define ARM_TTE_BLOCK_L1_MASK       0x0000ffffc0000000ULL          /* mask to extract phys address from L1 block entry */
1054 #define ARM_TTE_BLOCK_L1_SHIFT      ARM_TT_L1_SHIFT                /* block shift for 1GB section */
1055 #define ARM_TTE_BLOCK_L2_MASK       0x0000ffffffe00000ULL          /* mask to extract phys address from Level 2 Translation Block entry */
1056 #define ARM_TTE_BLOCK_L2_SHIFT      ARM_TT_L2_SHIFT                /* block shift for 2MB section */
1057 #endif
1058
1059 #define ARM_TTE_BLOCK_APSHIFT       6
1060 #define ARM_TTE_BLOCK_AP(x)         ((x)<<ARM_TTE_BLOCK_APSHIFT)   /* access protection */
1061 #define ARM_TTE_BLOCK_APMASK        (0x3 << ARM_TTE_BLOCK_APSHIFT)
1062
1063 #define ARM_TTE_BLOCK_ATTRINDX(x)   ((x) << 2)                     /* memory attributes index */
1064 #define ARM_TTE_BLOCK_ATTRINDXMASK  (0x7ULL << 2)                  /* mask memory attributes index */
1065
1066 #define ARM_TTE_BLOCK_SH(x)         ((x) << 8)                     /* access shared */
1067 #define ARM_TTE_BLOCK_SHMASK        (0x3ULL << 8)                  /* mask access shared */
1068
1069 #define ARM_TTE_BLOCK_AF            0x0000000000000400ULL          /* value for access */
1070 #define ARM_TTE_BLOCK_AFMASK        0x0000000000000400ULL          /* access mask */
1071
1072 #define ARM_TTE_BLOCK_NG            0x0000000000000800ULL          /* value for a global mapping */
1073 #define ARM_TTE_BLOCK_NG_MASK       0x0000000000000800ULL          /* notGlobal mapping mask */
1074
1075 #define ARM_TTE_BLOCK_NS            0x0000000000000020ULL          /* value for a secure mapping */
1076 #define ARM_TTE_BLOCK_NS_MASK       0x0000000000000020ULL          /* notSecure mapping mask */
1077
1078 #define ARM_TTE_BLOCK_PNX           0x0020000000000000ULL          /* value for privilege no execute bit */
1079 #define ARM_TTE_BLOCK_PNXMASK       0x0020000000000000ULL          /* privilege no execute mask */
1080
1081 #define ARM_TTE_BLOCK_NX            0x0040000000000000ULL          /* value for no execute */
1082 #define ARM_TTE_BLOCK_NXMASK        0x0040000000000000ULL          /* no execute mask */
1083
1084 #define ARM_TTE_BLOCK_WIRED         0x0400000000000000ULL          /* value for software wired bit */
1085 #define ARM_TTE_BLOCK_WIREDMASK     0x0400000000000000ULL          /* software wired mask */
1086
1087 #define ARM_TTE_BLOCK_WRITEABLE     0x0800000000000000ULL          /* value for software writeable bit */
1088 #define ARM_TTE_BLOCK_WRITEABLEMASK 0x0800000000000000ULL          /* software writeable mask */
1089
1090 #ifdef __ARM_16K_PG__
1091 /*
1092  * TODO: Do we care about the low bits being unused?  It should technically
1093  * work either way, but masking them out should be future proof; it is only a
1094  * matter of time before someone wants to shove something into the free bits.
1095  */
1096 #define ARM_TTE_TABLE_MASK          (0x0000ffffffffc000ULL)        /* mask for extracting pointer to next table (works at any level) */
1097 #else
1098 #define ARM_TTE_TABLE_MASK          (0x0000fffffffff000ULL)        /* mask for extracting pointer to next table (works at any level) */
1099 #endif
1100
1101 #define ARM_TTE_TABLE_APSHIFT       61
1102 #define ARM_TTE_TABLE_AP(x)         ((x)<<TTE_BLOCK_APSHIFT)       /* access protection */
1103
1104 #define ARM_TTE_TABLE_NS            0x8000000000000020ULL          /* value for a secure mapping */
1105 #define ARM_TTE_TABLE_NS_MASK       0x8000000000000020ULL          /* notSecure mapping mask */
1106
1107 #define ARM_TTE_TABLE_XN            0x1000000000000000ULL          /* value for no execute */
1108 #define ARM_TTE_TABLE_XNMASK        0x1000000000000000ULL          /* no execute mask */
1109
1110 #define ARM_TTE_TABLE_PXN           0x0800000000000000ULL          /* value for privilege no execute bit */
1111 #define ARM_TTE_TABLE_PXNMASK       0x0800000000000000ULL          /* privilege execute mask */
1112
1113 #if __ARM_KERNEL_PROTECT__
1114 #define ARM_TTE_BOOT_BLOCK \
1115         (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) | \
1116          ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF | ARM_TTE_BLOCK_NG)
1117 #else /* __ARM_KERNEL_PROTECT__ */
1118 #define ARM_TTE_BOOT_BLOCK \
1119         (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) | \
1120          ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF)
1121 #endif /* __ARM_KERNEL_PROTECT__ */
1122
1123 #define ARM_TTE_BOOT_TABLE (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID )
1124 /*
1125  *  L3 Translation table
1126  *
1127  *  4KB granule size:
1128  *    Each translation table is 4KB
1129  *    512 64-bit entries of 4KB (2^12) of address space.
1130  *    Covers 2MB (2^21) of address space.
1131  *
1132  *  16KB granule size:
1133  *    Each translation table is 16KB
1134  *    2048 64-bit entries of 16KB (2^14) of address space.
1135  *    Covers 32MB (2^25) of address space.
1136  */
1137
1138 #ifdef __ARM_16K_PG__
1139 #define ARM_PTE_SIZE    0x0000000000004000ULL /* size of area covered by a tte */
1140 #define ARM_PTE_OFFMASK 0x0000000000003fffULL /* offset within pte area */
1141 #define ARM_PTE_SHIFT   14                    /* page descriptor shift */
1142 #define ARM_PTE_MASK    0x0000ffffffffc000ULL /* mask for output address in PTE */
1143 #else
1144 #define ARM_PTE_SIZE    0x0000000000001000ULL /* size of area covered by a tte */
1145 #define ARM_PTE_OFFMASK 0x0000000000000fffULL /* offset within pte area */
1146 #define ARM_PTE_SHIFT   12                    /* page descriptor shift */
1147 #define ARM_PTE_MASK    0x0000fffffffff000ULL /* mask for output address in PTE */
1148 #endif
1149
1150 /*
1151  * L3 Page table entries
1152  *
1153  * The following page table entry types are possible:
1154  *
1155  * fault page entry
1156  *  63                            2  0
1157  * +------------------------------+--+
1158  * |    ignored                   |00|
1159  * +------------------------------+--+
1160  *
1161  *
1162  *  63 59 58  55 54  53   52 51  48 47                  12 11 10 9  8 7  6  5 4     2 1 0
1163  * +-----+------+--+---+----+------+----------------------+--+--+----+----+--+-------+-+-+
1164  * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:12] |nG|AF| SH | AP |NS|AttrIdx|1|V|
1165  * +-----+------+--+---+----+------+----------------------+--+--+----+----+--+-------+-+-+
1166  *
1167  * where:
1168  *   nG:      notGlobal bit
1169  *   SH:      Shareability field
1170  *   AP:      access protection
1171  *   XN:      eXecute Never bit
1172  *   PXN:     Privilege eXecute Never bit
1173  *   NS:      Non-Secure bit
1174  *   HINT:    16 entry continuguous output hint
1175  *   AttrIdx: Memory Attribute Index
1176  */
1177
1178 #define PTE_SHIFT               3                     /* shift width of a pte (sizeof(pte) == (1 << PTE_SHIFT)) */
1179 #ifdef __ARM_16K_PG__
1180 #define PTE_PGENTRIES           (16384 >> PTE_SHIFT)  /* number of ptes per page */
1181 #else
1182 #define PTE_PGENTRIES           (4096 >> PTE_SHIFT)   /* number of ptes per page */
1183 #endif
1184
1185 #define ARM_PTE_EMPTY           0x0000000000000000ULL /* unassigned - invalid entry */
1186
1187 /* markers for (invalid) PTE for a page sent to compressor */
1188 #define ARM_PTE_COMPRESSED      0x8000000000000000ULL /* compressed... */
1189 #define ARM_PTE_COMPRESSED_ALT  0x4000000000000000ULL /* ... and was "alt_acct" */
1190 #define ARM_PTE_COMPRESSED_MASK 0xC000000000000000ULL
1191
1192 #define ARM_PTE_IS_COMPRESSED(x, p) \
1193         ((((x) & 0x3) == 0) && /* PTE is not valid... */                      \
1194          ((x) & ARM_PTE_COMPRESSED) && /* ...has "compressed" marker" */      \
1195          ((!((x) & ~ARM_PTE_COMPRESSED_MASK)) || /* ...no other bits */       \
1196          (panic("compressed PTE %p 0x%llx has extra bits 0x%llx: corrupted?", \
1197                 (p), (x), (x) & ~ARM_PTE_COMPRESSED_MASK), FALSE)))
1198
1199 #define ARM_PTE_TYPE               0x0000000000000003ULL /* valid L3 entry: includes bit #1 (counterintuitively) */
1200 #define ARM_PTE_TYPE_VALID         0x0000000000000003ULL /* valid L3 entry: includes bit #1 (counterintuitively) */
1201 #define ARM_PTE_TYPE_FAULT         0x0000000000000000ULL /* invalid L3 entry */
1202 #define ARM_PTE_TYPE_MASK          0x0000000000000002ULL /* mask to get pte type */
1203
1204 #ifdef __ARM_16K_PG__
1205 /* TODO: What does the shift mean here? */
1206 #define ARM_PTE_PAGE_MASK          0x0000FFFFFFFFC000ULL /* mask for 16KB page */
1207 #else
1208 #define ARM_PTE_PAGE_MASK          0x0000FFFFFFFFF000ULL /* mask for  4KB page */
1209 #define ARM_PTE_PAGE_SHIFT         12                    /* page shift for 4KB page */
1210 #endif
1211
1212 #define ARM_PTE_AP(x)              ((x) << 6)            /* access protections */
1213 #define ARM_PTE_APMASK             (0x3ULL << 6)         /* mask access protections */
1214 #define ARM_PTE_EXTRACT_AP(x)      (((x) >> 6) & 0x3ULL) /* extract access protections from PTE */
1215
1216 #define ARM_PTE_ATTRINDX(x)        ((x) << 2)            /* memory attributes index */
1217 #define ARM_PTE_ATTRINDXMASK       (0x7ULL << 2)         /* mask memory attributes index */
1218
1219 #define ARM_PTE_SH(x)              ((x) << 8)            /* access shared */
1220 #define ARM_PTE_SHMASK             (0x3ULL << 8)         /* mask access shared */
1221
1222 #define ARM_PTE_AF                 0x0000000000000400ULL /* value for access */
1223 #define ARM_PTE_AFMASK             0x0000000000000400ULL /* access mask */
1224
1225 #define ARM_PTE_NG                 0x0000000000000800ULL /* value for a global mapping */
1226 #define ARM_PTE_NG_MASK            0x0000000000000800ULL /* notGlobal mapping mask */
1227
1228 #define ARM_PTE_NS                 0x0000000000000020ULL /* value for a secure mapping */
1229 #define ARM_PTE_NS_MASK            0x0000000000000020ULL /* notSecure mapping mask */
1230
1231 #define ARM_PTE_HINT               0x0010000000000000ULL /* value for contiguous entries hint */
1232 #define ARM_PTE_HINT_MASK          0x0010000000000000ULL /* mask for contiguous entries hint */
1233
1234 #if __ARM_16K_PG__
1235 #define ARM_PTE_HINT_ENTRIES       128ULL                /* number of entries the hint covers */
1236 #define ARM_PTE_HINT_ENTRIES_SHIFT 7ULL                  /* shift to construct the number of entries */
1237 #define ARM_PTE_HINT_ADDR_MASK     0x0000FFFFFFE00000ULL /* mask to extract the starting hint address */
1238 #define ARM_PTE_HINT_ADDR_SHIFT    21                    /* shift for the hint address */
1239 #define ARM_KVA_HINT_ADDR_MASK     0xFFFFFFFFFFE00000ULL /* mask to extract the starting hint address */
1240 #else
1241 #define ARM_PTE_HINT_ENTRIES       16ULL                 /* number of entries the hint covers */
1242 #define ARM_PTE_HINT_ENTRIES_SHIFT 4ULL                  /* shift to construct the number of entries */
1243 #define ARM_PTE_HINT_ADDR_MASK     0x0000FFFFFFFF0000ULL /* mask to extract the starting hint address */
1244 #define ARM_PTE_HINT_ADDR_SHIFT    16                    /* shift for the hint address */
1245 #define ARM_KVA_HINT_ADDR_MASK     0xFFFFFFFFFFFF0000ULL /* mask to extract the starting hint address */
1246 #endif
1247
1248 #define ARM_PTE_PNX                0x0020000000000000ULL /* value for privilege no execute bit */
1249 #define ARM_PTE_PNXMASK            0x0020000000000000ULL /* privilege no execute mask */
1250
1251 #define ARM_PTE_NX                 0x0040000000000000ULL /* value for no execute bit */
1252 #define ARM_PTE_NXMASK             0x0040000000000000ULL /* no execute mask */
1253
1254 #define ARM_PTE_WIRED              0x0400000000000000ULL /* value for software wired bit */
1255 #define ARM_PTE_WIRED_MASK         0x0400000000000000ULL /* software wired mask */
1256
1257 #define ARM_PTE_WRITEABLE          0x0800000000000000ULL /* value for software writeable bit */
1258 #define ARM_PTE_WRITEABLE_MASK     0x0800000000000000ULL /* software writeable mask */
1259
1260 #if CONFIG_PGTRACE
1261 #define ARM_PTE_PGTRACE            0x0200000000000000ULL /* value for software trace bit */
1262 #define ARM_PTE_PGTRACE_MASK       0x0200000000000000ULL /* software trace mask */
1263 #endif
1264
1265 #define ARM_PTE_BOOT_PAGE_BASE \
1266         (ARM_PTE_TYPE_VALID | ARM_PTE_SH(SH_OUTER_MEMORY) |       \
1267          ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_PTE_AF)
1268
1269 #if __ARM_KERNEL_PROTECT__
1270 #define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE | ARM_PTE_NG)
1271 #else /* __ARM_KERNEL_PROTECT__ */
1272 #define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE)
1273 #endif /* __ARM_KERNEL_PROTECT__ */
1274
1275 /*
1276  * TLBI appers to only deal in 4KB page addresses, so give
1277  * it an explicit shift of 12.
1278  */
1279 #define TLBI_ADDR_SHIFT (0)
1280 #define TLBI_ADDR_SIZE  (44)
1281 #define TLBI_ADDR_MASK  ((1ULL << TLBI_ADDR_SIZE) - 1)
1282 #define TLBI_ASID_SHIFT (48)
1283 #define TLBI_ASID_SIZE  (16)
1284 #define TLBI_ASID_MASK  (((1ULL << TLBI_ASID_SIZE) - 1))
1285
1286 #define RTLBI_ADDR_SIZE (37)
1287 #define RTLBI_ADDR_MASK ((1ULL << RTLBI_ADDR_SIZE) - 1)
1288 #define RTLBI_ADDR_SHIFT ARM_TT_L3_SHIFT
1289 #define RTLBI_TG ((uint64_t)(((ARM_TT_L3_SHIFT - 12) >> 1) + 1) << 46)
1290 #define RTLBI_SCALE_SHIFT (44)
1291 #define RTLBI_NUM_SHIFT (39)
1292
1293 /*
1294  * Exception Syndrome Register
1295  *
1296  *  31  26 25 24               0
1297  * +------+--+------------------+
1298  * |  EC  |IL|       ISS        |
1299  * +------+--+------------------+
1300  *
1301  * EC  - Exception Class
1302  * IL  - Instruction Length
1303  * ISS - Instruction Specific Syndrome
1304  *
1305  * Note: The ISS can have many forms. These are defined separately below.
1306  */
1307
1308 #define ESR_EC_SHIFT           26
1309 #define ESR_EC_MASK            (0x3FULL << ESR_EC_SHIFT)
1310 #define ESR_EC(x)              ((x & ESR_EC_MASK) >> ESR_EC_SHIFT)
1311
1312 #define ESR_IL_SHIFT           25
1313 #define ESR_IL                 (1 << ESR_IL_SHIFT)
1314
1315 #define ESR_INSTR_IS_2BYTES(x) (!(x & ESR_IL))
1316
1317 #define ESR_ISS_MASK           0x01FFFFFF
1318 #define ESR_ISS(x)             (x & ESR_ISS_MASK)
1319
1320 #ifdef __ASSEMBLER__
1321 /* Define only the classes we need to test in the exception vectors. */
1322 #define ESR_EC_IABORT_EL1      0x21
1323 #define ESR_EC_DABORT_EL1      0x25
1324 #define ESR_EC_SP_ALIGN        0x26
1325 #else
1326 typedef enum {
1327         ESR_EC_UNCATEGORIZED       = 0x00,
1328         ESR_EC_WFI_WFE             = 0x01,
1329         ESR_EC_MCR_MRC_CP15_TRAP   = 0x03,
1330         ESR_EC_MCRR_MRRC_CP15_TRAP = 0x04,
1331         ESR_EC_MCR_MRC_CP14_TRAP   = 0x05,
1332         ESR_EC_LDC_STC_CP14_TRAP   = 0x06,
1333         ESR_EC_TRAP_SIMD_FP        = 0x07,
1334         ESR_EC_MCRR_MRRC_CP14_TRAP = 0x0c,
1335         ESR_EC_ILLEGAL_INSTR_SET   = 0x0e,
1336         ESR_EC_SVC_32              = 0x11,
1337         ESR_EC_SVC_64              = 0x15,
1338         ESR_EC_MSR_TRAP            = 0x18,
1339         ESR_EC_IABORT_EL0          = 0x20,
1340         ESR_EC_IABORT_EL1          = 0x21,
1341         ESR_EC_PC_ALIGN            = 0x22,
1342         ESR_EC_DABORT_EL0          = 0x24,
1343         ESR_EC_DABORT_EL1          = 0x25,
1344         ESR_EC_SP_ALIGN            = 0x26,
1345         ESR_EC_FLOATING_POINT_32   = 0x28,
1346         ESR_EC_FLOATING_POINT_64   = 0x2C,
1347         ESR_EC_BKPT_REG_MATCH_EL0  = 0x30, // Breakpoint Debug event taken to the EL from a lower EL.
1348         ESR_EC_BKPT_REG_MATCH_EL1  = 0x31, // Breakpoint Debug event taken to the EL from the EL.
1349         ESR_EC_SW_STEP_DEBUG_EL0   = 0x32, // Software Step Debug event taken to the EL from a lower EL.
1350         ESR_EC_SW_STEP_DEBUG_EL1   = 0x33, // Software Step Debug event taken to the EL from the EL.
1351         ESR_EC_WATCHPT_MATCH_EL0   = 0x34, // Watchpoint Debug event taken to the EL from a lower EL.
1352         ESR_EC_WATCHPT_MATCH_EL1   = 0x35, // Watchpoint Debug event taken to the EL from the EL.
1353         ESR_EC_BKPT_AARCH32        = 0x38,
1354         ESR_EC_BRK_AARCH64         = 0x3C,
1355 } esr_exception_class_t;
1356
1357 typedef enum {
1358         FSC_TRANSLATION_FAULT_L0   = 0x04,
1359         FSC_TRANSLATION_FAULT_L1   = 0x05,
1360         FSC_TRANSLATION_FAULT_L2   = 0x06,
1361         FSC_TRANSLATION_FAULT_L3   = 0x07,
1362         FSC_ACCESS_FLAG_FAULT_L1   = 0x09,
1363         FSC_ACCESS_FLAG_FAULT_L2   = 0x0A,
1364         FSC_ACCESS_FLAG_FAULT_L3   = 0x0B,
1365         FSC_PERMISSION_FAULT_L1    = 0x0D,
1366         FSC_PERMISSION_FAULT_L2    = 0x0E,
1367         FSC_PERMISSION_FAULT_L3    = 0x0F,
1368         FSC_SYNC_EXT_ABORT         = 0x10,
1369         FSC_ASYNC_EXT_ABORT        = 0x11,
1370         FSC_SYNC_EXT_ABORT_TT_L1   = 0x15,
1371         FSC_SYNC_EXT_ABORT_TT_L2   = 0x16,
1372         FSC_SYNC_EXT_ABORT_TT_L3   = 0x17,
1373         FSC_SYNC_PARITY            = 0x18,
1374         FSC_ASYNC_PARITY           = 0x19,
1375         FSC_SYNC_PARITY_TT_L1      = 0x1D,
1376         FSC_SYNC_PARITY_TT_L2      = 0x1E,
1377         FSC_SYNC_PARITY_TT_L3      = 0x1F,
1378         FSC_ALIGNMENT_FAULT        = 0x21,
1379         FSC_DEBUG_FAULT            = 0x22
1380 } fault_status_t;
1381 #endif /* ASSEMBLER */
1382
1383 /*
1384  * Software step debug event ISS (EL1)
1385  *  24  23                6  5    0
1386  * +---+-----------------+--+------+
1387  * |ISV|00000000000000000|EX| IFSC |
1388  * +---+-----------------+--+------+
1389  *
1390  * where:
1391  *   ISV:  Instruction syndrome valid
1392  *   EX:   Exclusive access
1393  *   IFSC: Instruction Fault Status Code
1394  */
1395
1396 #define ISS_SSDE_ISV_SHIFT 24
1397 #define ISS_SSDE_ISV       (0x1 << ISS_SSDE_ISV_SHIFT)
1398
1399 #define ISS_SSDE_EX_SHIFT  6
1400 #define ISS_SSDE_EX        (0x1 << ISS_SSDE_EX_SHIFT)
1401
1402 #define ISS_SSDE_FSC_MASK  0x3F
1403 #define ISS_SSDE_FSC(x)    (x & ISS_SSDE_FSC_MASK)
1404
1405 /*
1406  * Instruction Abort ISS (EL1)
1407  *  24           10 9      5    0
1408  * +---------------+--+---+------+
1409  * |000000000000000|EA|000| IFSC |
1410  * +---------------+--+---+------+
1411  *
1412  * where:
1413  *   EA:   External Abort type
1414  *   IFSC: Instruction Fault Status Code
1415  */
1416
1417 #define ISS_IA_EA_SHIFT 9
1418 #define ISS_IA_EA       (0x1 << ISS_IA_EA_SHIFT)
1419
1420 #define ISS_IA_FSC_MASK 0x3F
1421 #define ISS_IA_FSC(x)   (x & ISS_IA_FSC_MASK)
1422
1423
1424 /*
1425  * Data Abort ISS (EL1)
1426  *
1427  *  24              9  8  7  6  5  0
1428  * +---------------+--+--+-+---+----+
1429  * |000000000000000|EA|CM|0|WnR|DFSC|
1430  * +---------------+--+--+-+---+----+
1431  *
1432  * where:
1433  *   EA:   External Abort type
1434  *   CM:   Cache Maintenance operation
1435  *   WnR:  Write not Read
1436  *   DFSC: Data Fault Status Code
1437  */
1438 #define ISS_DA_EA_SHIFT  9
1439 #define ISS_DA_EA        (0x1 << ISS_DA_EA_SHIFT)
1440
1441 #define ISS_DA_CM_SHIFT  8
1442 #define ISS_DA_CM        (0x1 << ISS_DA_CM_SHIFT)
1443
1444 #define ISS_DA_WNR_SHIFT 6
1445 #define ISS_DA_WNR       (0x1 << ISS_DA_WNR_SHIFT)
1446
1447 #define ISS_DA_FSC_MASK  0x3F
1448 #define ISS_DA_FSC(x)    (x & ISS_DA_FSC_MASK)
1449
1450 /*
1451  * Floating Point Exception ISS (EL1)
1452  *
1453  * 24  23 22            8  7      4   3   2   1   0
1454  * +-+---+---------------+---+--+---+---+---+---+---+
1455  * |0|TFV|000000000000000|IDF|00|IXF|UFF|OFF|DZF|IOF|
1456  * +-+---+---------------+---+--+---+---+---+---+---+
1457  *
1458  * where:
1459  *   TFV: Trapped Fault Valid
1460  *   IDF: Input Denormal Exception
1461  *   IXF: Input Inexact Exception
1462  *   UFF: Underflow Exception
1463  *   OFF: Overflow Exception
1464  *   DZF: Divide by Zero Exception
1465  *   IOF: Invalid Operation Exception
1466  */
1467 #define ISS_FP_TFV_SHIFT 23
1468 #define ISS_FP_TFV       (0x1 << ISS_FP_TFV_SHIFT)
1469
1470 #define ISS_FP_IDF_SHIFT 7
1471 #define ISS_FP_IDF       (0x1 << ISS_FP_IDF_SHIFT)
1472
1473 #define ISS_FP_IXF_SHIFT 4
1474 #define ISS_FP_IXF       (0x1 << ISS_FP_IXF_SHIFT)
1475
1476 #define ISS_FP_UFF_SHIFT 3
1477 #define ISS_FP_UFF       (0x1 << ISS_FP_UFF_SHIFT)
1478
1479 #define ISS_FP_OFF_SHIFT 2
1480 #define ISS_FP_OFF       (0x1 << ISS_FP_OFF_SHIFT)
1481
1482 #define ISS_FP_DZF_SHIFT 1
1483 #define ISS_FP_DZF       (0x1 << ISS_FP_DZF_SHIFT)
1484
1485 #define ISS_FP_IOF_SHIFT 0
1486 #define ISS_FP_IOF       (0x1 << ISS_FP_IOF_SHIFT)
1487
1488
1489 /*
1490  * Physical Address Register (EL1)
1491  */
1492 #define PAR_F_SHIFT 0
1493 #define PAR_F       (0x1 << PAR_F_SHIFT)
1494
1495 #define PLATFORM_SYSCALL_TRAP_NO 0x80000000
1496
1497 #define ARM64_SYSCALL_CODE_REG_NUM (16)
1498
1499 #define ARM64_CLINE_SHIFT 6
1500
1501 #if defined(APPLE_ARM64_ARCH_FAMILY)
1502 #define L2CERRSTS_DATSBEESV (1ULL << 2) /* L2C data single bit ECC error */
1503 #define L2CERRSTS_DATDBEESV (1ULL << 4) /* L2C data double bit ECC error */
1504 #endif
1505
1506 /*
1507  * Timer definitions.
1508  */
1509 #define CNTKCTL_EL1_PL0PTEN      (0x1 << 9)           /* 1: EL0 access to physical timer regs permitted */
1510 #define CNTKCTL_EL1_PL0VTEN      (0x1 << 8)           /* 1: EL0 access to virtual timer regs permitted */
1511 #define CNTKCTL_EL1_EVENTI_MASK  (0x000000f0)         /* Mask for bits describing which bit to use for triggering event stream */
1512 #define CNTKCTL_EL1_EVENTI_SHIFT (0x4)                /* Shift for same */
1513 #define CNTKCTL_EL1_EVENTDIR     (0x1 << 3)           /* 1: one-to-zero transition of specified bit causes event */
1514 #define CNTKCTL_EL1_EVNTEN       (0x1 << 2)           /* 1: enable event stream */
1515 #define CNTKCTL_EL1_PL0VCTEN     (0x1 << 1)           /* 1: EL0 access to physical timebase + frequency reg enabled */
1516 #define CNTKCTL_EL1_PL0PCTEN     (0x1 << 0)           /* 1: EL0 access to virtual timebase + frequency reg enabled */
1517
1518 #define CNTV_CTL_EL0_ISTATUS     (0x1 << 2)           /* (read only): whether interrupt asserted */
1519 #define CNTV_CTL_EL0_IMASKED     (0x1 << 1)           /* 1: interrupt masked */
1520 #define CNTV_CTL_EL0_ENABLE      (0x1 << 0)           /* 1: virtual timer enabled */
1521
1522 #define CNTP_CTL_EL0_ISTATUS     CNTV_CTL_EL0_ISTATUS
1523 #define CNTP_CTL_EL0_IMASKED     CNTV_CTL_EL0_IMASKED
1524 #define CNTP_CTL_EL0_ENABLE      CNTV_CTL_EL0_ENABLE
1525
1526 /*
1527  * At present all other uses of ARM_DBG_* are shared bit compatibly with the 32bit definitons.
1528  * (cf. osfmk/arm/proc_reg.h)
1529  */
1530 #define ARM_DBG_VR_ADDRESS_MASK64 0xFFFFFFFFFFFFFFFCull /* BVR & WVR */
1531
1532 #define MIDR_EL1_REV_SHIFT  0
1533 #define MIDR_EL1_REV_MASK   (0xf << MIDR_EL1_REV_SHIFT)
1534 #define MIDR_EL1_PNUM_SHIFT 4
1535 #define MIDR_EL1_PNUM_MASK  (0xfff << MIDR_EL1_PNUM_SHIFT)
1536 #define MIDR_EL1_ARCH_SHIFT 16
1537 #define MIDR_EL1_ARCH_MASK  (0xf << MIDR_EL1_ARCH_SHIFT)
1538 #define MIDR_EL1_VAR_SHIFT  20
1539 #define MIDR_EL1_VAR_MASK   (0xf << MIDR_EL1_VAR_SHIFT)
1540 #define MIDR_EL1_IMP_SHIFT  24
1541 #define MIDR_EL1_IMP_MASK   (0xff << MIDR_EL1_IMP_SHIFT)
1542
1543 /*
1544  * CoreSight debug registers
1545  */
1546 #define CORESIGHT_ED  0
1547 #define CORESIGHT_CTI 1
1548 #define CORESIGHT_PMU 2
1549 #define CORESIGHT_UTT 3 /* Not truly a coresight thing, but at a fixed convenient location right after the coresight region */
1550
1551 #define CORESIGHT_OFFSET(x) ((x) * 0x10000)
1552 #define CORESIGHT_REGIONS   4
1553 #define CORESIGHT_SIZE      0x1000
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564 /*
1565  * ID_AA64ISAR0_EL1 - AArch64 Instruction Set Attribute Register 0
1566  *
1567  *  63      24 23    20 19  16 15  12 11   8 7   4 3    0
1568  * +----------+--------+------+------+------+-----+------+
1569  * | reserved | atomic |crc32 | sha2 | sha1 | aes | res0 |
1570  * +----------+--------+------+------+------+-----+------+
1571  */
1572
1573 #define ID_AA64ISAR0_EL1_FHM_OFFSET    48
1574 #define ID_AA64ISAR0_EL1_FHM_MASK      (0xfull << ID_AA64ISAR0_EL1_FHM_OFFSET)
1575 #define ID_AA64ISAR0_EL1_FHM_8_2       (1ull << ID_AA64ISAR0_EL1_FHM_OFFSET)
1576
1577 #define ID_AA64ISAR0_EL1_ATOMIC_OFFSET 20
1578 #define ID_AA64ISAR0_EL1_ATOMIC_MASK   (0xfull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET)
1579 #define ID_AA64ISAR0_EL1_ATOMIC_8_1    (2ull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET)
1580
1581 #define ID_AA64ISAR0_EL1_CRC32_OFFSET  16
1582 #define ID_AA64ISAR0_EL1_CRC32_MASK    (0xfull << ID_AA64ISAR0_EL1_CRC32_OFFSET)
1583 #define ID_AA64ISAR0_EL1_CRC32_EN      (1ull << ID_AA64ISAR0_EL1_CRC32_OFFSET)
1584
1585 #define ID_AA64ISAR0_EL1_SHA2_OFFSET   12
1586 #define ID_AA64ISAR0_EL1_SHA2_MASK     (0xfull << ID_AA64ISAR0_EL1_SHA2_OFFSET)
1587 #define ID_AA64ISAR0_EL1_SHA2_EN       (1ull << ID_AA64ISAR0_EL1_SHA2_OFFSET)
1588
1589 #define ID_AA64ISAR0_EL1_SHA1_OFFSET   8
1590 #define ID_AA64ISAR0_EL1_SHA1_MASK     (0xfull << ID_AA64ISAR0_EL1_SHA1_OFFSET)
1591 #define ID_AA64ISAR0_EL1_SHA1_EN       (1ull << ID_AA64ISAR0_EL1_SHA1_OFFSET)
1592
1593 #define ID_AA64ISAR0_EL1_AES_OFFSET    4
1594 #define ID_AA64ISAR0_EL1_AES_MASK      (0xfull << ID_AA64ISAR0_EL1_AES_OFFSET)
1595 #define ID_AA64ISAR0_EL1_AES_EN        (1ull << ID_AA64ISAR0_EL1_AES_OFFSET)
1596 #define ID_AA64ISAR0_EL1_AES_PMULL_EN  (2ull << ID_AA64ISAR0_EL1_AES_OFFSET)
1597
1598
1599 #if __APCFG_SUPPORTED__
1600 /*
1601  * APCFG_EL1
1602  *
1603  *  63       2 1 0
1604  * +----------+-+-+
1605  * | reserved |K|R|
1606  * +----------+-+-+
1607  *
1608  * where:
1609  *   R: Reserved
1610  *   K: ElXEnKey - Enable ARMV8.3 defined {IA,IB,DA,DB} keys when CPU is
1611  *                 operating in EL1 (or higher) and when under Apple-Mode
1612  */
1613
1614 #define APCFG_EL1_ELXENKEY_OFFSET      1
1615 #define APCFG_EL1_ELXENKEY_MASK        (0x1ULL << APCFG_EL1_ELXENKEY_OFFSET)
1616 #define APCFG_EL1_ELXENKEY             APCFG_EL1_ELXENKEY_MASK
1617 #endif /* __APCFG_SUPPORTED__ */
1618
1619 #define APSTATE_G_SHIFT  (0)
1620 #define APSTATE_P_SHIFT  (1)
1621 #define APSTATE_A_SHIFT  (2)
1622
1623 #ifdef __APSTS_SUPPORTED__
1624 #define APCTL_EL1_AppleMode  (1ULL << 0)
1625 #define APCTL_EL1_KernKeyEn  (1ULL << 1)
1626 #define APCTL_EL1_EnAPKey0   (1ULL << 2)
1627 #define APCTL_EL1_EnAPKey1   (1ULL << 3)
1628 #define APSTS_EL1_MKEYVld    (1ULL << 0)
1629 #else
1630 #define APCTL_EL1_AppleMode  (1ULL << 0)
1631 #define APCTL_EL1_MKEYVld    (1ULL << 1)
1632 #define APCTL_EL1_KernKeyEn  (1ULL << 2)
1633 #endif
1634
1635 #define ACTLR_EL1_DisHWP_OFFSET  3
1636 #define ACTLR_EL1_DisHWP_MASK    (1ULL << ACTLR_EL1_DisHWP_OFFSET)
1637 #define ACTLR_EL1_DisHWP         ACTLR_EL1_DisHWP_MASK
1638
1639
1640 #if defined(HAS_APPLE_PAC)
1641 // The value of ptrauth_string_discriminator("recover"), hardcoded so it can be used from assembly code
1642 #define PAC_DISCRIMINATOR_RECOVER    0x1e02
1643 #endif
1644
1645 #ifdef __ASSEMBLER__
1646
1647 /*
1648  * Compute CPU version:
1649  * Version is constructed as [4 bits of MIDR variant]:[4 bits of MIDR revision]
1650  *
1651  * Where the "variant" is the major number and the "revision" is the minor number.
1652  *
1653  * For example:
1654  *   Cyclone A0 is variant 0, revision 0, i.e. 0.
1655  *   Cyclone B0 is variant 1, revision 0, i.e. 0x10
1656  * $0 - register to place value in
1657  */
1658 .macro GET_MIDR_CPU_VERSION
1659 mrs  $0, MIDR_EL1                                  // Read MIDR_EL1 for CPUID
1660 bfi  $0, $0, #(MIDR_EL1_VAR_SHIFT - 4), #4         // move bits 3:0 (revision) to 19:16 (below variant) to get values adjacent
1661 ubfx $0, $0, #(MIDR_EL1_VAR_SHIFT - 4), #8         // And extract the concatenated bitstring to beginning of register
1662 .endmacro
1663
1664 /*
1665  * To apply a workaround for CPU versions less than a given value
1666  * (e.g. earlier than when a fix arrived)
1667  *
1668  * $0 - scratch register1
1669  * $1 - version at which to stop applying workaround
1670  * $2 - label to branch to  (at end of workaround)
1671  */
1672 .macro SKIP_IF_CPU_VERSION_GREATER_OR_EQUAL
1673 GET_MIDR_CPU_VERSION $0
1674 cmp  $0, $1
1675 b.pl $2                         // Unsigned "greater or equal"
1676 .endmacro
1677
1678 /*
1679  * To apply a workaround for CPU versions greater than a given value
1680  * (e.g. starting when a bug was introduced)
1681  *
1682  * $0 - scratch register1
1683  * $1 - version at which to stop applying workaround
1684  * $2 - label to branch to  (at end of workaround)
1685  */
1686 .macro SKIP_IF_CPU_VERSION_LESS_THAN
1687 GET_MIDR_CPU_VERSION $0
1688 cmp  $0, $1
1689 b.mi $2                         // Unsigned "strictly less than"
1690 .endmacro
1691
1692 #endif /* __ASSEMBLER__ */
1693
1694 #define MSR(reg, src)  __asm__ volatile ("msr " reg ", %0" :: "r" (src))
1695 #define MRS(dest, reg) __asm__ volatile ("mrs %0, " reg : "=r" (dest))
1696
1697
1698 #endif /* _ARM64_PROC_REG_H_ */