osfmk/arm64/proc_reg.h

   1 /*
   2  * Copyright (c) 2007-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * Processor registers for ARM64
  30  */
  31 #ifndef _ARM64_PROC_REG_H_
  32 #define _ARM64_PROC_REG_H_
  33
  34 #include <arm/proc_reg.h>
  35
  36 #if __ARM_KERNEL_PROTECT__
  37 /*
  38  * __ARM_KERNEL_PROTECT__ is a feature intended to guard against potential
  39  * architectural or microarchitectural vulnerabilities that could allow cores to
  40  * read/access EL1-only mappings while in EL0 mode.  This is achieved by
  41  * removing as many mappings as possible when the core transitions to EL0 mode
  42  * from EL1 mode, and restoring those mappings when the core transitions to EL1
  43  * mode from EL0 mode.
  44  *
  45  * At the moment, this is achieved through use of ASIDs and TCR_EL1.  TCR_EL1 is
  46  * used to map and unmap the ordinary kernel mappings, by contracting and
  47  * expanding translation zone size for TTBR1 when exiting and entering EL1,
  48  * respectively:
  49  *
  50  * Kernel EL0 Mappings: TTBR1 mappings that must remain mapped while the core is
  51  *   is in EL0.
  52  * Kernel EL1 Mappings: TTBR1 mappings that must be mapped while the core is in
  53  *   EL1.
  54  *
  55  * T1SZ_USER: T1SZ_BOOT + 1
  56  * TTBR1_EL1_BASE_BOOT: (2^64) - (2^(64 - T1SZ_BOOT)
  57  * TTBR1_EL1_BASE_USER: (2^64) - (2^(64 - T1SZ_USER)
  58  * TTBR1_EL1_MAX: (2^64) - 1
  59  *
  60  * When in EL1, we program TCR_EL1 (specifically, TCR_EL1.T1SZ) to give the
  61  * the following TTBR1 layout:
  62  *
  63  *  TTBR1_EL1_BASE_BOOT   TTBR1_EL1_BASE_USER   TTBR1_EL1_MAX
  64  * +---------------------------------------------------------+
  65  * | Kernel EL0 Mappings |        Kernel EL1 Mappings        |
  66  * +---------------------------------------------------------+
  67  *
  68  * And when in EL0, we program TCR_EL1 to give the following TTBR1 layout:
  69  *
  70  *  TTBR1_EL1_BASE_USER                         TTBR1_EL1_MAX
  71  * +---------------------------------------------------------+
  72  * |                   Kernel EL0 Mappings                   |
  73  * +---------------------------------------------------------+
  74  *
  75  * With the current implementation, both the EL0 and EL1 mappings for the kernel
  76  * use otherwise empty translation tables for mapping the exception vectors (so
  77  * that we do not need to TLB flush the exception vector address when switching
  78  * between EL0 and EL1).  The rationale here is that the TLBI would require a
  79  * DSB, and DSBs can be extremely expensive.
  80  *
  81  * Each pmap is given two ASIDs: (n & ~1) as an EL0 ASID, and (n | 1) as an EL1
  82  * ASID.  The core switches between ASIDs on EL transitions, so that the TLB
  83  * does not need to be fully invalidated on an EL transition.
  84  *
  85  * Most kernel mappings will be marked non-global in this configuration, as
  86  * global mappings would be visible to userspace unless we invalidate them on
  87  * eret.
  88  */
  89 #if XNU_MONITOR
  90 /*
  91  * Please note that because we indirect through the thread register in order to
  92  * locate the kernel, and because we unmap most of the kernel, the security
  93  * model of the PPL is undermined by __ARM_KERNEL_PROTECT__, as we rely on
  94  * kernel controlled data to direct codeflow in the exception vectors.
  95  *
  96  * If we want to ship XNU_MONITOR paired with __ARM_KERNEL_PROTECT__, we will
  97  * need to find a performant solution to this problem.
  98  */
  99 #endif
 100 #endif /* __ARM_KERNEL_PROTECT */
 101
 102 #if ARM_PARAMETERIZED_PMAP
 103 /*
 104  * ARM_PARAMETERIZED_PMAP configures the kernel to get the characteristics of
 105  * the page tables (number of levels, size of the root allocation) from the
 106  * pmap data structure, rather than treating them as compile-time constants.
 107  * This allows the pmap code to dynamically adjust how it deals with page
 108  * tables.
 109  */
 110 #endif /* ARM_PARAMETERIZED_PMAP */
 111
 112 #if __ARM_MIXED_PAGE_SIZE__
 113 /*
 114  * __ARM_MIXED_PAGE_SIZE__ configures the kernel to support page tables that do
 115  * not use the kernel page size.  This is primarily meant to support running
 116  * 4KB page processes on a 16KB page kernel.
 117  *
 118  * This only covers support in the pmap/machine dependent layers.  Any support
 119  * elsewhere in the kernel must be managed separately.
 120  */
 121 #if !ARM_PARAMETERIZED_PMAP
 122 /*
 123  * Page tables that use non-kernel page sizes require us to reprogram TCR based
 124  * on the page tables we are switching to.  This means that the parameterized
 125  * pmap support is required.
 126  */
 127 #error __ARM_MIXED_PAGE_SIZE__ requires ARM_PARAMETERIZED_PMAP
 128 #endif /* !ARM_PARAMETERIZED_PMAP */
 129 #if __ARM_KERNEL_PROTECT__
 130 /*
 131  * Because switching the page size requires updating TCR based on the pmap, and
 132  * __ARM_KERNEL_PROTECT__ relies on TCR being programmed with constants, XNU
 133  * does not currently support support configurations that use both
 134  * __ARM_KERNEL_PROTECT__ and __ARM_MIXED_PAGE_SIZE__.
 135  */
 136 #error __ARM_MIXED_PAGE_SIZE__ and __ARM_KERNEL_PROTECT__ are mutually exclusive
 137 #endif /* __ARM_KERNEL_PROTECT__ */
 138 #endif /* __ARM_MIXED_PAGE_SIZE__ */
 139
 140 /*
 141  * 64-bit Program Status Register (PSR64)
 142  *
 143  *  31      27 23  22 21 20 19      10 9       5 4   0
 144  * +-+-+-+-+-----+---+--+--+----------+-+-+-+-+-+-----+
 145  * |N|Z|C|V|00000|PAN|SS|IL|0000000000|D|A|I|F|0|  M  |
 146  * +-+-+-+-+-+---+---+--+--+----------+-+-+-+-+-+-----+
 147  *
 148  * where:
 149  *   NZCV: Comparison flags
 150  *   PAN:  Privileged Access Never
 151  *   SS:   Single step
 152  *   IL:   Illegal state
 153  *   DAIF: Interrupt masks
 154  *   M:    Mode field
 155  */
 156
 157 #define PSR64_NZCV_SHIFT 28
 158 #define PSR64_NZCV_MASK  (0xF << PSR64_NZCV_SHIFT)
 159
 160 #define PSR64_N_SHIFT    31
 161 #define PSR64_N          (1 << PSR64_N_SHIFT)
 162
 163 #define PSR64_Z_SHIFT    30
 164 #define PSR64_Z          (1 << PSR64_Z_SHIFT)
 165
 166 #define PSR64_C_SHIFT    29
 167 #define PSR64_C          (1 << PSR64_C_SHIFT)
 168
 169 #define PSR64_V_SHIFT    28
 170 #define PSR64_V          (1 << PSR64_V_SHIFT)
 171
 172 #define PSR64_PAN_SHIFT  22
 173 #define PSR64_PAN        (1 << PSR64_PAN_SHIFT)
 174
 175 #define PSR64_SS_SHIFT   21
 176 #define PSR64_SS         (1 << PSR64_SS_SHIFT)
 177
 178 #define PSR64_IL_SHIFT   20
 179 #define PSR64_IL         (1 << PSR64_IL_SHIFT)
 180
 181 /*
 182  * SSBS is bit 12 for A64 SPSR and bit 23 for A32 SPSR
 183  * I do not want to talk about it!
 184  */
 185 #define PSR64_SSBS_SHIFT_32   23
 186 #define PSR64_SSBS_SHIFT_64   12
 187 #define PSR64_SSBS_32         (1 << PSR64_SSBS_SHIFT_32)
 188 #define PSR64_SSBS_64         (1 << PSR64_SSBS_SHIFT_64)
 189
 190 /*
 191  * msr DAIF, Xn and mrs Xn, DAIF transfer into
 192  * and out of bits 9:6
 193  */
 194 #define DAIF_DEBUG_SHIFT      9
 195 #define DAIF_DEBUGF           (1 << DAIF_DEBUG_SHIFT)
 196
 197 #define DAIF_ASYNC_SHIFT      8
 198 #define DAIF_ASYNCF           (1 << DAIF_ASYNC_SHIFT)
 199
 200 #define DAIF_IRQF_SHIFT       7
 201 #define DAIF_IRQF             (1 << DAIF_IRQF_SHIFT)
 202
 203 #define DAIF_FIQF_SHIFT       6
 204 #define DAIF_FIQF             (1 << DAIF_FIQF_SHIFT)
 205
 206 #define DAIF_ALL              (DAIF_DEBUGF | DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF)
 207 #define DAIF_STANDARD_DISABLE (DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF)
 208
 209 #define SPSR_INTERRUPTS_ENABLED(x) (!(x & DAIF_FIQF))
 210
 211 #define PSR64_SSBS_U32_DEFAULT  (0)
 212 #define PSR64_SSBS_U64_DEFAULT  (0)
 213 #define PSR64_SSBS_KRN_DEFAULT  (0)
 214
 215 /*
 216  * msr DAIFSet, Xn, and msr DAIFClr, Xn transfer
 217  * from bits 3:0.
 218  */
 219 #define DAIFSC_DEBUGF           (1 << 3)
 220 #define DAIFSC_ASYNCF           (1 << 2)
 221 #define DAIFSC_IRQF             (1 << 1)
 222 #define DAIFSC_FIQF             (1 << 0)
 223 #define DAIFSC_ALL              (DAIFSC_DEBUGF | DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF)
 224 #define DAIFSC_STANDARD_DISABLE (DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF)
 225
 226 /*
 227  * ARM64_TODO: unify with ARM?
 228  */
 229 #define PSR64_CF         0x20000000 /* Carry/Borrow/Extend */
 230
 231 #define PSR64_MODE_MASK         0x1F
 232
 233 #define PSR64_USER_MASK         PSR64_NZCV_MASK
 234
 235 #define PSR64_MODE_USER32_THUMB 0x20
 236
 237 #define PSR64_MODE_RW_SHIFT     4
 238 #define PSR64_MODE_RW_64        0
 239 #define PSR64_MODE_RW_32        (0x1 << PSR64_MODE_RW_SHIFT)
 240
 241 #define PSR64_MODE_EL_SHIFT     2
 242 #define PSR64_MODE_EL_MASK      (0x3 << PSR64_MODE_EL_SHIFT)
 243 #define PSR64_MODE_EL3          (0x3 << PSR64_MODE_EL_SHIFT)
 244 #define PSR64_MODE_EL2          (0x2 << PSR64_MODE_EL_SHIFT)
 245 #define PSR64_MODE_EL1          (0x1 << PSR64_MODE_EL_SHIFT)
 246 #define PSR64_MODE_EL0          0
 247
 248 #define PSR64_MODE_SPX          0x1
 249 #define PSR64_MODE_SP0          0
 250
 251 #define PSR64_USER32_DEFAULT    (PSR64_MODE_RW_32 | PSR64_MODE_EL0 | PSR64_MODE_SP0 | PSR64_SSBS_U32_DEFAULT)
 252 #define PSR64_USER64_DEFAULT    (PSR64_MODE_RW_64 | PSR64_MODE_EL0 | PSR64_MODE_SP0 | PSR64_SSBS_U64_DEFAULT)
 253 #define PSR64_KERNEL_STANDARD   (DAIF_STANDARD_DISABLE | PSR64_MODE_RW_64 | PSR64_MODE_EL1 | PSR64_MODE_SP0 | PSR64_SSBS_KRN_DEFAULT)
 254 #if __ARM_PAN_AVAILABLE__
 255 #define PSR64_KERNEL_DEFAULT    (PSR64_KERNEL_STANDARD | PSR64_PAN)
 256 #else
 257 #define PSR64_KERNEL_DEFAULT    PSR64_KERNEL_STANDARD
 258 #endif
 259 #define PSR64_KERNEL_POISON     (PSR64_IL | PSR64_MODE_EL1)
 260
 261 #define PSR64_IS_KERNEL(x)      ((x & PSR64_MODE_EL_MASK) > PSR64_MODE_EL0)
 262 #define PSR64_IS_USER(x)        ((x & PSR64_MODE_EL_MASK) == PSR64_MODE_EL0)
 263
 264 #define PSR64_IS_USER32(x)      (PSR64_IS_USER(x) && (x & PSR64_MODE_RW_32))
 265 #define PSR64_IS_USER64(x)      (PSR64_IS_USER(x) && !(x & PSR64_MODE_RW_32))
 266
 267
 268
 269 /*
 270  * System Control Register (SCTLR)
 271  */
 272
 273 #define SCTLR_DSSBS               (1ULL << 44)
 274
 275 #define SCTLR_RESERVED     ((3ULL << 28) | (1ULL << 20))
 276 #if defined(HAS_APPLE_PAC)
 277
 278 // 31    PACIA_ENABLED AddPACIA and AuthIA functions enabled
 279 #define SCTLR_PACIA_ENABLED_SHIFT 31
 280 #define SCTLR_PACIA_ENABLED       (1ULL << SCTLR_PACIA_ENABLED_SHIFT)
 281 // 30    PACIB_ENABLED AddPACIB and AuthIB functions enabled
 282 #define SCTLR_PACIB_ENABLED       (1ULL << 30)
 283 // 29:28 RES1 11
 284 // 27    PACDA_ENABLED AddPACDA and AuthDA functions enabled
 285 #define SCTLR_PACDA_ENABLED       (1ULL << 27)
 286 // 13    PACDB_ENABLED  AddPACDB and AuthDB functions enabled
 287 #define SCTLR_PACDB_ENABLED       (1ULL << 13)
 288
 289 #define SCTLR_JOP_KEYS_ENABLED (SCTLR_PACIA_ENABLED | SCTLR_PACDA_ENABLED | SCTLR_PACDB_ENABLED)
 290 #endif /* defined(HAS_APPLE_PAC) */
 291
 292 // 26    UCI User Cache Instructions
 293 #define SCTLR_UCI_ENABLED         (1ULL << 26)
 294
 295 // 25    EE             Exception Endianness
 296 #define SCTLR_EE_BIG_ENDIAN       (1ULL << 25)
 297
 298 // 24    E0E            EL0 Endianness
 299 #define SCTLR_E0E_BIG_ENDIAN      (1ULL << 24)
 300
 301 // 23    SPAN           Set PAN
 302 #define SCTLR_PAN_UNCHANGED       (1ULL << 23)
 303
 304 // 22    EIS            Taking an exception is a context synchronization event
 305 #define SCTLR_EIS                 (1ULL << 22)
 306
 307 // 21    RES0           0
 308 // 20    RES1           1
 309
 310 // 19    WXN            Writeable implies eXecute Never
 311 #define SCTLR_WXN_ENABLED         (1ULL << 19)
 312
 313 // 18    nTWE           Not trap WFE from EL0
 314 #define SCTLR_nTWE_WFE_ENABLED    (1ULL << 18)
 315
 316 // 17    RES0           0
 317
 318 // 16    nTWI           Not trap WFI from EL0
 319 #define SCTRL_nTWI_WFI_ENABLED    (1ULL << 16)
 320
 321 // 15    UCT            User Cache Type register (CTR_EL0)
 322 #define SCTLR_UCT_ENABLED         (1ULL << 15)
 323
 324 // 14    DZE            User Data Cache Zero (DC ZVA)
 325 #define SCTLR_DZE_ENABLED         (1ULL << 14)
 326
 327 // 12    I              Instruction cache enable
 328 #define SCTLR_I_ENABLED           (1ULL << 12)
 329
 330 // 11    EOS            Exception return is a context synchronization event
 331 #define SCTLR_EOS                 (1ULL << 11)
 332
 333 // 10    RES0           0
 334
 335 // 9     UMA            User Mask Access
 336 #define SCTLR_UMA_ENABLED         (1ULL << 9)
 337
 338 // 8     SED            SETEND Disable
 339 #define SCTLR_SED_DISABLED        (1ULL << 8)
 340
 341 // 7     ITD            IT Disable
 342 #define SCTLR_ITD_DISABLED        (1ULL << 7)
 343
 344 // 6     RES0           0
 345
 346 // 5     CP15BEN        CP15 Barrier ENable
 347 #define SCTLR_CP15BEN_ENABLED     (1ULL << 5)
 348
 349 // 4     SA0            Stack Alignment check for EL0
 350 #define SCTLR_SA0_ENABLED         (1ULL << 4)
 351
 352 // 3     SA             Stack Alignment check
 353 #define SCTLR_SA_ENABLED          (1ULL << 3)
 354
 355 // 2     C              Cache enable
 356 #define SCTLR_C_ENABLED           (1ULL << 2)
 357
 358 // 1     A              Alignment check
 359 #define SCTLR_A_ENABLED           (1ULL << 1)
 360
 361 // 0     M              MMU enable
 362 #define SCTLR_M_ENABLED           (1ULL << 0)
 363
 364 #define SCTLR_CSEH_DEFAULT        (SCTLR_EIS | SCTLR_EOS)
 365 #define SCTLR_DSSBS_DEFAULT       (0)
 366
 367 #define SCTLR_EL1_DEFAULT \
 368         (SCTLR_RESERVED | SCTLR_UCI_ENABLED | SCTLR_nTWE_WFE_ENABLED | SCTLR_DZE_ENABLED | \
 369          SCTLR_I_ENABLED | SCTLR_SED_DISABLED | SCTLR_CP15BEN_ENABLED |                    \
 370          SCTLR_SA0_ENABLED | SCTLR_SA_ENABLED | SCTLR_C_ENABLED | SCTLR_M_ENABLED |        \
 371          SCTLR_CSEH_DEFAULT | SCTLR_DSSBS_DEFAULT)
 372
 373 /*
 374  * Coprocessor Access Control Register (CPACR)
 375  *
 376  *  31  28  27  22 21  20 19                 0
 377  * +---+---+------+------+--------------------+
 378  * |000|TTA|000000| FPEN |00000000000000000000|
 379  * +---+---+------+------+--------------------+
 380  *
 381  * where:
 382  *   TTA:  Trace trap
 383  *   FPEN: Floating point enable
 384  */
 385 #define CPACR_TTA_SHIFT     28
 386 #define CPACR_TTA           (1 << CPACR_TTA_SHIFT)
 387
 388 #define CPACR_FPEN_SHIFT    20
 389 #define CPACR_FPEN_EL0_TRAP (0x1 << CPACR_FPEN_SHIFT)
 390 #define CPACR_FPEN_ENABLE   (0x3 << CPACR_FPEN_SHIFT)
 391
 392 /*
 393  *  FPSR: Floating Point Status Register
 394  *
 395  *  31 30 29 28 27 26                  7   6  4   3   2   1   0
 396  * +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+
 397  * | N| Z| C| V|QC|0000000000000000000|IDC|00|IXC|UFC|OFC|DZC|IOC|
 398  * +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+
 399  */
 400
 401 #define FPSR_N_SHIFT   31
 402 #define FPSR_Z_SHIFT   30
 403 #define FPSR_C_SHIFT   29
 404 #define FPSR_V_SHIFT   28
 405 #define FPSR_QC_SHIFT  27
 406 #define FPSR_IDC_SHIFT 7
 407 #define FPSR_IXC_SHIFT 4
 408 #define FPSR_UFC_SHIFT 3
 409 #define FPSR_OFC_SHIFT 2
 410 #define FPSR_DZC_SHIFT 1
 411 #define FPSR_IOC_SHIFT 0
 412 #define FPSR_N         (1 << FPSR_N_SHIFT)
 413 #define FPSR_Z         (1 << FPSR_Z_SHIFT)
 414 #define FPSR_C         (1 << FPSR_C_SHIFT)
 415 #define FPSR_V         (1 << FPSR_V_SHIFT)
 416 #define FPSR_QC        (1 << FPSR_QC_SHIFT)
 417 #define FPSR_IDC       (1 << FPSR_IDC_SHIFT)
 418 #define FPSR_IXC       (1 << FPSR_IXC_SHIFT)
 419 #define FPSR_UFC       (1 << FPSR_UFC_SHIFT)
 420 #define FPSR_OFC       (1 << FPSR_OFC_SHIFT)
 421 #define FPSR_DZC       (1 << FPSR_DZC_SHIFT)
 422 #define FPSR_IOC       (1 << FPSR_IOC_SHIFT)
 423
 424 /*
 425  * A mask for all for all of the bits that are not RAZ for FPSR; this
 426  * is primarily for converting between a 32-bit view of NEON state
 427  * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR).
 428  */
 429 #define FPSR_MASK \
 430         (FPSR_N | FPSR_Z | FPSR_C | FPSR_V | FPSR_QC | FPSR_IDC | FPSR_IXC | \
 431          FPSR_UFC | FPSR_OFC | FPSR_DZC | FPSR_IOC)
 432
 433 /*
 434  *  FPCR: Floating Point Control Register
 435  *
 436  *  31    26  25 24 23    21     19 18  15  14 12  11  10  9   8   7      0
 437  * +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+
 438  * |00000|AHP|DN|FZ|RMODE|STRIDE| 0|LEN|IDE|00|IXE|UFE|OFE|DZE|IOE|00000000|
 439  * +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+
 440  */
 441
 442 #define FPCR_AHP_SHIFT    26
 443 #define FPCR_DN_SHIFT     25
 444 #define FPCR_FZ_SHIFT     24
 445 #define FPCR_RMODE_SHIFT  22
 446 #define FPCR_STRIDE_SHIFT 20
 447 #define FPCR_LEN_SHIFT    16
 448 #define FPCR_IDE_SHIFT    15
 449 #define FPCR_IXE_SHIFT    12
 450 #define FPCR_UFE_SHIFT    11
 451 #define FPCR_OFE_SHIFT    10
 452 #define FPCR_DZE_SHIFT    9
 453 #define FPCR_IOE_SHIFT    8
 454 #define FPCR_AHP          (1 << FPCR_AHP_SHIFT)
 455 #define FPCR_DN           (1 << FPCR_DN_SHIFT)
 456 #define FPCR_FZ           (1 << FPCR_FZ_SHIFT)
 457 #define FPCR_RMODE        (0x3 << FPCR_RMODE_SHIFT)
 458 #define FPCR_STRIDE       (0x3 << FPCR_STRIDE_SHIFT)
 459 #define FPCR_LEN          (0x7 << FPCR_LEN_SHIFT)
 460 #define FPCR_IDE          (1 << FPCR_IDE_SHIFT)
 461 #define FPCR_IXE          (1 << FPCR_IXE_SHIFT)
 462 #define FPCR_UFE          (1 << FPCR_UFE_SHIFT)
 463 #define FPCR_OFE          (1 << FPCR_OFE_SHIFT)
 464 #define FPCR_DZE          (1 << FPCR_DZE_SHIFT)
 465 #define FPCR_IOE          (1 << FPCR_IOE_SHIFT)
 466 #define FPCR_DEFAULT      (0)
 467 #define FPCR_DEFAULT_32   (FPCR_DN|FPCR_FZ)
 468
 469 /*
 470  * A mask for all for all of the bits that are not RAZ for FPCR; this
 471  * is primarily for converting between a 32-bit view of NEON state
 472  * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR).
 473  */
 474 #define FPCR_MASK \
 475         (FPCR_AHP | FPCR_DN | FPCR_FZ | FPCR_RMODE | FPCR_STRIDE | FPCR_LEN | \
 476          FPCR_IDE | FPCR_IXE | FPCR_UFE | FPCR_OFE | FPCR_DZE | FPCR_IOE)
 477
 478 /*
 479  * Translation Control Register (TCR)
 480  *
 481  * Legacy:
 482  *
 483  *  63  39   38   37 36   34 32    30 29 28 27 26 25 24   23 22 21  16    14 13 12 11 10 9   8    7   5  0
 484  * +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+
 485  * | zero |TBI1|TBI0|AS|z| IPS |z|TG1| SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ |z|TG0| SH0 |ORGN0|IRGN0|EPD0|z|T0SZ|
 486  * +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+
 487  *
 488  * Current (with 16KB granule support):
 489  *
 490  *  63  39   38   37 36   34 32    30 29 28 27 26 25 24   23 22 21  16    14 13 12 11 10 9   8    7   5  0
 491  * +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+
 492  * | zero |TBI1|TBI0|AS|z| IPS | TG1 | SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ | TG0 | SH0 |ORGN0|IRGN0|EPD0|z|T0SZ|
 493  * +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+
 494  *
 495  * TBI1:  Top Byte Ignored for TTBR1 region
 496  * TBI0:  Top Byte Ignored for TTBR0 region
 497  * AS:    ASID Size
 498  * IPS:   Physical Address Size limit
 499  * TG1:   Granule Size for TTBR1 region
 500  * SH1:   Shareability for TTBR1 region
 501  * ORGN1: Outer Cacheability for TTBR1 region
 502  * IRGN1: Inner Cacheability for TTBR1 region
 503  * EPD1:  Translation table walk disable for TTBR1
 504  * A1:    ASID selection from TTBR1 enable
 505  * T1SZ:  Virtual address size for TTBR1
 506  * TG0:   Granule Size for TTBR0 region
 507  * SH0:   Shareability for TTBR0 region
 508  * ORGN0: Outer Cacheability for TTBR0 region
 509  * IRGN0: Inner Cacheability for TTBR0 region
 510  * T0SZ:  Virtual address size for TTBR0
 511  */
 512
 513 #define TCR_T0SZ_SHIFT          0ULL
 514 #define TCR_TSZ_BITS            6ULL
 515 #define TCR_TSZ_MASK            ((1ULL << TCR_TSZ_BITS) - 1ULL)
 516
 517 #define TCR_IRGN0_SHIFT         8ULL
 518 #define TCR_IRGN0_DISABLED      (0ULL << TCR_IRGN0_SHIFT)
 519 #define TCR_IRGN0_WRITEBACK     (1ULL << TCR_IRGN0_SHIFT)
 520 #define TCR_IRGN0_WRITETHRU     (2ULL << TCR_IRGN0_SHIFT)
 521 #define TCR_IRGN0_WRITEBACKNO   (3ULL << TCR_IRGN0_SHIFT)
 522
 523 #define TCR_ORGN0_SHIFT         10ULL
 524 #define TCR_ORGN0_DISABLED      (0ULL << TCR_ORGN0_SHIFT)
 525 #define TCR_ORGN0_WRITEBACK     (1ULL << TCR_ORGN0_SHIFT)
 526 #define TCR_ORGN0_WRITETHRU     (2ULL << TCR_ORGN0_SHIFT)
 527 #define TCR_ORGN0_WRITEBACKNO   (3ULL << TCR_ORGN0_SHIFT)
 528
 529 #define TCR_SH0_SHIFT           12ULL
 530 #define TCR_SH0_NONE            (0ULL << TCR_SH0_SHIFT)
 531 #define TCR_SH0_OUTER           (2ULL << TCR_SH0_SHIFT)
 532 #define TCR_SH0_INNER           (3ULL << TCR_SH0_SHIFT)
 533
 534 #define TCR_TG0_GRANULE_SHIFT   (14ULL)
 535 #define TCR_TG0_GRANULE_BITS    (2ULL)
 536 #define TCR_TG0_GRANULE_MASK    ((1ULL << TCR_TG0_GRANULE_BITS) - 1ULL)
 537
 538 #define TCR_TG0_GRANULE_4KB     (0ULL << TCR_TG0_GRANULE_SHIFT)
 539 #define TCR_TG0_GRANULE_64KB    (1ULL << TCR_TG0_GRANULE_SHIFT)
 540 #define TCR_TG0_GRANULE_16KB    (2ULL << TCR_TG0_GRANULE_SHIFT)
 541
 542 #if __ARM_16K_PG__
 543 #define TCR_TG0_GRANULE_SIZE    (TCR_TG0_GRANULE_16KB)
 544 #else
 545 #define TCR_TG0_GRANULE_SIZE    (TCR_TG0_GRANULE_4KB)
 546 #endif
 547
 548 #define TCR_T1SZ_SHIFT          16ULL
 549
 550 #define TCR_A1_ASID1            (1ULL << 22ULL)
 551 #define TCR_EPD1_TTBR1_DISABLED (1ULL << 23ULL)
 552
 553 #define TCR_IRGN1_SHIFT          24ULL
 554 #define TCR_IRGN1_DISABLED       (0ULL << TCR_IRGN1_SHIFT)
 555 #define TCR_IRGN1_WRITEBACK      (1ULL << TCR_IRGN1_SHIFT)
 556 #define TCR_IRGN1_WRITETHRU      (2ULL << TCR_IRGN1_SHIFT)
 557 #define TCR_IRGN1_WRITEBACKNO    (3ULL << TCR_IRGN1_SHIFT)
 558
 559 #define TCR_ORGN1_SHIFT          26ULL
 560 #define TCR_ORGN1_DISABLED       (0ULL << TCR_ORGN1_SHIFT)
 561 #define TCR_ORGN1_WRITEBACK      (1ULL << TCR_ORGN1_SHIFT)
 562 #define TCR_ORGN1_WRITETHRU      (2ULL << TCR_ORGN1_SHIFT)
 563 #define TCR_ORGN1_WRITEBACKNO    (3ULL << TCR_ORGN1_SHIFT)
 564
 565 #define TCR_SH1_SHIFT            28ULL
 566 #define TCR_SH1_NONE             (0ULL << TCR_SH1_SHIFT)
 567 #define TCR_SH1_OUTER            (2ULL << TCR_SH1_SHIFT)
 568 #define TCR_SH1_INNER            (3ULL << TCR_SH1_SHIFT)
 569
 570 #define TCR_TG1_GRANULE_SHIFT    30ULL
 571
 572 #define TCR_TG1_GRANULE_16KB     (1ULL << TCR_TG1_GRANULE_SHIFT)
 573 #define TCR_TG1_GRANULE_4KB      (2ULL << TCR_TG1_GRANULE_SHIFT)
 574 #define TCR_TG1_GRANULE_64KB     (3ULL << TCR_TG1_GRANULE_SHIFT)
 575
 576 #if __ARM_16K_PG__
 577 #define TCR_TG1_GRANULE_SIZE     (TCR_TG1_GRANULE_16KB)
 578 #else
 579 #define TCR_TG1_GRANULE_SIZE     (TCR_TG1_GRANULE_4KB)
 580 #endif
 581
 582 #define TCR_IPS_SHIFT            32ULL
 583 #define TCR_IPS_BITS             3ULL
 584 #define TCR_IPS_MASK             ((1ULL << TCR_IPS_BITS) - 1ULL)
 585 #define TCR_IPS_32BITS           (0ULL << TCR_IPS_SHIFT)
 586 #define TCR_IPS_36BITS           (1ULL << TCR_IPS_SHIFT)
 587 #define TCR_IPS_40BITS           (2ULL << TCR_IPS_SHIFT)
 588 #define TCR_IPS_42BITS           (3ULL << TCR_IPS_SHIFT)
 589 #define TCR_IPS_44BITS           (4ULL << TCR_IPS_SHIFT)
 590 #define TCR_IPS_48BITS           (5ULL << TCR_IPS_SHIFT)
 591
 592 #define TCR_AS_16BIT_ASID        (1ULL << 36)
 593 #define TCR_TBI0_TOPBYTE_IGNORED (1ULL << 37)
 594 #define TCR_TBI1_TOPBYTE_IGNORED (1ULL << 38)
 595 #define TCR_TBID0_TBI_DATA_ONLY  (1ULL << 51)
 596 #define TCR_TBID1_TBI_DATA_ONLY  (1ULL << 52)
 597
 598 #if defined(HAS_APPLE_PAC)
 599 #define TCR_TBID0_ENABLE         TCR_TBID0_TBI_DATA_ONLY
 600 #else
 601 #define TCR_TBID0_ENABLE         0
 602 #endif
 603
 604 #define TCR_E0PD0_BIT            (1ULL << 55)
 605 #define TCR_E0PD1_BIT            (1ULL << 56)
 606
 607 #if defined(HAS_E0PD)
 608 #define TCR_E0PD_VALUE           (TCR_E0PD1_BIT)
 609 #else
 610 #define TCR_E0PD_VALUE           0
 611 #endif
 612
 613
 614 /*
 615  * Multiprocessor Affinity Register (MPIDR_EL1)
 616  *
 617  * +64-----------------------------31+30+29-25+24+23-16+15-8+7--0+
 618  * |000000000000000000000000000000001| U|00000|MT| Aff2|Aff1|Aff0|
 619  * +---------------------------------+--+-----+--+-----+----+----+
 620  *
 621  * where
 622  *   U:    Uniprocessor
 623  *   MT:   Multi-threading at lowest affinity level
 624  *   Aff2: "1" - PCORE, "0" - ECORE
 625  *   Aff1: Cluster ID
 626  *   Aff0: CPU ID
 627  */
 628 #define MPIDR_AFF0_SHIFT 0
 629 #define MPIDR_AFF0_WIDTH 8
 630 #define MPIDR_AFF0_MASK  (((1 << MPIDR_AFF0_WIDTH) - 1) << MPIDR_AFF0_SHIFT)
 631 #define MPIDR_AFF1_SHIFT 8
 632 #define MPIDR_AFF1_WIDTH 8
 633 #define MPIDR_AFF1_MASK  (((1 << MPIDR_AFF1_WIDTH) - 1) << MPIDR_AFF1_SHIFT)
 634 #define MPIDR_AFF2_SHIFT 16
 635 #define MPIDR_AFF2_WIDTH 8
 636 #define MPIDR_AFF2_MASK  (((1 << MPIDR_AFF2_WIDTH) - 1) << MPIDR_AFF2_SHIFT)
 637
 638 /*
 639  * TXSZ indicates the size of the range a TTBR covers.  Currently,
 640  * we support the following:
 641  *
 642  * 4KB pages, full page L1: 39 bit range.
 643  * 4KB pages, sub-page L1: 38 bit range.
 644  * 16KB pages, full page L1: 47 bit range.
 645  * 16KB pages, sub-page L1: 39 bit range.
 646  * 16KB pages, two level page tables: 36 bit range.
 647  */
 648 #if __ARM_KERNEL_PROTECT__
 649 /*
 650  * If we are configured to use __ARM_KERNEL_PROTECT__, the first half of the
 651  * address space is used for the mappings that will remain in place when in EL0.
 652  * As a result, 1 bit less of address space is available to the rest of the
 653  * the kernel.
 654  */
 655 #endif /* __ARM_KERNEL_PROTECT__ */
 656 #ifdef __ARM_16K_PG__
 657 #if __ARM64_PMAP_SUBPAGE_L1__
 658 #define T0SZ_BOOT 25ULL
 659 #else /* !__ARM64_PMAP_SUBPAGE_L1__ */
 660 #define T0SZ_BOOT 17ULL
 661 #endif /* !__ARM64_PMAP_SUBPAGE_L1__ */
 662 #else /* __ARM_16K_PG__ */
 663 #if __ARM64_PMAP_SUBPAGE_L1__
 664 #define T0SZ_BOOT 26ULL
 665 #else /* __ARM64_PMAP_SUBPAGE_L1__ */
 666 #define T0SZ_BOOT 25ULL
 667 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */
 668 #endif /* __ARM_16K_PG__ */
 669
 670 #if defined(APPLE_ARM64_ARCH_FAMILY)
 671 /* T0SZ must be the same as T1SZ */
 672 #define T1SZ_BOOT T0SZ_BOOT
 673 #else /* defined(APPLE_ARM64_ARCH_FAMILY) */
 674 #ifdef __ARM_16K_PG__
 675 #if __ARM64_PMAP_SUBPAGE_L1__
 676 #define T1SZ_BOOT 25ULL
 677 #else /* !__ARM64_PMAP_SUBPAGE_L1__ */
 678 #define T1SZ_BOOT 17ULL
 679 #endif /* !__ARM64_PMAP_SUBPAGE_L1__ */
 680 #else /* __ARM_16K_PG__ */
 681 #if __ARM64_PMAP_SUBPAGE_L1__
 682 #define T1SZ_BOOT 26ULL
 683 #else /* __ARM64_PMAP_SUBPAGE_L1__ */
 684 #define T1SZ_BOOT 25ULL
 685 #endif /*__ARM64_PMAP_SUBPAGE_L1__*/
 686 #endif /* __ARM_16K_PG__ */
 687 #endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
 688
 689 #if __ARM_42BIT_PA_SPACE__
 690 #define TCR_IPS_VALUE TCR_IPS_42BITS
 691 #else /* !__ARM_42BIT_PA_SPACE__ */
 692 #define TCR_IPS_VALUE TCR_IPS_40BITS
 693 #endif /* !__ARM_42BIT_PA_SPACE__ */
 694
 695 #define TCR_EL1_BASE \
 696         (TCR_IPS_VALUE | TCR_SH0_OUTER | TCR_ORGN0_WRITEBACK |         \
 697          TCR_IRGN0_WRITEBACK | (T0SZ_BOOT << TCR_T0SZ_SHIFT) |          \
 698          TCR_SH1_OUTER | TCR_ORGN1_WRITEBACK | \
 699          TCR_IRGN1_WRITEBACK | (TCR_TG1_GRANULE_SIZE) |                 \
 700          TCR_TBI0_TOPBYTE_IGNORED | (TCR_TBID0_ENABLE) | TCR_E0PD_VALUE)
 701
 702 #if __ARM_KERNEL_PROTECT__
 703 #define TCR_EL1_BOOT (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE))
 704 #define T1SZ_USER (T1SZ_BOOT + 1)
 705 #define TCR_EL1_USER (TCR_EL1_BASE | (T1SZ_USER << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE))
 706 #else
 707 #define TCR_EL1_BOOT (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE))
 708 #endif /* __ARM_KERNEL_PROTECT__ */
 709
 710 #define TCR_EL1_4KB  (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_4KB))
 711 #define TCR_EL1_16KB (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_16KB))
 712
 713
 714
 715
 716 /*
 717  * Monitor Debug System Control Register (MDSCR)
 718  */
 719
 720 #define MDSCR_TFO_SHIFT                 31
 721 #define MDSCR_TFO                       (1ULL << MDSCR_TFO_SHIFT)
 722 #define MDSCR_RXFULL_SHIFT              30
 723 #define MDSCR_RXFULL                    (1ULL << MDSCR_RXFULL_SHIFT)
 724 #define MDSCR_TXFULL_SHIFT              29
 725 #define MDSCR_TXFULL                    (1ULL << MDSCR_TXFULL_SHIFT)
 726 #define MDSCR_RXO_SHIFT                 27
 727 #define MDSCR_RXO                       (1ULL << MDSCR_RXO_SHIFT)
 728 #define MDSCR_TXU_SHIFT                 26
 729 #define MDSCR_TXU                       (1ULL << MDSCR_TXU_SHIFT)
 730 #define MDSCR_INTDIS_SHIFT              22
 731 #define MDSCR_INTDIS_MASK               (0x2U << MDSCR_INTDIS_SHIFT)
 732 #define MDSCR_TDA_SHIFT                 21
 733 #define MDSCR_TDA                       (1ULL << MDSCR_TDA_SHIFT)
 734 #define MDSCR_SC2_SHIFT                 19
 735 #define MDSCR_SC2                       (1ULL << MDSCR_SC2_SHIFT)
 736 #define MDSCR_MDE_SHIFT                 15
 737 #define MDSCR_MDE                       (1ULL << MDSCR_MDE_SHIFT)
 738 #define MDSCR_HDE_SHIFT                 14
 739 #define MDSCR_HDE                       (1ULL << MDSCR_HDE_SHIFT)
 740 #define MDSCR_KDE_SHIFT                 13
 741 #define MDSCR_KDE                       (1ULL << MDSCR_KDE_SHIFT)
 742 #define MDSCR_TDCC_SHIFT                12
 743 #define MDSCR_TDCC                      (1ULL << MDSCR_TDCC_SHIFT)
 744 #define MDSCR_ERR_SHIFT                 6
 745 #define MDSCR_ERR                       (1ULL << MDSCR_ERR_SHIFT)
 746 #define MDSCR_SS_SHIFT                  0
 747 #define MDSCR_SS                        (1ULL << MDSCR_SS_SHIFT)
 748
 749 /*
 750  * Translation Table Base Register (TTBR)
 751  *
 752  *  63    48 47               x x-1  0
 753  * +--------+------------------+------+
 754  * |  ASID  |   Base Address   | zero |
 755  * +--------+------------------+------+
 756  *
 757  */
 758 #define TTBR_ASID_SHIFT 48
 759 #define TTBR_ASID_MASK  0xffff000000000000
 760
 761 #define TTBR_BADDR_MASK 0x0000ffffffffffff
 762
 763 /*
 764  * Memory Attribute Indirection Register
 765  *
 766  *  63   56 55   48 47   40 39   32 31   24 23   16 15    8 7     0
 767  * +-------+-------+-------+-------+-------+-------+-------+-------+
 768  * | Attr7 | Attr6 | Attr5 | Attr4 | Attr3 | Attr2 | Attr1 | Attr0 |
 769  * +-------+-------+-------+-------+-------+-------+-------+-------+
 770  *
 771  */
 772
 773 #define MAIR_ATTR_SHIFT(x)          (8*(x))
 774
 775 /* Strongly ordered or device memory attributes */
 776 #define MAIR_OUTER_STRONGLY_ORDERED 0x0
 777 #define MAIR_OUTER_DEVICE           0x0
 778
 779 #define MAIR_INNER_STRONGLY_ORDERED 0x0
 780 #define MAIR_INNER_DEVICE           0x4
 781
 782 /* Normal memory attributes */
 783 #define MAIR_OUTER_NON_CACHEABLE    0x40
 784 #define MAIR_OUTER_WRITE_THROUGH    0x80
 785 #define MAIR_OUTER_WRITE_BACK       0xc0
 786
 787 #define MAIR_INNER_NON_CACHEABLE    0x4
 788 #define MAIR_INNER_WRITE_THROUGH    0x8
 789 #define MAIR_INNER_WRITE_BACK       0xc
 790
 791 /* Allocate policy for cacheable memory */
 792 #define MAIR_OUTER_WRITE_ALLOCATE   0x10
 793 #define MAIR_OUTER_READ_ALLOCATE    0x20
 794
 795 #define MAIR_INNER_WRITE_ALLOCATE   0x1
 796 #define MAIR_INNER_READ_ALLOCATE    0x2
 797
 798 /* Memory Atribute Encoding */
 799
 800 /*
 801  * Device memory types:
 802  * G (gathering): multiple reads/writes can be combined
 803  * R (reordering): reads or writes may reach device out of program order
 804  * E (early-acknowledge): writes may return immediately (e.g. PCIe posted writes)
 805  */
 806 #define MAIR_DISABLE                   0x00 /* Device Memory, nGnRnE (strongly ordered) */
 807 #define MAIR_POSTED                    0x04 /* Device Memory, nGnRE (strongly ordered, posted writes) */
 808 #define MAIR_POSTED_REORDERED          0x08 /* Device Memory, nGRE (reorderable, posted writes) */
 809 #define MAIR_POSTED_COMBINED_REORDERED 0x0C /* Device Memory, GRE (reorderable, gathered writes, posted writes) */
 810 #define MAIR_WRITECOMB                 0x44 /* Normal Memory, Outer Non-Cacheable, Inner Non-Cacheable */
 811 #define MAIR_WRITETHRU                 0xBB /* Normal Memory, Outer Write-through, Inner Write-through */
 812 #define MAIR_WRITEBACK                 0xFF /* Normal Memory, Outer Write-back, Inner Write-back */
 813 #define MAIR_INNERWRITEBACK            0x4F /* Normal Memory, Outer Non-Cacheable, Inner Write-back */
 814
 815
 816 /*
 817  * ARM 4-level Page Table support - 2*1024TB (2^48) of address space
 818  */
 819
 820
 821 /*
 822  *  Memory Attribute Index
 823  */
 824 #define CACHE_ATTRINDX_WRITEBACK                 0x0 /* cache enabled, buffer enabled  (normal memory) */
 825 #define CACHE_ATTRINDX_WRITECOMB                 0x1 /* no cache, buffered writes (normal memory) */
 826 #define CACHE_ATTRINDX_WRITETHRU                 0x2 /* cache enabled, buffer disabled (normal memory) */
 827 #define CACHE_ATTRINDX_DISABLE                   0x3 /* no cache, no buffer (device memory) */
 828 #define CACHE_ATTRINDX_INNERWRITEBACK            0x4 /* inner cache enabled, buffer enabled, write allocate (normal memory) */
 829 #define CACHE_ATTRINDX_POSTED                    0x5 /* no cache, no buffer, posted writes (device memory) */
 830 #define CACHE_ATTRINDX_POSTED_REORDERED          0x6 /* no cache, reorderable access, posted writes (device memory) */
 831 #define CACHE_ATTRINDX_POSTED_COMBINED_REORDERED 0x7 /* no cache, write gathering, reorderable access, posted writes (device memory) */
 832 #define CACHE_ATTRINDX_DEFAULT                   CACHE_ATTRINDX_WRITEBACK
 833
 834
 835 /*
 836  * Access protection bit values (TTEs and PTEs), stage 1
 837  *
 838  * Bit 1 controls access type (1=RO, 0=RW), bit 0 controls user (1=access, 0=no access)
 839  */
 840 #define AP_RWNA 0x0 /* priv=read-write, user=no-access */
 841 #define AP_RWRW 0x1 /* priv=read-write, user=read-write */
 842 #define AP_RONA 0x2 /* priv=read-only, user=no-access */
 843 #define AP_RORO 0x3 /* priv=read-only, user=read-only */
 844 #define AP_MASK 0x3 /* mask to find ap bits */
 845
 846 /*
 847  * Shareability attributes
 848  */
 849 #define SH_NONE         0x0 /* Non shareable  */
 850 #define SH_NONE         0x0 /* Device shareable */
 851 #define SH_DEVICE       0x2 /* Normal memory Inner non shareable - Outer non shareable */
 852 #define SH_OUTER_MEMORY 0x2 /* Normal memory Inner shareable - Outer shareable */
 853 #define SH_INNER_MEMORY 0x3 /* Normal memory Inner shareable - Outer non shareable */
 854
 855
 856 /*
 857  * ARM Page Granule
 858  */
 859 #ifdef __ARM_16K_PG__
 860 #define ARM_PGSHIFT 14
 861 #else
 862 #define ARM_PGSHIFT 12
 863 #endif
 864 #define ARM_PGBYTES (1 << ARM_PGSHIFT)
 865 #define ARM_PGMASK  (ARM_PGBYTES-1)
 866
 867 /*
 868  *  L0 Translation table
 869  *
 870  *  4KB granule size:
 871  *    Each translation table is 4KB
 872  *    512 64-bit entries of 512GB (2^39) of address space.
 873  *    Covers 256TB (2^48) of address space.
 874  *
 875  *  16KB granule size:
 876  *    Each translation table is 16KB
 877  *    2 64-bit entries of 128TB (2^47) of address space.
 878  *    Covers 256TB (2^48) of address space.
 879  */
 880
 881 /* 16K L0 */
 882 #define ARM_16K_TT_L0_SIZE       0x0000800000000000ULL /* size of area covered by a tte */
 883 #define ARM_16K_TT_L0_OFFMASK    0x00007fffffffffffULL /* offset within an L0 entry */
 884 #define ARM_16K_TT_L0_SHIFT      47                    /* page descriptor shift */
 885 #define ARM_16K_TT_L0_INDEX_MASK 0x0000800000000000ULL /* mask for getting index in L0 table from virtual address */
 886
 887 /* 4K L0 */
 888 #define ARM_4K_TT_L0_SIZE       0x0000008000000000ULL /* size of area covered by a tte */
 889 #define ARM_4K_TT_L0_OFFMASK    0x0000007fffffffffULL /* offset within an L0 entry */
 890 #define ARM_4K_TT_L0_SHIFT      39                    /* page descriptor shift */
 891 #define ARM_4K_TT_L0_INDEX_MASK 0x0000ff8000000000ULL /* mask for getting index in L0 table from virtual address */
 892
 893 /*
 894  *  L1 Translation table
 895  *
 896  *  4KB granule size:
 897  *    Each translation table is 4KB
 898  *    512 64-bit entries of 1GB (2^30) of address space.
 899  *    Covers 512GB (2^39) of address space.
 900  *
 901  *  16KB granule size:
 902  *    Each translation table is 16KB
 903  *    2048 64-bit entries of 64GB (2^36) of address space.
 904  *    Covers 128TB (2^47) of address space.
 905  */
 906
 907 /* 16K L1 */
 908 #define ARM_16K_TT_L1_SIZE       0x0000001000000000ULL /* size of area covered by a tte */
 909 #define ARM_16K_TT_L1_OFFMASK    0x0000000fffffffffULL /* offset within an L1 entry */
 910 #define ARM_16K_TT_L1_SHIFT      36                    /* page descriptor shift */
 911 #if __ARM64_PMAP_SUBPAGE_L1__ && __ARM_16K_PG__
 912 /* This config supports 512GB per TTBR. */
 913 #define ARM_16K_TT_L1_INDEX_MASK 0x0000007000000000ULL /* mask for getting index into L1 table from virtual address */
 914 #else /* __ARM64_PMAP_SUBPAGE_L1__ */
 915 #define ARM_16K_TT_L1_INDEX_MASK 0x00007ff000000000ULL /* mask for getting index into L1 table from virtual address */
 916 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */
 917
 918 /* 4K L1 */
 919 #define ARM_4K_TT_L1_SIZE       0x0000000040000000ULL /* size of area covered by a tte */
 920 #define ARM_4K_TT_L1_OFFMASK    0x000000003fffffffULL /* offset within an L1 entry */
 921 #define ARM_4K_TT_L1_SHIFT      30                    /* page descriptor shift */
 922 #if __ARM64_PMAP_SUBPAGE_L1__ && !__ARM_16K_PG__
 923 /* This config supports 256GB per TTBR. */
 924 #define ARM_4K_TT_L1_INDEX_MASK 0x0000003fc0000000ULL /* mask for getting index into L1 table from virtual address */
 925 #else /* __ARM64_PMAP_SUBPAGE_L1__ */
 926 #define ARM_4K_TT_L1_INDEX_MASK 0x0000007fc0000000ULL /* mask for getting index into L1 table from virtual address */
 927 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */
 928
 929 /* some sugar for getting pointers to page tables and entries */
 930
 931 #define L1_TABLE_INDEX(va) (((va) & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT)
 932 #define L2_TABLE_INDEX(va) (((va) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT)
 933 #define L3_TABLE_INDEX(va) (((va) & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT)
 934
 935 #define L2_TABLE_VA(tte)  ((tt_entry_t*) phystokv((*(tte)) & ARM_TTE_TABLE_MASK))
 936 #define L3_TABLE_VA(tte2) ((pt_entry_t*) phystokv((*(tte2)) & ARM_TTE_TABLE_MASK))
 937
 938 /*
 939  *  L2 Translation table
 940  *
 941  *  4KB granule size:
 942  *    Each translation table is 4KB
 943  *    512 64-bit entries of 2MB (2^21) of address space.
 944  *    Covers 1GB (2^30) of address space.
 945  *
 946  *  16KB granule size:
 947  *    Each translation table is 16KB
 948  *    2048 64-bit entries of 32MB (2^25) of address space.
 949  *    Covers 64GB (2^36) of address space.
 950  */
 951
 952 /* 16K L2 */
 953 #define ARM_16K_TT_L2_SIZE       0x0000000002000000ULL /* size of area covered by a tte */
 954 #define ARM_16K_TT_L2_OFFMASK    0x0000000001ffffffULL /* offset within an L2 entry */
 955 #define ARM_16K_TT_L2_SHIFT      25                    /* page descriptor shift */
 956 #define ARM_16K_TT_L2_INDEX_MASK 0x0000000ffe000000ULL /* mask for getting index in L2 table from virtual address */
 957
 958 /* 4K L2 */
 959 #define ARM_4K_TT_L2_SIZE       0x0000000000200000ULL /* size of area covered by a tte */
 960 #define ARM_4K_TT_L2_OFFMASK    0x00000000001fffffULL /* offset within an L2 entry */
 961 #define ARM_4K_TT_L2_SHIFT      21                    /* page descriptor shift */
 962 #define ARM_4K_TT_L2_INDEX_MASK 0x000000003fe00000ULL /* mask for getting index in L2 table from virtual address */
 963
 964 /*
 965  *  L3 Translation table
 966  *
 967  *  4KB granule size:
 968  *    Each translation table is 4KB
 969  *    512 64-bit entries of 4KB (2^12) of address space.
 970  *    Covers 2MB (2^21) of address space.
 971  *
 972  *  16KB granule size:
 973  *    Each translation table is 16KB
 974  *    2048 64-bit entries of 16KB (2^14) of address space.
 975  *    Covers 32MB (2^25) of address space.
 976  */
 977
 978 /* 16K L3 */
 979 #define ARM_16K_TT_L3_SIZE       0x0000000000004000ULL /* size of area covered by a tte */
 980 #define ARM_16K_TT_L3_OFFMASK    0x0000000000003fffULL /* offset within L3 PTE */
 981 #define ARM_16K_TT_L3_SHIFT      14                    /* page descriptor shift */
 982 #define ARM_16K_TT_L3_INDEX_MASK 0x0000000001ffc000ULL /* mask for page descriptor index */
 983
 984 /* 4K L3 */
 985 #define ARM_4K_TT_L3_SIZE       0x0000000000001000ULL /* size of area covered by a tte */
 986 #define ARM_4K_TT_L3_OFFMASK    0x0000000000000fffULL /* offset within L3 PTE */
 987 #define ARM_4K_TT_L3_SHIFT      12                    /* page descriptor shift */
 988 #define ARM_4K_TT_L3_INDEX_MASK 0x00000000001ff000ULL /* mask for page descriptor index */
 989
 990 #ifdef __ARM_16K_PG__
 991
 992 /* Native L0 defines */
 993 #define ARM_TT_L0_SIZE       ARM_16K_TT_L0_SIZE
 994 #define ARM_TT_L0_OFFMASK    ARM_16K_TT_L0_OFFMASK
 995 #define ARM_TT_L0_SHIFT      ARM_16K_TT_L0_SHIFT
 996 #define ARM_TT_L0_INDEX_MASK ARM_16K_TT_L0_INDEX_MASK
 997
 998 /* Native L1 defines */
 999 #define ARM_TT_L1_SIZE       ARM_16K_TT_L1_SIZE
1000 #define ARM_TT_L1_OFFMASK    ARM_16K_TT_L1_OFFMASK
1001 #define ARM_TT_L1_SHIFT      ARM_16K_TT_L1_SHIFT
1002 #define ARM_TT_L1_INDEX_MASK ARM_16K_TT_L1_INDEX_MASK
1003
1004 /* Native L2 defines */
1005 #define ARM_TT_L2_SIZE       ARM_16K_TT_L2_SIZE
1006 #define ARM_TT_L2_OFFMASK    ARM_16K_TT_L2_OFFMASK
1007 #define ARM_TT_L2_SHIFT      ARM_16K_TT_L2_SHIFT
1008 #define ARM_TT_L2_INDEX_MASK ARM_16K_TT_L2_INDEX_MASK
1009
1010 /* Native L3 defines */
1011 #define ARM_TT_L3_SIZE       ARM_16K_TT_L3_SIZE
1012 #define ARM_TT_L3_OFFMASK    ARM_16K_TT_L3_OFFMASK
1013 #define ARM_TT_L3_SHIFT      ARM_16K_TT_L3_SHIFT
1014 #define ARM_TT_L3_INDEX_MASK ARM_16K_TT_L3_INDEX_MASK
1015
1016 #else /* !__ARM_16K_PG__ */
1017
1018 /* Native L0 defines */
1019 #define ARM_TT_L0_SIZE       ARM_4K_TT_L0_SIZE
1020 #define ARM_TT_L0_OFFMASK    ARM_4K_TT_L0_OFFMASK
1021 #define ARM_TT_L0_SHIFT      ARM_4K_TT_L0_SHIFT
1022 #define ARM_TT_L0_INDEX_MASK ARM_4K_TT_L0_INDEX_MASK
1023
1024 /* Native L1 defines */
1025 #define ARM_TT_L1_SIZE       ARM_4K_TT_L1_SIZE
1026 #define ARM_TT_L1_OFFMASK    ARM_4K_TT_L1_OFFMASK
1027 #define ARM_TT_L1_SHIFT      ARM_4K_TT_L1_SHIFT
1028 #define ARM_TT_L1_INDEX_MASK ARM_4K_TT_L1_INDEX_MASK
1029
1030 /* Native L2 defines */
1031 #define ARM_TT_L2_SIZE       ARM_4K_TT_L2_SIZE
1032 #define ARM_TT_L2_OFFMASK    ARM_4K_TT_L2_OFFMASK
1033 #define ARM_TT_L2_SHIFT      ARM_4K_TT_L2_SHIFT
1034 #define ARM_TT_L2_INDEX_MASK ARM_4K_TT_L2_INDEX_MASK
1035
1036 /* Native L3 defines */
1037 #define ARM_TT_L3_SIZE       ARM_4K_TT_L3_SIZE
1038 #define ARM_TT_L3_OFFMASK    ARM_4K_TT_L3_OFFMASK
1039 #define ARM_TT_L3_SHIFT      ARM_4K_TT_L3_SHIFT
1040 #define ARM_TT_L3_INDEX_MASK ARM_4K_TT_L3_INDEX_MASK
1041
1042 #endif /* !__ARM_16K_PG__ */
1043
1044 /*
1045  * Convenience definitions for:
1046  *   ARM_TT_LEAF: The last level of the configured page table format.
1047  *   ARM_TT_TWIG: The second to last level of the configured page table format.
1048  *   ARM_TT_ROOT: The first level of the configured page table format.
1049  *
1050  *   My apologies to any botanists who may be reading this.
1051  */
1052 #define ARM_TT_LEAF_SIZE       ARM_TT_L3_SIZE
1053 #define ARM_TT_LEAF_OFFMASK    ARM_TT_L3_OFFMASK
1054 #define ARM_TT_LEAF_SHIFT      ARM_TT_L3_SHIFT
1055 #define ARM_TT_LEAF_INDEX_MASK ARM_TT_L3_INDEX_MASK
1056
1057 #define ARM_TT_TWIG_SIZE       ARM_TT_L2_SIZE
1058 #define ARM_TT_TWIG_OFFMASK    ARM_TT_L2_OFFMASK
1059 #define ARM_TT_TWIG_SHIFT      ARM_TT_L2_SHIFT
1060 #define ARM_TT_TWIG_INDEX_MASK ARM_TT_L2_INDEX_MASK
1061
1062 #define ARM_TT_ROOT_SIZE       ARM_TT_L1_SIZE
1063 #define ARM_TT_ROOT_OFFMASK    ARM_TT_L1_OFFMASK
1064 #define ARM_TT_ROOT_SHIFT      ARM_TT_L1_SHIFT
1065 #define ARM_TT_ROOT_INDEX_MASK ARM_TT_L1_INDEX_MASK
1066
1067 /*
1068  * 4KB granule size:
1069  *
1070  * Level 0 Translation Table Entry
1071  *
1072  *  63 62 61 60  59 58   52 51  48 47                  12 11    2 1 0
1073  * +--+-----+--+---+-------+------+----------------------+-------+-+-+
1074  * |NS|  AP |XN|PXN|ignored| zero | L1TableOutputAddress |ignored|1|V|
1075  * +--+-----+--+---+-------+------+----------------------+-------+-+-+
1076  *
1077  * Level 1 Translation Table Entry
1078  *
1079  *  63 62 61 60  59 58   52 51  48 47                  12 11    2 1 0
1080  * +--+-----+--+---+-------+------+----------------------+-------+-+-+
1081  * |NS|  AP |XN|PXN|ignored| zero | L2TableOutputAddress |ignored|1|V|
1082  * +--+-----+--+---+-------+------+----------------------+-------+-+-+
1083  *
1084  * Level 1 Translation Block Entry
1085  *
1086  *  63 59 58  55 54  53   52 51  48 47                  30 29  12 11 10 9  8 7  6  5 4     2 1 0
1087  * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
1088  * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:30] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V|
1089  * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
1090  *
1091  * Level 2 Translation Table Entry
1092  *
1093  *  63 62 61 60  59 58   52 51  48 47                  12 11    2 1 0
1094  * +--+-----+--+---+-------+------+----------------------+-------+-+-+
1095  * |NS|  AP |XN|PXN|ignored| zero | L3TableOutputAddress |ignored|1|V|
1096  * +--+-----+--+---+-------+------+----------------------+-------+-+-+
1097  *
1098  * Level 2 Translation Block Entry
1099  *
1100  *  63 59 58  55 54  53   52 51  48 47                  21 20  12 11 10 9  8 7  6  5 4     2 1 0
1101  * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
1102  * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:21] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V|
1103  * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
1104  *
1105  * 16KB granule size:
1106  *
1107  * Level 0 Translation Table Entry
1108  *
1109  *  63 62 61 60  59 58   52 51  48 47                  14 13    2 1 0
1110  * +--+-----+--+---+-------+------+----------------------+-------+-+-+
1111  * |NS|  AP |XN|PXN|ignored| zero | L1TableOutputAddress |ignored|1|V|
1112  * +--+-----+--+---+-------+------+----------------------+-------+-+-+
1113  *
1114  * Level 1 Translation Table Entry
1115  *
1116  *  63 62 61 60  59 58   52 51  48 47                  14 13    2 1 0
1117  * +--+-----+--+---+-------+------+----------------------+-------+-+-+
1118  * |NS|  AP |XN|PXN|ignored| zero | L2TableOutputAddress |ignored|1|V|
1119  * +--+-----+--+---+-------+------+----------------------+-------+-+-+
1120  *
1121  * Level 2 Translation Table Entry
1122  *
1123  *  63 62 61 60  59 58   52 51  48 47                  14 13    2 1 0
1124  * +--+-----+--+---+-------+------+----------------------+-------+-+-+
1125  * |NS|  AP |XN|PXN|ignored| zero | L3TableOutputAddress |ignored|1|V|
1126  * +--+-----+--+---+-------+------+----------------------+-------+-+-+
1127  *
1128  * Level 2 Translation Block Entry
1129  *
1130  *  63 59 58  55 54  53   52 51  48 47                  25 24  12 11 10 9  8 7  6  5 4     2 1 0
1131  * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
1132  * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:25] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V|
1133  * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
1134  *
1135  * where:
1136  *   nG:      notGlobal bit
1137  *   SH:      Shareability field
1138  *   AP:      access protection
1139  *   XN:      eXecute Never bit
1140  *   PXN:     Privilege eXecute Never bit
1141  *   NS:      Non-Secure bit
1142  *   HINT:    16 entry continuguous output hint
1143  *   AttrIdx: Memory Attribute Index
1144  */
1145
1146 #define TTE_SHIFT                   3                              /* shift width of a tte (sizeof(tte) == (1 << TTE_SHIFT)) */
1147 #ifdef __ARM_16K_PG__
1148 #define TTE_PGENTRIES               (16384 >> TTE_SHIFT)           /* number of ttes per page */
1149 #else
1150 #define TTE_PGENTRIES               (4096 >> TTE_SHIFT)            /* number of ttes per page */
1151 #endif
1152
1153 #define ARM_TTE_MAX                 (TTE_PGENTRIES)
1154
1155 #define ARM_TTE_EMPTY               0x0000000000000000ULL          /* unasigned - invalid entry */
1156 #define ARM_TTE_TYPE_FAULT          0x0000000000000000ULL          /* unasigned - invalid entry */
1157
1158 #define ARM_TTE_VALID               0x0000000000000001ULL          /* valid entry */
1159
1160 #define ARM_TTE_TYPE_MASK           0x0000000000000002ULL          /* mask for extracting the type */
1161 #define ARM_TTE_TYPE_TABLE          0x0000000000000002ULL          /* page table type */
1162 #define ARM_TTE_TYPE_BLOCK          0x0000000000000000ULL          /* block entry type */
1163 #define ARM_TTE_TYPE_L3BLOCK        0x0000000000000002ULL
1164 #define ARM_TTE_TYPE_MASK           0x0000000000000002ULL          /* mask for extracting the type */
1165
1166 #ifdef __ARM_16K_PG__
1167 /*
1168  * Note that L0/L1 block entries are disallowed for the 16KB granule size; what
1169  * are we doing with these?
1170  */
1171 #define ARM_TTE_BLOCK_SHIFT         12                             /* entry shift for a 16KB L3 TTE entry */
1172 #define ARM_TTE_BLOCK_L0_SHIFT      ARM_TT_L0_SHIFT                /* block shift for 128TB section */
1173 #define ARM_TTE_BLOCK_L1_MASK       0x0000fff000000000ULL          /* mask to extract phys address from L1 block entry */
1174 #define ARM_TTE_BLOCK_L1_SHIFT      ARM_TT_L1_SHIFT                /* block shift for 64GB section */
1175 #define ARM_TTE_BLOCK_L2_MASK       0x0000fffffe000000ULL          /* mask to extract phys address from Level 2 Translation Block entry */
1176 #define ARM_TTE_BLOCK_L2_SHIFT      ARM_TT_L2_SHIFT                /* block shift for 32MB section */
1177 #else
1178 #define ARM_TTE_BLOCK_SHIFT         12                             /* entry shift for a 4KB L3 TTE entry */
1179 #define ARM_TTE_BLOCK_L0_SHIFT      ARM_TT_L0_SHIFT                /* block shift for 2048GB section */
1180 #define ARM_TTE_BLOCK_L1_MASK       0x0000ffffc0000000ULL          /* mask to extract phys address from L1 block entry */
1181 #define ARM_TTE_BLOCK_L1_SHIFT      ARM_TT_L1_SHIFT                /* block shift for 1GB section */
1182 #define ARM_TTE_BLOCK_L2_MASK       0x0000ffffffe00000ULL          /* mask to extract phys address from Level 2 Translation Block entry */
1183 #define ARM_TTE_BLOCK_L2_SHIFT      ARM_TT_L2_SHIFT                /* block shift for 2MB section */
1184 #endif
1185
1186 #define ARM_TTE_BLOCK_APSHIFT       6
1187 #define ARM_TTE_BLOCK_AP(x)         ((x)<<ARM_TTE_BLOCK_APSHIFT)   /* access protection */
1188 #define ARM_TTE_BLOCK_APMASK        (0x3 << ARM_TTE_BLOCK_APSHIFT)
1189
1190 #define ARM_TTE_BLOCK_ATTRINDX(x)   ((x) << 2)                     /* memory attributes index */
1191 #define ARM_TTE_BLOCK_ATTRINDXMASK  (0x7ULL << 2)                  /* mask memory attributes index */
1192
1193 #define ARM_TTE_BLOCK_SH(x)         ((x) << 8)                     /* access shared */
1194 #define ARM_TTE_BLOCK_SHMASK        (0x3ULL << 8)                  /* mask access shared */
1195
1196 #define ARM_TTE_BLOCK_AF            0x0000000000000400ULL          /* value for access */
1197 #define ARM_TTE_BLOCK_AFMASK        0x0000000000000400ULL          /* access mask */
1198
1199 #define ARM_TTE_BLOCK_NG            0x0000000000000800ULL          /* value for a global mapping */
1200 #define ARM_TTE_BLOCK_NG_MASK       0x0000000000000800ULL          /* notGlobal mapping mask */
1201
1202 #define ARM_TTE_BLOCK_NS            0x0000000000000020ULL          /* value for a secure mapping */
1203 #define ARM_TTE_BLOCK_NS_MASK       0x0000000000000020ULL          /* notSecure mapping mask */
1204
1205 #define ARM_TTE_BLOCK_PNX           0x0020000000000000ULL          /* value for privilege no execute bit */
1206 #define ARM_TTE_BLOCK_PNXMASK       0x0020000000000000ULL          /* privilege no execute mask */
1207
1208 #define ARM_TTE_BLOCK_NX            0x0040000000000000ULL          /* value for no execute */
1209 #define ARM_TTE_BLOCK_NXMASK        0x0040000000000000ULL          /* no execute mask */
1210
1211 #define ARM_TTE_BLOCK_WIRED         0x0400000000000000ULL          /* value for software wired bit */
1212 #define ARM_TTE_BLOCK_WIREDMASK     0x0400000000000000ULL          /* software wired mask */
1213
1214 #define ARM_TTE_BLOCK_WRITEABLE     0x0800000000000000ULL          /* value for software writeable bit */
1215 #define ARM_TTE_BLOCK_WRITEABLEMASK 0x0800000000000000ULL          /* software writeable mask */
1216
1217 #define ARM_TTE_TABLE_MASK          0x0000fffffffff000ULL          /* mask for extracting pointer to next table (works at any level) */
1218
1219 #define ARM_TTE_TABLE_APSHIFT       61
1220 #define ARM_TTE_TABLE_AP(x)         ((x)<<TTE_BLOCK_APSHIFT)       /* access protection */
1221
1222 #define ARM_TTE_TABLE_NS            0x8000000000000020ULL          /* value for a secure mapping */
1223 #define ARM_TTE_TABLE_NS_MASK       0x8000000000000020ULL          /* notSecure mapping mask */
1224
1225 #define ARM_TTE_TABLE_XN            0x1000000000000000ULL          /* value for no execute */
1226 #define ARM_TTE_TABLE_XNMASK        0x1000000000000000ULL          /* no execute mask */
1227
1228 #define ARM_TTE_TABLE_PXN           0x0800000000000000ULL          /* value for privilege no execute bit */
1229 #define ARM_TTE_TABLE_PXNMASK       0x0800000000000000ULL          /* privilege execute mask */
1230
1231 #if __ARM_KERNEL_PROTECT__
1232 #define ARM_TTE_BOOT_BLOCK \
1233         (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) | \
1234          ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF | ARM_TTE_BLOCK_NG)
1235 #else /* __ARM_KERNEL_PROTECT__ */
1236 #define ARM_TTE_BOOT_BLOCK \
1237         (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) | \
1238          ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF)
1239 #endif /* __ARM_KERNEL_PROTECT__ */
1240
1241 #define ARM_TTE_BOOT_TABLE (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID )
1242 /*
1243  *  L3 Translation table
1244  *
1245  *  4KB granule size:
1246  *    Each translation table is 4KB
1247  *    512 64-bit entries of 4KB (2^12) of address space.
1248  *    Covers 2MB (2^21) of address space.
1249  *
1250  *  16KB granule size:
1251  *    Each translation table is 16KB
1252  *    2048 64-bit entries of 16KB (2^14) of address space.
1253  *    Covers 32MB (2^25) of address space.
1254  */
1255
1256 #ifdef __ARM_16K_PG__
1257 #define ARM_PTE_SIZE    0x0000000000004000ULL /* size of area covered by a tte */
1258 #define ARM_PTE_OFFMASK 0x0000000000003fffULL /* offset within pte area */
1259 #define ARM_PTE_SHIFT   14                    /* page descriptor shift */
1260 #define ARM_PTE_MASK    0x0000ffffffffc000ULL /* mask for output address in PTE */
1261 #else
1262 #define ARM_PTE_SIZE    0x0000000000001000ULL /* size of area covered by a tte */
1263 #define ARM_PTE_OFFMASK 0x0000000000000fffULL /* offset within pte area */
1264 #define ARM_PTE_SHIFT   12                    /* page descriptor shift */
1265 #define ARM_PTE_MASK    0x0000fffffffff000ULL /* mask for output address in PTE */
1266 #endif
1267
1268 #define ARM_TTE_PA_MASK 0x0000fffffffff000ULL
1269
1270 /*
1271  * L3 Page table entries
1272  *
1273  * The following page table entry types are possible:
1274  *
1275  * fault page entry
1276  *  63                            2  0
1277  * +------------------------------+--+
1278  * |    ignored                   |00|
1279  * +------------------------------+--+
1280  *
1281  *
1282  *  63 59 58  55 54  53   52 51  48 47                  12 11 10 9  8 7  6  5 4     2 1 0
1283  * +-----+------+--+---+----+------+----------------------+--+--+----+----+--+-------+-+-+
1284  * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:12] |nG|AF| SH | AP |NS|AttrIdx|1|V|
1285  * +-----+------+--+---+----+------+----------------------+--+--+----+----+--+-------+-+-+
1286  *
1287  * where:
1288  *   nG:      notGlobal bit
1289  *   SH:      Shareability field
1290  *   AP:      access protection
1291  *   XN:      eXecute Never bit
1292  *   PXN:     Privilege eXecute Never bit
1293  *   NS:      Non-Secure bit
1294  *   HINT:    16 entry continuguous output hint
1295  *   AttrIdx: Memory Attribute Index
1296  */
1297
1298 #define PTE_SHIFT               3                     /* shift width of a pte (sizeof(pte) == (1 << PTE_SHIFT)) */
1299 #ifdef __ARM_16K_PG__
1300 #define PTE_PGENTRIES           (16384 >> PTE_SHIFT)  /* number of ptes per page */
1301 #else
1302 #define PTE_PGENTRIES           (4096 >> PTE_SHIFT)   /* number of ptes per page */
1303 #endif
1304
1305 #define ARM_PTE_EMPTY           0x0000000000000000ULL /* unassigned - invalid entry */
1306
1307 /* markers for (invalid) PTE for a page sent to compressor */
1308 #define ARM_PTE_COMPRESSED      0x8000000000000000ULL /* compressed... */
1309 #define ARM_PTE_COMPRESSED_ALT  0x4000000000000000ULL /* ... and was "alt_acct" */
1310 #define ARM_PTE_COMPRESSED_MASK 0xC000000000000000ULL
1311
1312 #define ARM_PTE_IS_COMPRESSED(x, p) \
1313         ((((x) & 0x3) == 0) && /* PTE is not valid... */                      \
1314          ((x) & ARM_PTE_COMPRESSED) && /* ...has "compressed" marker" */      \
1315          ((!((x) & ~ARM_PTE_COMPRESSED_MASK)) || /* ...no other bits */       \
1316          (panic("compressed PTE %p 0x%llx has extra bits 0x%llx: corrupted?", \
1317                 (p), (x), (x) & ~ARM_PTE_COMPRESSED_MASK), FALSE)))
1318
1319 #define ARM_PTE_TYPE               0x0000000000000003ULL /* valid L3 entry: includes bit #1 (counterintuitively) */
1320 #define ARM_PTE_TYPE_VALID         0x0000000000000003ULL /* valid L3 entry: includes bit #1 (counterintuitively) */
1321 #define ARM_PTE_TYPE_FAULT         0x0000000000000000ULL /* invalid L3 entry */
1322 #define ARM_PTE_TYPE_MASK          0x0000000000000002ULL /* mask to get pte type */
1323
1324 /* This mask works for both 16K and 4K pages because bits 12-13 will be zero in 16K pages */
1325 #define ARM_PTE_PAGE_MASK          0x0000FFFFFFFFF000ULL /* output address mask for page */
1326 #define ARM_PTE_PAGE_SHIFT         12                    /* page shift for the output address in the entry */
1327
1328 #define ARM_PTE_AP(x)              ((x) << 6)            /* access protections */
1329 #define ARM_PTE_APMASK             (0x3ULL << 6)         /* mask access protections */
1330 #define ARM_PTE_EXTRACT_AP(x)      (((x) >> 6) & 0x3ULL) /* extract access protections from PTE */
1331
1332 #define ARM_PTE_ATTRINDX(x)        ((x) << 2)            /* memory attributes index */
1333 #define ARM_PTE_ATTRINDXMASK       (0x7ULL << 2)         /* mask memory attributes index */
1334
1335 #define ARM_PTE_SH(x)              ((x) << 8)            /* access shared */
1336 #define ARM_PTE_SHMASK             (0x3ULL << 8)         /* mask access shared */
1337
1338 #define ARM_PTE_AF                 0x0000000000000400ULL /* value for access */
1339 #define ARM_PTE_AFMASK             0x0000000000000400ULL /* access mask */
1340
1341 #define ARM_PTE_NG                 0x0000000000000800ULL /* value for a global mapping */
1342 #define ARM_PTE_NG_MASK            0x0000000000000800ULL /* notGlobal mapping mask */
1343
1344 #define ARM_PTE_NS                 0x0000000000000020ULL /* value for a secure mapping */
1345 #define ARM_PTE_NS_MASK            0x0000000000000020ULL /* notSecure mapping mask */
1346
1347 #define ARM_PTE_HINT               0x0010000000000000ULL /* value for contiguous entries hint */
1348 #define ARM_PTE_HINT_MASK          0x0010000000000000ULL /* mask for contiguous entries hint */
1349
1350 #if __ARM_16K_PG__
1351 #define ARM_PTE_HINT_ENTRIES       128ULL                /* number of entries the hint covers */
1352 #define ARM_PTE_HINT_ENTRIES_SHIFT 7ULL                  /* shift to construct the number of entries */
1353 #define ARM_PTE_HINT_ADDR_MASK     0x0000FFFFFFE00000ULL /* mask to extract the starting hint address */
1354 #define ARM_PTE_HINT_ADDR_SHIFT    21                    /* shift for the hint address */
1355 #define ARM_KVA_HINT_ADDR_MASK     0xFFFFFFFFFFE00000ULL /* mask to extract the starting hint address */
1356 #else
1357 #define ARM_PTE_HINT_ENTRIES       16ULL                 /* number of entries the hint covers */
1358 #define ARM_PTE_HINT_ENTRIES_SHIFT 4ULL                  /* shift to construct the number of entries */
1359 #define ARM_PTE_HINT_ADDR_MASK     0x0000FFFFFFFF0000ULL /* mask to extract the starting hint address */
1360 #define ARM_PTE_HINT_ADDR_SHIFT    16                    /* shift for the hint address */
1361 #define ARM_KVA_HINT_ADDR_MASK     0xFFFFFFFFFFFF0000ULL /* mask to extract the starting hint address */
1362 #endif
1363
1364 #define ARM_PTE_PNX                0x0020000000000000ULL /* value for privilege no execute bit */
1365 #define ARM_PTE_PNXMASK            0x0020000000000000ULL /* privilege no execute mask */
1366
1367 #define ARM_PTE_NX                 0x0040000000000000ULL /* value for no execute bit */
1368 #define ARM_PTE_NXMASK             0x0040000000000000ULL /* no execute mask */
1369
1370 #define ARM_PTE_WIRED              0x0400000000000000ULL /* value for software wired bit */
1371 #define ARM_PTE_WIRED_MASK         0x0400000000000000ULL /* software wired mask */
1372
1373 #define ARM_PTE_WRITEABLE          0x0800000000000000ULL /* value for software writeable bit */
1374 #define ARM_PTE_WRITEABLE_MASK     0x0800000000000000ULL /* software writeable mask */
1375
1376 #if CONFIG_PGTRACE
1377 #define ARM_PTE_PGTRACE            0x0200000000000000ULL /* value for software trace bit */
1378 #define ARM_PTE_PGTRACE_MASK       0x0200000000000000ULL /* software trace mask */
1379 #endif
1380
1381 #define ARM_PTE_BOOT_PAGE_BASE \
1382         (ARM_PTE_TYPE_VALID | ARM_PTE_SH(SH_OUTER_MEMORY) |       \
1383          ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_PTE_AF)
1384
1385 #if __ARM_KERNEL_PROTECT__
1386 #define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE | ARM_PTE_NG)
1387 #else /* __ARM_KERNEL_PROTECT__ */
1388 #define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE)
1389 #endif /* __ARM_KERNEL_PROTECT__ */
1390
1391 /*
1392  * TLBI appers to only deal in 4KB page addresses, so give
1393  * it an explicit shift of 12.
1394  */
1395 #define TLBI_ADDR_SHIFT (0)
1396 #define TLBI_ADDR_SIZE  (44)
1397 #define TLBI_ADDR_MASK  ((1ULL << TLBI_ADDR_SIZE) - 1)
1398 #define TLBI_ASID_SHIFT (48)
1399 #define TLBI_ASID_SIZE  (16)
1400 #define TLBI_ASID_MASK  (((1ULL << TLBI_ASID_SIZE) - 1))
1401
1402 #define RTLBI_ADDR_SIZE (37)
1403 #define RTLBI_ADDR_MASK ((1ULL << RTLBI_ADDR_SIZE) - 1)
1404 #define RTLBI_ADDR_SHIFT ARM_TT_L3_SHIFT
1405 #define RTLBI_TG(_page_shift_) ((uint64_t)((((_page_shift_) - 12) >> 1) + 1) << 46)
1406 #define RTLBI_SCALE_SHIFT (44)
1407 #define RTLBI_NUM_SHIFT (39)
1408
1409 /*
1410  * Exception Syndrome Register
1411  *
1412  *  31  26 25 24               0
1413  * +------+--+------------------+
1414  * |  EC  |IL|       ISS        |
1415  * +------+--+------------------+
1416  *
1417  * EC  - Exception Class
1418  * IL  - Instruction Length
1419  * ISS - Instruction Specific Syndrome
1420  *
1421  * Note: The ISS can have many forms. These are defined separately below.
1422  */
1423
1424 #define ESR_EC_SHIFT           26
1425 #define ESR_EC_MASK            (0x3FULL << ESR_EC_SHIFT)
1426 #define ESR_EC(x)              ((x & ESR_EC_MASK) >> ESR_EC_SHIFT)
1427
1428 #define ESR_IL_SHIFT           25
1429 #define ESR_IL                 (1 << ESR_IL_SHIFT)
1430
1431 #define ESR_INSTR_IS_2BYTES(x) (!(x & ESR_IL))
1432
1433 #define ESR_ISS_MASK           0x01FFFFFF
1434 #define ESR_ISS(x)             (x & ESR_ISS_MASK)
1435
1436 #ifdef __ASSEMBLER__
1437 /* Define only the classes we need to test in the exception vectors. */
1438 #define ESR_EC_IABORT_EL1      0x21
1439 #define ESR_EC_DABORT_EL1      0x25
1440 #define ESR_EC_SP_ALIGN        0x26
1441 #else
1442 typedef enum {
1443         ESR_EC_UNCATEGORIZED       = 0x00,
1444         ESR_EC_WFI_WFE             = 0x01,
1445         ESR_EC_MCR_MRC_CP15_TRAP   = 0x03,
1446         ESR_EC_MCRR_MRRC_CP15_TRAP = 0x04,
1447         ESR_EC_MCR_MRC_CP14_TRAP   = 0x05,
1448         ESR_EC_LDC_STC_CP14_TRAP   = 0x06,
1449         ESR_EC_TRAP_SIMD_FP        = 0x07,
1450         ESR_EC_PTRAUTH_INSTR_TRAP  = 0x09,
1451         ESR_EC_MCRR_MRRC_CP14_TRAP = 0x0c,
1452         ESR_EC_ILLEGAL_INSTR_SET   = 0x0e,
1453         ESR_EC_SVC_32              = 0x11,
1454         ESR_EC_SVC_64              = 0x15,
1455         ESR_EC_MSR_TRAP            = 0x18,
1456         ESR_EC_IABORT_EL0          = 0x20,
1457         ESR_EC_IABORT_EL1          = 0x21,
1458         ESR_EC_PC_ALIGN            = 0x22,
1459         ESR_EC_DABORT_EL0          = 0x24,
1460         ESR_EC_DABORT_EL1          = 0x25,
1461         ESR_EC_SP_ALIGN            = 0x26,
1462         ESR_EC_FLOATING_POINT_32   = 0x28,
1463         ESR_EC_FLOATING_POINT_64   = 0x2C,
1464         ESR_EC_BKPT_REG_MATCH_EL0  = 0x30, // Breakpoint Debug event taken to the EL from a lower EL.
1465         ESR_EC_BKPT_REG_MATCH_EL1  = 0x31, // Breakpoint Debug event taken to the EL from the EL.
1466         ESR_EC_SW_STEP_DEBUG_EL0   = 0x32, // Software Step Debug event taken to the EL from a lower EL.
1467         ESR_EC_SW_STEP_DEBUG_EL1   = 0x33, // Software Step Debug event taken to the EL from the EL.
1468         ESR_EC_WATCHPT_MATCH_EL0   = 0x34, // Watchpoint Debug event taken to the EL from a lower EL.
1469         ESR_EC_WATCHPT_MATCH_EL1   = 0x35, // Watchpoint Debug event taken to the EL from the EL.
1470         ESR_EC_BKPT_AARCH32        = 0x38,
1471         ESR_EC_BRK_AARCH64         = 0x3C,
1472 } esr_exception_class_t;
1473
1474 typedef enum {
1475         FSC_TRANSLATION_FAULT_L0   = 0x04,
1476         FSC_TRANSLATION_FAULT_L1   = 0x05,
1477         FSC_TRANSLATION_FAULT_L2   = 0x06,
1478         FSC_TRANSLATION_FAULT_L3   = 0x07,
1479         FSC_ACCESS_FLAG_FAULT_L1   = 0x09,
1480         FSC_ACCESS_FLAG_FAULT_L2   = 0x0A,
1481         FSC_ACCESS_FLAG_FAULT_L3   = 0x0B,
1482         FSC_PERMISSION_FAULT_L1    = 0x0D,
1483         FSC_PERMISSION_FAULT_L2    = 0x0E,
1484         FSC_PERMISSION_FAULT_L3    = 0x0F,
1485         FSC_SYNC_EXT_ABORT         = 0x10,
1486         FSC_ASYNC_EXT_ABORT        = 0x11,
1487         FSC_SYNC_EXT_ABORT_TT_L1   = 0x15,
1488         FSC_SYNC_EXT_ABORT_TT_L2   = 0x16,
1489         FSC_SYNC_EXT_ABORT_TT_L3   = 0x17,
1490         FSC_SYNC_PARITY            = 0x18,
1491         FSC_ASYNC_PARITY           = 0x19,
1492         FSC_SYNC_PARITY_TT_L1      = 0x1D,
1493         FSC_SYNC_PARITY_TT_L2      = 0x1E,
1494         FSC_SYNC_PARITY_TT_L3      = 0x1F,
1495         FSC_ALIGNMENT_FAULT        = 0x21,
1496         FSC_DEBUG_FAULT            = 0x22,
1497 } fault_status_t;
1498 #endif /* ASSEMBLER */
1499
1500 /*
1501  * Software step debug event ISS (EL1)
1502  *  24  23                6  5    0
1503  * +---+-----------------+--+------+
1504  * |ISV|00000000000000000|EX| IFSC |
1505  * +---+-----------------+--+------+
1506  *
1507  * where:
1508  *   ISV:  Instruction syndrome valid
1509  *   EX:   Exclusive access
1510  *   IFSC: Instruction Fault Status Code
1511  */
1512
1513 #define ISS_SSDE_ISV_SHIFT 24
1514 #define ISS_SSDE_ISV       (0x1 << ISS_SSDE_ISV_SHIFT)
1515
1516 #define ISS_SSDE_EX_SHIFT  6
1517 #define ISS_SSDE_EX        (0x1 << ISS_SSDE_EX_SHIFT)
1518
1519 #define ISS_SSDE_FSC_MASK  0x3F
1520 #define ISS_SSDE_FSC(x)    (x & ISS_SSDE_FSC_MASK)
1521
1522 /*
1523  * Instruction Abort ISS (EL1)
1524  *  24           10 9      5    0
1525  * +---------------+--+---+------+
1526  * |000000000000000|EA|000| IFSC |
1527  * +---------------+--+---+------+
1528  *
1529  * where:
1530  *   EA:   External Abort type
1531  *   IFSC: Instruction Fault Status Code
1532  */
1533
1534 #define ISS_IA_EA_SHIFT 9
1535 #define ISS_IA_EA       (0x1 << ISS_IA_EA_SHIFT)
1536
1537 #define ISS_IA_FSC_MASK 0x3F
1538 #define ISS_IA_FSC(x)   (x & ISS_IA_FSC_MASK)
1539
1540
1541 /*
1542  * Data Abort ISS (EL1)
1543  *
1544  *  24              9  8  7  6  5  0
1545  * +---------------+--+--+-+---+----+
1546  * |000000000000000|EA|CM|S1PTW|WnR|DFSC|
1547  * +---------------+--+--+-+---+----+
1548  *
1549  * where:
1550  *   EA:    External Abort type
1551  *   CM:    Cache Maintenance operation
1552  *   WnR:   Write not Read
1553  *   S1PTW: Stage 2 exception on Stage 1 page table walk
1554  *   DFSC:  Data Fault Status Code
1555  */
1556 #define ISS_DA_EA_SHIFT  9
1557 #define ISS_DA_EA        (0x1 << ISS_DA_EA_SHIFT)
1558
1559 #define ISS_DA_CM_SHIFT  8
1560 #define ISS_DA_CM        (0x1 << ISS_DA_CM_SHIFT)
1561
1562 #define ISS_DA_WNR_SHIFT 6
1563 #define ISS_DA_WNR       (0x1 << ISS_DA_WNR_SHIFT)
1564
1565 #define ISS_DA_S1PTW_SHIFT 7
1566 #define ISS_DA_S1PTW     (0x1 << ISS_DA_S1PTW_SHIFT)
1567
1568 #define ISS_DA_FSC_MASK  0x3F
1569 #define ISS_DA_FSC(x)    (x & ISS_DA_FSC_MASK)
1570
1571 /*
1572  * Floating Point Exception ISS (EL1)
1573  *
1574  * 24  23 22            8  7      4   3   2   1   0
1575  * +-+---+---------------+---+--+---+---+---+---+---+
1576  * |0|TFV|000000000000000|IDF|00|IXF|UFF|OFF|DZF|IOF|
1577  * +-+---+---------------+---+--+---+---+---+---+---+
1578  *
1579  * where:
1580  *   TFV: Trapped Fault Valid
1581  *   IDF: Input Denormal Exception
1582  *   IXF: Input Inexact Exception
1583  *   UFF: Underflow Exception
1584  *   OFF: Overflow Exception
1585  *   DZF: Divide by Zero Exception
1586  *   IOF: Invalid Operation Exception
1587  */
1588 #define ISS_FP_TFV_SHIFT 23
1589 #define ISS_FP_TFV       (0x1 << ISS_FP_TFV_SHIFT)
1590
1591 #define ISS_FP_IDF_SHIFT 7
1592 #define ISS_FP_IDF       (0x1 << ISS_FP_IDF_SHIFT)
1593
1594 #define ISS_FP_IXF_SHIFT 4
1595 #define ISS_FP_IXF       (0x1 << ISS_FP_IXF_SHIFT)
1596
1597 #define ISS_FP_UFF_SHIFT 3
1598 #define ISS_FP_UFF       (0x1 << ISS_FP_UFF_SHIFT)
1599
1600 #define ISS_FP_OFF_SHIFT 2
1601 #define ISS_FP_OFF       (0x1 << ISS_FP_OFF_SHIFT)
1602
1603 #define ISS_FP_DZF_SHIFT 1
1604 #define ISS_FP_DZF       (0x1 << ISS_FP_DZF_SHIFT)
1605
1606 #define ISS_FP_IOF_SHIFT 0
1607 #define ISS_FP_IOF       (0x1 << ISS_FP_IOF_SHIFT)
1608
1609 /*
1610  * Breakpoint Exception ISS (EL1)
1611  *  24     16          0
1612  * +---------+---------+
1613  * |000000000| Comment |
1614  * +---------+---------+
1615  *
1616  * where:
1617  *   Comment: Instruction Comment Field Value
1618  */
1619 #define ISS_BRK_COMMENT_MASK    0xFFFF
1620 #define ISS_BRK_COMMENT(x)      (x & ISS_BRK_COMMENT_MASK)
1621
1622
1623
1624
1625
1626 /*
1627  * Physical Address Register (EL1)
1628  */
1629 #define PAR_F_SHIFT 0
1630 #define PAR_F       (0x1 << PAR_F_SHIFT)
1631
1632 #define PLATFORM_SYSCALL_TRAP_NO 0x80000000
1633
1634 #define ARM64_SYSCALL_CODE_REG_NUM (16)
1635
1636 #define ARM64_CLINE_SHIFT 6
1637
1638 #if defined(APPLE_ARM64_ARCH_FAMILY)
1639 #define L2CERRSTS_DATSBEESV (1ULL << 2) /* L2C data single bit ECC error */
1640 #define L2CERRSTS_DATDBEESV (1ULL << 4) /* L2C data double bit ECC error */
1641 #endif
1642
1643 /*
1644  * Timer definitions.
1645  */
1646 #define CNTKCTL_EL1_PL0PTEN      (0x1 << 9)           /* 1: EL0 access to physical timer regs permitted */
1647 #define CNTKCTL_EL1_PL0VTEN      (0x1 << 8)           /* 1: EL0 access to virtual timer regs permitted */
1648 #define CNTKCTL_EL1_EVENTI_MASK  (0x000000f0)         /* Mask for bits describing which bit to use for triggering event stream */
1649 #define CNTKCTL_EL1_EVENTI_SHIFT (0x4)                /* Shift for same */
1650 #define CNTKCTL_EL1_EVENTDIR     (0x1 << 3)           /* 1: one-to-zero transition of specified bit causes event */
1651 #define CNTKCTL_EL1_EVNTEN       (0x1 << 2)           /* 1: enable event stream */
1652 #define CNTKCTL_EL1_PL0VCTEN     (0x1 << 1)           /* 1: EL0 access to virtual timebase + frequency reg enabled */
1653 #define CNTKCTL_EL1_PL0PCTEN     (0x1 << 0)           /* 1: EL0 access to physical timebase + frequency reg enabled */
1654
1655 #define CNTV_CTL_EL0_ISTATUS     (0x1 << 2)           /* (read only): whether interrupt asserted */
1656 #define CNTV_CTL_EL0_IMASKED     (0x1 << 1)           /* 1: interrupt masked */
1657 #define CNTV_CTL_EL0_ENABLE      (0x1 << 0)           /* 1: virtual timer enabled */
1658
1659 #define CNTP_CTL_EL0_ISTATUS     CNTV_CTL_EL0_ISTATUS
1660 #define CNTP_CTL_EL0_IMASKED     CNTV_CTL_EL0_IMASKED
1661 #define CNTP_CTL_EL0_ENABLE      CNTV_CTL_EL0_ENABLE
1662
1663 /*
1664  * At present all other uses of ARM_DBG_* are shared bit compatibly with the 32bit definitons.
1665  * (cf. osfmk/arm/proc_reg.h)
1666  */
1667 #define ARM_DBG_VR_ADDRESS_MASK64 0xFFFFFFFFFFFFFFFCull /* BVR & WVR */
1668
1669 #define MIDR_EL1_REV_SHIFT  0
1670 #define MIDR_EL1_REV_MASK   (0xf << MIDR_EL1_REV_SHIFT)
1671 #define MIDR_EL1_PNUM_SHIFT 4
1672 #define MIDR_EL1_PNUM_MASK  (0xfff << MIDR_EL1_PNUM_SHIFT)
1673 #define MIDR_EL1_ARCH_SHIFT 16
1674 #define MIDR_EL1_ARCH_MASK  (0xf << MIDR_EL1_ARCH_SHIFT)
1675 #define MIDR_EL1_VAR_SHIFT  20
1676 #define MIDR_EL1_VAR_MASK   (0xf << MIDR_EL1_VAR_SHIFT)
1677 #define MIDR_EL1_IMP_SHIFT  24
1678 #define MIDR_EL1_IMP_MASK   (0xff << MIDR_EL1_IMP_SHIFT)
1679
1680 #define MIDR_FIJI             (0x002 << MIDR_EL1_PNUM_SHIFT)
1681 #define MIDR_CAPRI            (0x003 << MIDR_EL1_PNUM_SHIFT)
1682 #define MIDR_MAUI             (0x004 << MIDR_EL1_PNUM_SHIFT)
1683 #define MIDR_ELBA             (0x005 << MIDR_EL1_PNUM_SHIFT)
1684 #define MIDR_CAYMAN           (0x006 << MIDR_EL1_PNUM_SHIFT)
1685 #define MIDR_MYST             (0x007 << MIDR_EL1_PNUM_SHIFT)
1686 #define MIDR_SKYE_MONSOON     (0x008 << MIDR_EL1_PNUM_SHIFT)
1687 #define MIDR_SKYE_MISTRAL     (0x009 << MIDR_EL1_PNUM_SHIFT)
1688 #define MIDR_CYPRUS_VORTEX    (0x00B << MIDR_EL1_PNUM_SHIFT)
1689 #define MIDR_CYPRUS_TEMPEST   (0x00C << MIDR_EL1_PNUM_SHIFT)
1690 #define MIDR_M9               (0x00F << MIDR_EL1_PNUM_SHIFT)
1691 #define MIDR_ARUBA_VORTEX     (0x010 << MIDR_EL1_PNUM_SHIFT)
1692 #define MIDR_ARUBA_TEMPEST    (0x011 << MIDR_EL1_PNUM_SHIFT)
1693
1694 #ifdef APPLELIGHTNING
1695 #define MIDR_CEBU_LIGHTNING   (0x012 << MIDR_EL1_PNUM_SHIFT)
1696 #define MIDR_CEBU_THUNDER     (0x013 << MIDR_EL1_PNUM_SHIFT)
1697 #define MIDR_TURKS            (0x026 << MIDR_EL1_PNUM_SHIFT)
1698 #endif
1699
1700
1701
1702 /*
1703  * Apple-ISA-Extensions ID Register.
1704  */
1705 #define AIDR_MUL53            (1 << 0)
1706 #define AIDR_WKDM             (1 << 1)
1707 #define AIDR_ARCHRETENTION    (1 << 2)
1708
1709
1710 /*
1711  * CoreSight debug registers
1712  */
1713 #define CORESIGHT_ED  0
1714 #define CORESIGHT_CTI 1
1715 #define CORESIGHT_PMU 2
1716 #define CORESIGHT_UTT 3 /* Not truly a coresight thing, but at a fixed convenient location right after the coresight region */
1717
1718 #define CORESIGHT_OFFSET(x) ((x) * 0x10000)
1719 #define CORESIGHT_REGIONS   4
1720 #define CORESIGHT_SIZE      0x1000
1721
1722 #if __APRR_SUPPORTED__
1723 /*
1724  * APRR_EL0/APRR_EL1
1725  *
1726  *  63                 0
1727  * +--------------------+
1728  * | Attr[15:0]RWX[3:0] |
1729  * +--------------------+
1730  *
1731  * These registers consist of 16 4-bit fields.
1732  *
1733  * The attribute index consists of the access protection
1734  * and execution protections on a mapping.  The index
1735  * for a given mapping type is constructed as follows.
1736  *
1737  * Attribute Index
1738  *
1739  *     3       2      1     0
1740  * +-------+-------+-----+----+
1741  * | AP[1] | AP[0] | PXN | XN |
1742  * +-------+-------+-----+----+
1743  *
1744  * The attribute for a given index determines what
1745  * protections are disabled for that mappings type
1746  * (protections beyond the scope of the standard ARM
1747  * protections for a mapping cannot be granted via
1748  * APRR).
1749  *
1750  * Attribute
1751  *
1752  *       3      2   1   0
1753  * +----------+---+---+---+
1754  * | Reserved | R | W | X |
1755  * +----------+---+---+---+
1756  *
1757  * Where:
1758  *   R: Read is allowed.
1759  *   W: Write is allowed.
1760  *   X: Execute is allowed.
1761  */
1762
1763 #define APRR_IDX_XN  (1ULL)
1764 #define APRR_IDX_PXN (2ULL)
1765
1766
1767 #define APRR_IDX_XN_SHIFT (0ULL)
1768 #define APRR_IDX_PXN_SHIFT  (1ULL)
1769 #define APRR_IDX_APSHIFT   (2ULL)
1770
1771 #endif /* __APRR_SUPPORTED__ */
1772
1773
1774 #if __APRR_SUPPORTED__
1775
1776 #define APRR_ATTR_X (1ULL)
1777 #define APRR_ATTR_W (2ULL)
1778 #define APRR_ATTR_R (4ULL)
1779
1780 #define APRR_ATTR_WX  (APRR_ATTR_W | APRR_ATTR_X)
1781 #define APRR_ATTR_RX  (APRR_ATTR_R | APRR_ATTR_X)
1782 #define APRR_ATTR_RWX (APRR_ATTR_R | APRR_ATTR_W | APRR_ATTR_X)
1783
1784 #define APRR_ATTR_NONE (0ULL)
1785 #define APRR_ATTR_MASK (APRR_ATTR_RWX)
1786
1787 #define APRR_RESERVED_MASK (0x8888888888888888ULL)
1788 #endif /* __APRR_SUPPORTED__ */
1789
1790 #if __APRR_SUPPORTED__
1791 #define XPRR_FIRM_RX_PERM  (0ULL)
1792 #define XPRR_PPL_RW_PERM   (1ULL)
1793 #define XPRR_FIRM_RO_PERM  (2ULL)
1794 #define XPRR_KERN_RW_PERM  (3ULL)
1795 #define XPRR_FIRM_RW_PERM  (4ULL)
1796 #define XPRR_USER_JIT_PERM (5ULL)
1797 #define XPRR_KERN0_RW_PERM (6ULL)
1798 #define XPRR_USER_RW_PERM  (7ULL)
1799 #define XPRR_PPL_RX_PERM   (8ULL)
1800 #define XPRR_USER_XO_PERM  (9ULL)
1801 #define XPRR_KERN_RX_PERM  (10ULL)
1802 #define XPRR_KERN_RO_PERM  (11ULL)
1803 #define XPRR_KERN0_RX_PERM (12ULL)
1804 #define XPRR_USER_RX_PERM  (13ULL)
1805 #define XPRR_KERN0_RO_PERM (14ULL)
1806 #define XPRR_USER_RO_PERM  (15ULL)
1807 #define XPRR_MAX_PERM      (15ULL)
1808
1809 #define XPRR_VERSION_NONE    (0ULL)
1810 #define XPRR_VERSION_APRR    (1ULL)
1811
1812
1813 #endif /* __APRR_SUPPORTED__*/
1814
1815 #if __APRR_SUPPORTED__
1816 /* Indices for attributes, named based on how we intend to use them. */
1817 #define APRR_FIRM_RX_INDEX  (0ULL)  /* AP_RWNA, PX, X */
1818 #define APRR_FIRM_RO_INDEX  (1ULL)  /* AP_RWNA, PX, XN */
1819 #define APRR_PPL_RW_INDEX   (2ULL)  /* AP_RWNA, PXN, X */
1820 #define APRR_KERN_RW_INDEX  (3ULL)  /* AP_RWNA, PXN, XN */
1821 #define APRR_FIRM_RW_INDEX  (4ULL)  /* AP_RWRW, PX, X */
1822 #define APRR_KERN0_RW_INDEX (5ULL)  /* AP_RWRW, PX, XN */
1823 #define APRR_USER_JIT_INDEX (6ULL)  /* AP_RWRW, PXN, X */
1824 #define APRR_USER_RW_INDEX  (7ULL)  /* AP_RWRW, PXN, XN */
1825 #define APRR_PPL_RX_INDEX   (8ULL)  /* AP_RONA, PX, X */
1826 #define APRR_KERN_RX_INDEX  (9ULL)  /* AP_RONA, PX, XN */
1827 #define APRR_USER_XO_INDEX  (10ULL) /* AP_RONA, PXN, X */
1828 #define APRR_KERN_RO_INDEX  (11ULL) /* AP_RONA, PXN, XN */
1829 #define APRR_KERN0_RX_INDEX (12ULL) /* AP_RORO, PX, X */
1830 #define APRR_KERN0_RO_INDEX (13ULL) /* AP_RORO, PX, XN */
1831 #define APRR_USER_RX_INDEX  (14ULL) /* AP_RORO, PXN, X */
1832 #define APRR_USER_RO_INDEX  (15ULL) /* AP_RORO, PXN, XN */
1833 #define APRR_MAX_INDEX      (15ULL) /* For sanity checking index values */
1834 #endif /* __APRR_SUPPORTED */
1835
1836
1837 #if __APRR_SUPPORTED__
1838 #define APRR_SHIFT_FOR_IDX(x) \
1839         ((x) << 2ULL)
1840
1841 /* Shifts for attributes, named based on how we intend to use them. */
1842 #define APRR_FIRM_RX_SHIFT  (0ULL)  /* AP_RWNA, PX, X */
1843 #define APRR_FIRM_RO_SHIFT  (4ULL)  /* AP_RWNA, PX, XN */
1844 #define APRR_PPL_RW_SHIFT   (8ULL)  /* AP_RWNA, PXN, X */
1845 #define APRR_KERN_RW_SHIFT  (12ULL) /* AP_RWNA, PXN, XN */
1846 #define APRR_FIRM_RW_SHIFT  (16ULL) /* AP_RWRW, PX, X */
1847 #define APRR_KERN0_RW_SHIFT (20ULL) /* AP_RWRW, PX, XN */
1848 #define APRR_USER_JIT_SHIFT (24ULL) /* AP_RWRW, PXN, X */
1849 #define APRR_USER_RW_SHIFT  (28ULL) /* AP_RWRW, PXN, XN */
1850 #define APRR_PPL_RX_SHIFT   (32ULL) /* AP_RONA, PX, X */
1851 #define APRR_KERN_RX_SHIFT  (36ULL) /* AP_RONA, PX, XN */
1852 #define APRR_USER_XO_SHIFT  (40ULL) /* AP_RONA, PXN, X */
1853 #define APRR_KERN_RO_SHIFT  (44ULL) /* AP_RONA, PXN, XN */
1854 #define APRR_KERN0_RX_SHIFT (48ULL) /* AP_RORO, PX, X */
1855 #define APRR_KERN0_RO_SHIFT (52ULL) /* AP_RORO, PX, XN */
1856 #define APRR_USER_RX_SHIFT  (56ULL) /* AP_RORO, PXN, X */
1857 #define APRR_USER_RO_SHIFT  (60ULL) /* AP_RORO, PXN, XN */
1858
1859 #define ARM_PTE_APRR_MASK \
1860         (ARM_PTE_APMASK | ARM_PTE_PNXMASK | ARM_PTE_NXMASK)
1861
1862 #define ARM_PTE_XPRR_MASK ARM_PTE_APRR_MASK
1863
1864 #define APRR_INDEX_TO_PTE(x) \
1865         ((pt_entry_t) \
1866          (((x) & 0x8) ? ARM_PTE_AP(0x2) : 0) | \
1867          (((x) & 0x4) ? ARM_PTE_AP(0x1) : 0) | \
1868          (((x) & 0x2) ? ARM_PTE_PNX : 0) | \
1869          (((x) & 0x1) ? ARM_PTE_NX : 0))
1870
1871 #define PTE_TO_APRR_INDEX(x) \
1872         ((ARM_PTE_EXTRACT_AP(x) << APRR_IDX_APSHIFT) | \
1873         (((x) & ARM_PTE_PNXMASK) ? APRR_IDX_PXN : 0) | \
1874         (((x) & ARM_PTE_NXMASK) ? APRR_IDX_XN : 0))
1875
1876 #endif /* __APRR_SUPPORTED__ */
1877
1878 #if __APRR_SUPPORTED__
1879
1880 #define APRR_EXTRACT_IDX_ATTR(_aprr_value, _idx) \
1881         (((_aprr_value) >> APRR_SHIFT_FOR_IDX(_idx)) & APRR_ATTR_MASK)
1882
1883 #define APRR_REMOVE(x) (~(x))
1884
1885 #define APRR_EL1_UNRESTRICTED (0x4455445566666677ULL)
1886
1887 #define APRR_EL1_RESET \
1888         APRR_EL1_UNRESTRICTED
1889
1890 /*
1891  * XO mappings bypass PAN protection (rdar://58360875)
1892  * Revoke ALL kernel access permissions for XO mappings.
1893  */
1894 #define APRR_EL1_BASE \
1895         (APRR_EL1_UNRESTRICTED & \
1896         APRR_REMOVE(APRR_ATTR_R << APRR_USER_XO_SHIFT))
1897
1898 #if XNU_MONITOR
1899 #define APRR_EL1_DEFAULT \
1900         (APRR_EL1_BASE & \
1901          (APRR_REMOVE((APRR_ATTR_WX << APRR_PPL_RW_SHIFT) | \
1902          (APRR_ATTR_WX << APRR_USER_XO_SHIFT) | \
1903          (APRR_ATTR_WX << APRR_PPL_RX_SHIFT))))
1904
1905 #define APRR_EL1_PPL \
1906         (APRR_EL1_BASE & \
1907          (APRR_REMOVE((APRR_ATTR_X << APRR_PPL_RW_SHIFT) | \
1908          (APRR_ATTR_WX << APRR_USER_XO_SHIFT) | \
1909          (APRR_ATTR_W << APRR_PPL_RX_SHIFT))))
1910 #else
1911 #define APRR_EL1_DEFAULT \
1912         APRR_EL1_BASE
1913 #endif
1914
1915 #define APRR_EL0_UNRESTRICTED (0x4545010167670101ULL)
1916
1917 #define APRR_EL0_RESET \
1918         APRR_EL0_UNRESTRICTED
1919
1920 #if XNU_MONITOR
1921 #define APRR_EL0_BASE \
1922         (APRR_EL0_UNRESTRICTED & \
1923          (APRR_REMOVE((APRR_ATTR_RWX << APRR_PPL_RW_SHIFT) | \
1924          (APRR_ATTR_RWX << APRR_PPL_RX_SHIFT) | \
1925          (APRR_ATTR_RWX << APRR_USER_XO_SHIFT))))
1926 #else
1927 #define APRR_EL0_BASE \
1928         APRR_EL0_UNRESTRICTED
1929 #endif
1930
1931 #define APRR_EL0_JIT_RW \
1932         (APRR_EL0_BASE & APRR_REMOVE(APRR_ATTR_X << APRR_USER_JIT_SHIFT))
1933
1934 #define APRR_EL0_JIT_RX \
1935         (APRR_EL0_BASE & APRR_REMOVE(APRR_ATTR_W << APRR_USER_JIT_SHIFT))
1936
1937 #define APRR_EL0_JIT_RWX \
1938         APRR_EL0_BASE
1939
1940 #define APRR_EL0_DEFAULT \
1941         APRR_EL0_BASE
1942
1943 #endif /* __APRR_SUPPORTED__ */
1944
1945
1946 /*
1947  * ID_AA64ISAR0_EL1 - AArch64 Instruction Set Attribute Register 0
1948  *
1949  *  63      24 23    20 19  16 15  12 11   8 7   4 3    0
1950  * +----------+--------+------+------+------+-----+------+
1951  * | reserved | atomic |crc32 | sha2 | sha1 | aes | res0 |
1952  * +----------+--------+------+------+------+-----+------+
1953  */
1954
1955 #define ID_AA64ISAR0_EL1_FHM_OFFSET    48
1956 #define ID_AA64ISAR0_EL1_FHM_MASK      (0xfull << ID_AA64ISAR0_EL1_FHM_OFFSET)
1957 #define ID_AA64ISAR0_EL1_FHM_8_2       (1ull << ID_AA64ISAR0_EL1_FHM_OFFSET)
1958
1959 #define ID_AA64ISAR0_EL1_ATOMIC_OFFSET 20
1960 #define ID_AA64ISAR0_EL1_ATOMIC_MASK   (0xfull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET)
1961 #define ID_AA64ISAR0_EL1_ATOMIC_8_1    (2ull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET)
1962
1963 #define ID_AA64ISAR0_EL1_CRC32_OFFSET  16
1964 #define ID_AA64ISAR0_EL1_CRC32_MASK    (0xfull << ID_AA64ISAR0_EL1_CRC32_OFFSET)
1965 #define ID_AA64ISAR0_EL1_CRC32_EN      (1ull << ID_AA64ISAR0_EL1_CRC32_OFFSET)
1966
1967 #define ID_AA64ISAR0_EL1_SHA3_OFFSET   32
1968 #define ID_AA64ISAR0_EL1_SHA3_MASK     (0xfull << ID_AA64ISAR0_EL1_SHA3_OFFSET)
1969 #define ID_AA64ISAR0_EL1_SHA3_EN       (1ull << ID_AA64ISAR0_EL1_SHA3_OFFSET)
1970
1971 #define ID_AA64ISAR0_EL1_SHA2_OFFSET   12
1972 #define ID_AA64ISAR0_EL1_SHA2_MASK     (0xfull << ID_AA64ISAR0_EL1_SHA2_OFFSET)
1973 #define ID_AA64ISAR0_EL1_SHA2_EN       (1ull << ID_AA64ISAR0_EL1_SHA2_OFFSET)
1974
1975 #define ID_AA64ISAR0_EL1_SHA1_OFFSET   8
1976 #define ID_AA64ISAR0_EL1_SHA1_MASK     (0xfull << ID_AA64ISAR0_EL1_SHA1_OFFSET)
1977 #define ID_AA64ISAR0_EL1_SHA1_EN       (1ull << ID_AA64ISAR0_EL1_SHA1_OFFSET)
1978
1979 #define ID_AA64ISAR0_EL1_AES_OFFSET    4
1980 #define ID_AA64ISAR0_EL1_AES_MASK      (0xfull << ID_AA64ISAR0_EL1_AES_OFFSET)
1981 #define ID_AA64ISAR0_EL1_AES_EN        (1ull << ID_AA64ISAR0_EL1_AES_OFFSET)
1982 #define ID_AA64ISAR0_EL1_AES_PMULL_EN  (2ull << ID_AA64ISAR0_EL1_AES_OFFSET)
1983
1984
1985 #if __APCFG_SUPPORTED__
1986 /*
1987  * APCFG_EL1
1988  *
1989  *  63       2 1 0
1990  * +----------+-+-+
1991  * | reserved |K|R|
1992  * +----------+-+-+
1993  *
1994  * where:
1995  *   R: Reserved
1996  *   K: ElXEnKey - Enable ARMV8.3 defined {IA,IB,DA,DB} keys when CPU is
1997  *                 operating in EL1 (or higher) and when under Apple-Mode
1998  */
1999
2000 #define APCFG_EL1_ELXENKEY_OFFSET      1
2001 #define APCFG_EL1_ELXENKEY_MASK        (0x1ULL << APCFG_EL1_ELXENKEY_OFFSET)
2002 #define APCFG_EL1_ELXENKEY             APCFG_EL1_ELXENKEY_MASK
2003 #endif /* __APCFG_SUPPORTED__ */
2004
2005 #define APSTATE_G_SHIFT  (0)
2006 #define APSTATE_P_SHIFT  (1)
2007 #define APSTATE_A_SHIFT  (2)
2008 #define APSTATE_AP_MASK  ((1ULL << APSTATE_A_SHIFT) | (1ULL << APSTATE_P_SHIFT))
2009
2010 #ifdef __APSTS_SUPPORTED__
2011 #define APCTL_EL1_AppleMode  (1ULL << 0)
2012 #define APCTL_EL1_KernKeyEn  (1ULL << 1)
2013 #define APCTL_EL1_EnAPKey0   (1ULL << 2)
2014 #define APCTL_EL1_EnAPKey1   (1ULL << 3)
2015 #ifdef HAS_APCTL_EL1_USERKEYEN
2016 #define APCTL_EL1_UserKeyEn_OFFSET      4
2017 #define APCTL_EL1_UserKeyEn             (1ULL << APCTL_EL1_UserKeyEn_OFFSET)
2018 #endif /* HAS_APCTL_EL1_USERKEYEN */
2019 #define APSTS_EL1_MKEYVld    (1ULL << 0)
2020 #else
2021 #define APCTL_EL1_AppleMode  (1ULL << 0)
2022 #define APCTL_EL1_MKEYVld    (1ULL << 1)
2023 #define APCTL_EL1_KernKeyEn  (1ULL << 2)
2024 #endif
2025
2026 #define ACTLR_EL1_EnTSO   (1ULL << 1)
2027 #define ACTLR_EL1_EnAPFLG (1ULL << 4)
2028 #define ACTLR_EL1_EnAFP   (1ULL << 5)
2029 #define ACTLR_EL1_EnPRSV  (1ULL << 6)
2030
2031 #define ACTLR_EL1_DisHWP_OFFSET  3
2032 #define ACTLR_EL1_DisHWP_MASK    (1ULL << ACTLR_EL1_DisHWP_OFFSET)
2033 #define ACTLR_EL1_DisHWP         ACTLR_EL1_DisHWP_MASK
2034
2035
2036 #if HAS_IC_INVAL_FILTERS
2037 #define ACTLR_EL1_IC_IVAU_EnASID_OFFSET 12
2038 #define ACTLR_EL1_IC_IVAU_EnASID_MASK   (1ULL << ACTLR_EL1_IC_IVAU_EnASID_OFFSET)
2039 #define ACTLR_EL1_IC_IVAU_EnASID        ACTLR_EL1_IC_IVAU_EnASID_MASK
2040 #endif /* HAS_IC_INVAL_FILTERS */
2041
2042 #define AFPCR_DAZ_SHIFT  (0)
2043 #define AFPCR_FTZ_SHIFT  (1)
2044
2045 #if defined(HAS_APPLE_PAC)
2046 // The value of ptrauth_string_discriminator("recover"), hardcoded so it can be used from assembly code
2047 #define PAC_DISCRIMINATOR_RECOVER    0x1e02
2048 #endif
2049
2050
2051 #define CTR_EL0_L1Ip_OFFSET 14
2052 #define CTR_EL0_L1Ip_VIPT (2ULL << CTR_EL0_L1Ip_OFFSET)
2053 #define CTR_EL0_L1Ip_PIPT (3ULL << CTR_EL0_L1Ip_OFFSET)
2054 #define CTR_EL0_L1Ip_MASK (3ULL << CTR_EL0_L1Ip_OFFSET)
2055
2056
2057 #ifdef __ASSEMBLER__
2058
2059 /*
2060  * Conditionally write to system/special-purpose register.
2061  * The register is written to only when the first two arguments
2062  * do not match. If they do match, the macro jumps to a
2063  * caller-provided label.
2064  * The _ISB variant also conditionally issues an ISB after the MSR.
2065  *
2066  * $0 - System/special-purpose register to modify
2067  * $1 - Register containing current FPCR value
2068  * $2 - Register containing expected value
2069  * $3 - Label to jump to when register is already set to expected value
2070  */
2071 .macro CMSR
2072 cmp $1, $2
2073
2074 /* Skip expensive MSR if not required */
2075 b.eq $3f
2076 msr $0, $2
2077 .endmacro
2078
2079 .macro CMSR_ISB
2080 CMSR $0, $1, $2, $3
2081 isb sy
2082 .endmacro
2083
2084 /*
2085  * Modify FPCR only if it does not contain the XNU default value.
2086  * $0 - Register containing current FPCR value
2087  * $1 - Scratch register
2088  * $2 - Label to jump to when FPCR is already set to default value
2089  */
2090 .macro SANITIZE_FPCR
2091 mov $1, #FPCR_DEFAULT
2092 CMSR FPCR, $0, $1, $2
2093 .endmacro
2094
2095 /*
2096  * Family of macros that can be used to protect code sections such that they
2097  * are only executed on a particular SoC/Revision/CPU, and skipped otherwise.
2098  * All macros will forward-jump to 1f when the condition is not matched.
2099  * This label may be defined manually, or implicitly through the use of
2100  * the EXEC_END macro.
2101  * For cores, XX can be: EQ (equal), ALL (don't care).
2102  * For revisions, XX can be: EQ (equal), LO (lower than), HS (higher or same), ALL (don't care).
2103  */
2104
2105 /*
2106  * $0 - MIDR_SOC[_CORE], e.g. MIDR_ARUBA_VORTEX
2107  * $1 - CPU_VERSION_XX, e.g. CPU_VERSION_B1
2108  * $2 - GPR containing MIDR_EL1 value
2109  * $3 - Scratch register
2110  */
2111 .macro EXEC_COREEQ_REVEQ
2112 and $3, $2, #MIDR_EL1_PNUM_MASK
2113 cmp $3, $0
2114 b.ne 1f
2115
2116 mov $3, $2
2117 bfi  $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #4
2118 ubfx $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #8
2119 cmp $3, $1
2120 b.ne 1f
2121 .endmacro
2122
2123 .macro EXEC_COREEQ_REVLO
2124 and $3, $2, #MIDR_EL1_PNUM_MASK
2125 cmp $3, $0
2126 b.ne 1f
2127
2128 mov $3, $2
2129 bfi  $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #4
2130 ubfx $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #8
2131 cmp $3, $1
2132 b.pl 1f
2133 .endmacro
2134
2135 .macro EXEC_COREEQ_REVHS
2136 and $3, $2, #MIDR_EL1_PNUM_MASK
2137 cmp $3, $0
2138 b.ne 1f
2139
2140 mov $3, $2
2141 bfi  $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #4
2142 ubfx $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #8
2143 cmp $3, $1
2144 b.mi 1f
2145 .endmacro
2146
2147 /*
2148  * $0 - CPU_VERSION_XX, e.g. CPU_VERSION_B1
2149  * $1 - GPR containing MIDR_EL1 value
2150  * $2 - Scratch register
2151  */
2152 .macro EXEC_COREALL_REVEQ
2153 mov $2, $1
2154 bfi  $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4
2155 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8
2156 cmp $2, $0
2157 b.ne 1f
2158 .endmacro
2159
2160 .macro EXEC_COREALL_REVLO
2161 mov  $2, $1
2162 bfi  $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4
2163 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8
2164 cmp $2, $0
2165 b.pl 1f
2166 .endmacro
2167
2168 .macro EXEC_COREALL_REVHS
2169 mov $2, $1
2170 bfi  $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4
2171 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8
2172 cmp $2, $0
2173 b.mi 1f
2174 .endmacro
2175
2176 /*
2177  * $0 - MIDR_SOC[_CORE], e.g. MIDR_ARUBA_VORTEX
2178  * $1 - GPR containing MIDR_EL1 value
2179  * $2 - Scratch register
2180  */
2181 .macro EXEC_COREEQ_REVALL
2182 and $2, $1, #MIDR_EL1_PNUM_MASK
2183 cmp $2, $0
2184 b.ne 1f
2185 .endmacro
2186
2187 /*
2188  * $0 - CPU_VERSION_XX, e.g. CPU_VERSION_B1
2189  * $1 - GPR containing MIDR_EL1 value
2190  * $2 - Scratch register
2191  */
2192 .macro EXEC_PCORE_REVEQ
2193 mrs $2, MPIDR_EL1
2194 and $2, $2, #(MPIDR_PNE)
2195 cmp $2, xzr
2196 b.eq 1f
2197
2198 mov $2, $1
2199 bfi  $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4
2200 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8
2201 cmp $2, $0
2202 b.ne 1f
2203 .endmacro
2204
2205 .macro EXEC_PCORE_REVLO
2206 mrs $2, MPIDR_EL1
2207 and $2, $2, #(MPIDR_PNE)
2208 cmp $2, xzr
2209 b.eq 1f
2210
2211 mov $2, $1
2212 bfi  $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4
2213 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8
2214 cmp $2, $0
2215 b.pl 1f
2216 .endmacro
2217
2218 .macro EXEC_PCORE_REVHS
2219 mrs $2, MPIDR_EL1
2220 and $2, $2, #(MPIDR_PNE)
2221 cmp $2, xzr
2222 b.eq 1f
2223
2224 mov $2, $1
2225 bfi  $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4
2226 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8
2227 cmp $2, $0
2228 b.mi 1f
2229 .endmacro
2230
2231 .macro EXEC_ECORE_REVEQ
2232 mrs $2, MPIDR_EL1
2233 and $2, $2, #(MPIDR_PNE)
2234 cmp $2, xzr
2235 b.ne 1f
2236
2237 mov $2, $1
2238 bfi  $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4
2239 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8
2240 cmp $2, $0
2241 b.ne 1f
2242 .endmacro
2243
2244 .macro EXEC_ECORE_REVLO
2245 mrs $2, MPIDR_EL1
2246 and $2, $2, #(MPIDR_PNE)
2247 cmp $2, xzr
2248 b.ne 1f
2249
2250 mov $2, $1
2251 bfi  $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4
2252 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8
2253 cmp $2, $0
2254 b.pl 1f
2255 .endmacro
2256
2257 .macro EXEC_ECORE_REVHS
2258 mrs $2, MPIDR_EL1
2259 and $2, $2, #(MPIDR_PNE)
2260 cmp $2, xzr
2261 b.ne 1f
2262
2263 mov $2, $1
2264 bfi  $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4
2265 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8
2266 cmp $2, $0
2267 b.mi 1f
2268 .endmacro
2269
2270 /*
2271  * $0 - GPR containing MIDR_EL1 value
2272  * $1 - Scratch register
2273  */
2274 .macro EXEC_PCORE_REVALL
2275 mrs $1, MPIDR_EL1
2276 and $1, $1, #(MPIDR_PNE)
2277 cmp $1, xzr
2278 b.eq 1f
2279 .endmacro
2280
2281 .macro EXEC_ECORE_REVALL
2282 mrs $1, MPIDR_EL1
2283 and $1, $1, #(MPIDR_PNE)
2284 cmp $1, xzr
2285 b.ne 1f
2286 .endmacro
2287
2288
2289
2290 /*
2291  * Macro that defines the label that all EXEC_COREXX_REVXX macros jump to.
2292  */
2293 .macro EXEC_END
2294 1:
2295 .endmacro
2296
2297 /*
2298  * Sets bits in an SPR register.
2299  * arg0: Name of the register to be accessed.
2300  * arg1: Mask of bits to be set.
2301  * arg2: Scratch register
2302  */
2303 .macro HID_SET_BITS
2304 mrs $2, $0
2305 orr $2, $2, $1
2306 msr $0, $2
2307 .endmacro
2308
2309 /*
2310  * Clears bits in an SPR register.
2311  * arg0: Name of the register to be accessed.
2312  * arg1: Mask of bits to be cleared.
2313  * arg2: Scratch register
2314  */
2315 .macro HID_CLEAR_BITS
2316 mrs $2, $0
2317 bic $2, $2, $1
2318 msr $0, $2
2319 .endmacro
2320
2321 /*
2322  * Clears bits in an SPR register.
2323  * arg0: Name of the register to be accessed.
2324  * arg1: Mask of bits to be cleared.
2325  * arg2: Value to insert
2326  * arg3: Scratch register
2327  */
2328 .macro HID_INSERT_BITS
2329 mrs $3, $0
2330 bic $3, $3, $1
2331 orr $3, $3, $2
2332 msr $0, $3
2333 .endmacro
2334
2335 /*
2336  * Macro intended to be used as a replacement for ERET.
2337  * It prevents speculation past ERET instructions by padding
2338  * up to the decoder width.
2339  */
2340 .macro ERET_CONTEXT_SYNCHRONIZING
2341 eret
2342 #if __ARM_SB_AVAILABLE__
2343 sb                              // Technically unnecessary on Apple micro-architectures, may restrict mis-speculation on other architectures
2344 #else /* __ARM_SB_AVAILABLE__ */
2345 isb                             // ISB technically unnecessary on Apple micro-architectures, may restrict mis-speculation on other architectures
2346 nop                             // Sequence of six NOPs to pad out and terminate instruction decode group */
2347 nop
2348 nop
2349 nop
2350 nop
2351 nop
2352 #endif /* !__ARM_SB_AVAILABLE__ */
2353 .endmacro
2354
2355 #endif /* __ASSEMBLER__ */
2356
2357 #define MSR(reg, src)  __asm__ volatile ("msr " reg ", %0" :: "r" (src))
2358 #define MRS(dest, reg) __asm__ volatile ("mrs %0, " reg : "=r" (dest))
2359
2360 #if XNU_MONITOR
2361 #define __ARM_PTE_PHYSMAP__ 1
2362 #define PPL_STATE_KERNEL    0
2363 #define PPL_STATE_DISPATCH  1
2364 #define PPL_STATE_PANIC     2
2365 #define PPL_STATE_EXCEPTION 3
2366 #endif
2367
2368
2369 #endif /* _ARM64_PROC_REG_H_ */