xnu-6153.41.3.tar.gz

[apple/xnu.git] / osfmk / arm64 / proc_reg.h

diff --git a/osfmk/arm64/proc_reg.h b/osfmk/arm64/proc_reg.h
index ee13e1844255469b2bc4232e054c73c35ad8ccb7..66a551ffed01e8d1a37d0bb5e7540d22c47c4680 100644 (file)
--- a/osfmk/arm64/proc_reg.h
+++ b/osfmk/arm64/proc_reg.h
@@ -97,101 +97,107 @@
  * +-+-+-+-+-+---+---+--+--+----------+-+-+-+-+-+-----+
  *
  * where:
  * +-+-+-+-+-+---+---+--+--+----------+-+-+-+-+-+-----+
  *
  * where:

- *     NZCV    Comparison flags
- *     PAN             Privileged Access Never
- *  SS         Single step
- *     IL              Illegal state
- *     DAIF    Interrupt masks
- *     M               Mode field
+ *   NZCV: Comparison flags
+ *   PAN:  Privileged Access Never
+ *   SS:   Single step
+ *   IL:   Illegal state
+ *   DAIF: Interrupt masks
+ *   M:    Mode field

  */
 
  */
 

-#define PSR64_NZCV_SHIFT                28
-#define PSR64_NZCV_MASK                 (1 << PSR64_NZCV_SHIFT)
+#define PSR64_NZCV_SHIFT 28
+#define PSR64_NZCV_MASK  (1 << PSR64_NZCV_SHIFT)

 
 

-#define PSR64_N_SHIFT                   31
-#define PSR64_N                                 (1 << PSR64_N_SHIFT)
+#define PSR64_N_SHIFT    31
+#define PSR64_N          (1 << PSR64_N_SHIFT)

 
 

-#define PSR64_Z_SHIFT                   30
-#define PSR64_Z                                 (1 << PSR64_Z_SHIFT)
+#define PSR64_Z_SHIFT    30
+#define PSR64_Z          (1 << PSR64_Z_SHIFT)

 
 

-#define PSR64_C_SHIFT                   29
-#define PSR64_C                                 (1 << PSR64_C_SHIFT)
+#define PSR64_C_SHIFT    29
+#define PSR64_C          (1 << PSR64_C_SHIFT)

 
 

-#define PSR64_V_SHIFT                   28
-#define PSR64_V                                 (1 << PSR64_V_SHIFT)
+#define PSR64_V_SHIFT    28
+#define PSR64_V          (1 << PSR64_V_SHIFT)

 
 

-#define PSR64_PAN_SHIFT                 22
-#define PSR64_PAN                               (1 << PSR64_PAN_SHIFT)
+#define PSR64_PAN_SHIFT  22
+#define PSR64_PAN        (1 << PSR64_PAN_SHIFT)

 
 

-#define PSR64_SS_SHIFT                  21
-#define PSR64_SS                                (1 << PSR64_SS_SHIFT)
+#define PSR64_SS_SHIFT   21
+#define PSR64_SS         (1 << PSR64_SS_SHIFT)

 
 

-#define PSR64_IL_SHIFT                  20
-#define PSR64_IL                                (1 << PSR64_IL_SHIFT)
+#define PSR64_IL_SHIFT   20
+#define PSR64_IL         (1 << PSR64_IL_SHIFT)

 
 /*
  * msr DAIF, Xn and mrs Xn, DAIF transfer into
  * and out of bits 9:6
  */
 
 /*
  * msr DAIF, Xn and mrs Xn, DAIF transfer into
  * and out of bits 9:6
  */

-#define DAIF_DEBUG_SHIFT                9
-#define DAIF_DEBUGF                             (1 << DAIF_DEBUG_SHIFT)
+#define DAIF_DEBUG_SHIFT      9
+#define DAIF_DEBUGF           (1 << DAIF_DEBUG_SHIFT)

 
 

-#define DAIF_ASYNC_SHIFT                8
-#define DAIF_ASYNCF                             (1 << DAIF_ASYNC_SHIFT)
+#define DAIF_ASYNC_SHIFT      8
+#define DAIF_ASYNCF           (1 << DAIF_ASYNC_SHIFT)

 
 

-#define DAIF_IRQF_SHIFT                 7
-#define DAIF_IRQF                               (1 << DAIF_IRQF_SHIFT)
+#define DAIF_IRQF_SHIFT       7
+#define DAIF_IRQF             (1 << DAIF_IRQF_SHIFT)

 
 

-#define DAIF_FIQF_SHIFT                 6
-#define DAIF_FIQF                               (1 << DAIF_FIQF_SHIFT)
+#define DAIF_FIQF_SHIFT       6
+#define DAIF_FIQF             (1 << DAIF_FIQF_SHIFT)

 
 

-#define DAIF_ALL                                (DAIF_DEBUGF | DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF)
-#define DAIF_STANDARD_DISABLE   (DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF)
+#define DAIF_ALL              (DAIF_DEBUGF | DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF)
+#define DAIF_STANDARD_DISABLE (DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF)

 
 

-#define SPSR_INTERRUPTS_ENABLED(x)      (!(x & DAIF_FIQF))
+#define SPSR_INTERRUPTS_ENABLED(x) (!(x & DAIF_FIQF))

 
 /*
  * msr DAIFSet, Xn, and msr DAIFClr, Xn transfer
  * from bits 3:0.
  */
 
 /*
  * msr DAIFSet, Xn, and msr DAIFClr, Xn transfer
  * from bits 3:0.
  */

-#define DAIFSC_DEBUGF                   (1 << 3)
-#define DAIFSC_ASYNCF                   (1 << 2)
-#define DAIFSC_IRQF                             (1 << 1)
-#define DAIFSC_FIQF                             (1 << 0)
-#define DAIFSC_ALL                              (DAIFSC_DEBUGF | DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF)
+#define DAIFSC_DEBUGF           (1 << 3)
+#define DAIFSC_ASYNCF           (1 << 2)
+#define DAIFSC_IRQF             (1 << 1)
+#define DAIFSC_FIQF             (1 << 0)
+#define DAIFSC_ALL              (DAIFSC_DEBUGF | DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF)

 #define DAIFSC_STANDARD_DISABLE (DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF)
 
 /*
  * ARM64_TODO: unify with ARM?
  */
 #define DAIFSC_STANDARD_DISABLE (DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF)
 
 /*
  * ARM64_TODO: unify with ARM?
  */

-#define PSR64_CF                0x20000000      /* Carry/Borrow/Extend */
+#define PSR64_CF         0x20000000 /* Carry/Borrow/Extend */

 
 

-#define PSR64_MODE_MASK                 0x1F
+#define PSR64_MODE_MASK         0x1F

 
 

-#define PSR64_MODE_USER32_THUMB         0x20
+#define PSR64_MODE_USER32_THUMB 0x20

 
 

-#define PSR64_MODE_RW_SHIFT             4
-#define PSR64_MODE_RW_64                0
-#define PSR64_MODE_RW_32                (0x1 << PSR64_MODE_RW_SHIFT)
+#define PSR64_MODE_RW_SHIFT     4
+#define PSR64_MODE_RW_64        0
+#define PSR64_MODE_RW_32        (0x1 << PSR64_MODE_RW_SHIFT)

 
 

-#define PSR64_MODE_EL_SHIFT             2
-#define PSR64_MODE_EL_MASK              (0x3 << PSR64_MODE_EL_SHIFT)
-#define PSR64_MODE_EL3                  (0x3 << PSR64_MODE_EL_SHIFT)
-#define PSR64_MODE_EL1                  (0x1 << PSR64_MODE_EL_SHIFT)
-#define PSR64_MODE_EL0                  0
+#define PSR64_MODE_EL_SHIFT     2
+#define PSR64_MODE_EL_MASK      (0x3 << PSR64_MODE_EL_SHIFT)
+#define PSR64_MODE_EL3          (0x3 << PSR64_MODE_EL_SHIFT)
+#define PSR64_MODE_EL2          (0x2 << PSR64_MODE_EL_SHIFT)
+#define PSR64_MODE_EL1          (0x1 << PSR64_MODE_EL_SHIFT)
+#define PSR64_MODE_EL0          0

 
 

-#define PSR64_MODE_SPX                  0x1
-#define PSR64_MODE_SP0                  0
+#define PSR64_MODE_SPX          0x1
+#define PSR64_MODE_SP0          0

 
 

-#define PSR64_USER32_DEFAULT            (PSR64_MODE_RW_32 | PSR64_MODE_EL0 | PSR64_MODE_SP0)
-#define PSR64_USER64_DEFAULT            (PSR64_MODE_RW_64 | PSR64_MODE_EL0 | PSR64_MODE_SP0)
-#define PSR64_KERNEL_DEFAULT    (DAIF_STANDARD_DISABLE | PSR64_MODE_RW_64 | PSR64_MODE_EL1 | PSR64_MODE_SP0)
+#define PSR64_USER32_DEFAULT    (PSR64_MODE_RW_32 | PSR64_MODE_EL0 | PSR64_MODE_SP0)
+#define PSR64_USER64_DEFAULT    (PSR64_MODE_RW_64 | PSR64_MODE_EL0 | PSR64_MODE_SP0)
+#define PSR64_KERNEL_STANDARD   (DAIF_STANDARD_DISABLE | PSR64_MODE_RW_64 | PSR64_MODE_EL1 | PSR64_MODE_SP0)
+#if __ARM_PAN_AVAILABLE__
+#define PSR64_KERNEL_DEFAULT    (PSR64_KERNEL_STANDARD | PSR64_PAN)
+#else
+#define PSR64_KERNEL_DEFAULT    PSR64_KERNEL_STANDARD
+#endif

 
 

-#define PSR64_IS_KERNEL(x)              ((x & PSR64_MODE_EL_MASK) == PSR64_MODE_EL1)
-#define PSR64_IS_USER(x)                ((x & PSR64_MODE_EL_MASK) == PSR64_MODE_EL0)
+#define PSR64_IS_KERNEL(x)      ((x & PSR64_MODE_EL_MASK) > PSR64_MODE_EL0)
+#define PSR64_IS_USER(x)        ((x & PSR64_MODE_EL_MASK) == PSR64_MODE_EL0)

 
 

-#define PSR64_IS_USER32(x)              (PSR64_IS_USER(x) && (x & PSR64_MODE_RW_32))
-#define PSR64_IS_USER64(x)              (PSR64_IS_USER(x) && !(x & PSR64_MODE_RW_32))
+#define PSR64_IS_USER32(x)      (PSR64_IS_USER(x) && (x & PSR64_MODE_RW_32))
+#define PSR64_IS_USER64(x)      (PSR64_IS_USER(x) && !(x & PSR64_MODE_RW_32))

 
 
 
 
 
 


@@ -199,82 +205,95 @@
  * System Control Register (SCTLR)
  */
 
  * System Control Register (SCTLR)
  */
 

-#define SCTLR_RESERVED                  ((3 << 28) | (1 << 22) | (1 << 20) | (1 << 11))
+#define SCTLR_RESERVED     ((3ULL << 28) | (1ULL << 22) | (1ULL << 20) | (1ULL << 11))
+#if defined(HAS_APPLE_PAC)
+
+// 31    PACIA_ENABLED AddPACIA and AuthIA functions enabled
+#define SCTLR_PACIA_ENABLED_SHIFT 31
+#define SCTLR_PACIA_ENABLED       (1ULL << SCTLR_PACIA_ENABLED_SHIFT)
+// 30    PACIB_ENABLED AddPACIB and AuthIB functions enabled
+#define SCTLR_PACIB_ENABLED       (1ULL << 30)
+// 29:28 RES1 11
+// 27    PACDA_ENABLED AddPACDA and AuthDA functions enabled
+#define SCTLR_PACDA_ENABLED       (1ULL << 27)
+// 13    PACDB_ENABLED  AddPACDB and AuthDB functions enabled
+#define SCTLR_PACDB_ENABLED       (1ULL << 13)

 
 

-// 26          UCI             User Cache Instructions
-#define SCTLR_UCI_ENABLED               (1 << 26)
+#define SCTLR_JOP_KEYS_ENABLED (SCTLR_PACIA_ENABLED | SCTLR_PACDA_ENABLED | SCTLR_PACDB_ENABLED)
+#endif /* defined(HAS_APPLE_PAC) */

 
 

-// 25          EE              Exception Endianness
-#define SCTLR_EE_BIG_ENDIAN             (1 << 25)
+// 26    UCI User Cache Instructions
+#define SCTLR_UCI_ENABLED         (1ULL << 26)

 
 

-// 24          E0E             EL0 Endianness
-#define SCTLR_E0E_BIG_ENDIAN    (1 << 24)
+// 25    EE             Exception Endianness
+#define SCTLR_EE_BIG_ENDIAN       (1ULL << 25)

 
 

-// 23          SPAN    Set PAN
-#define SCTLR_PAN_UNCHANGED             (1 << 23)
+// 24    E0E            EL0 Endianness
+#define SCTLR_E0E_BIG_ENDIAN      (1ULL << 24)

 
 

-// 22          RES1    1
-// 21          RES0    0
-// 20          RES1    1
+// 23    SPAN           Set PAN
+#define SCTLR_PAN_UNCHANGED       (1ULL << 23)

 
 

-// 19          WXN             Writeable implies eXecute Never
-#define SCTLR_WXN_ENABLED               (1 << 19)
+// 22    RES1           1
+// 21    RES0           0
+// 20    RES1           1

 
 

-// 18          nTWE    Not trap WFE from EL0
-#define SCTLR_nTWE_WFE_ENABLED  (1 << 18)
+// 19    WXN            Writeable implies eXecute Never
+#define SCTLR_WXN_ENABLED         (1ULL << 19)

 
 

-// 17          RES0    0
+// 18    nTWE           Not trap WFE from EL0
+#define SCTLR_nTWE_WFE_ENABLED    (1ULL << 18)

 
 

-// 16          nTWI    Not trap WFI from EL0
-#define SCTRL_nTWI_WFI_ENABLED  (1 << 16)
+// 17    RES0           0

 
 

-// 15          UCT             User Cache Type register (CTR_EL0)
-#define SCTLR_UCT_ENABLED               (1 << 15)
+// 16    nTWI           Not trap WFI from EL0
+#define SCTRL_nTWI_WFI_ENABLED    (1ULL << 16)

 
 

-// 14          DZE             User Data Cache Zero (DC ZVA)
-#define SCTLR_DZE_ENABLED               (1 << 14)
+// 15    UCT            User Cache Type register (CTR_EL0)
+#define SCTLR_UCT_ENABLED         (1ULL << 15)

 
 

-// 13          PACDB_ENABLED            AddPACDB and AuthDB functions enabled
-#define SCTLR_PACDB_ENABLED             (1 << 13)
+// 14    DZE            User Data Cache Zero (DC ZVA)
+#define SCTLR_DZE_ENABLED         (1ULL << 14)

 
 

-// 12          I               Instruction cache enable
-#define SCTLR_I_ENABLED                 (1 << 12)
+// 12    I              Instruction cache enable
+#define SCTLR_I_ENABLED           (1ULL << 12)

 
 

-// 11          RES1    1
-// 10          RES0    0
+// 11    RES1           1
+// 10    RES0           0

 
 

-// 9           UMA             User Mask Access
-#define SCTLR_UMA_ENABLED               (1 << 9)
+// 9     UMA            User Mask Access
+#define SCTLR_UMA_ENABLED         (1ULL << 9)

 
 

-// 8           SED             SETEND Disable
-#define SCTLR_SED_DISABLED              (1 << 8)
+// 8     SED            SETEND Disable
+#define SCTLR_SED_DISABLED        (1ULL << 8)

 
 

-// 7           ITD             IT Disable
-#define SCTLR_ITD_DISABLED              (1 << 7)
+// 7     ITD            IT Disable
+#define SCTLR_ITD_DISABLED        (1ULL << 7)

 
 

-// 6           RES0    0
+// 6     RES0           0

 
 

-// 5           CP15BEN CP15 Barrier ENable
-#define SCTLR_CP15BEN_ENABLED   (1 << 5)
+// 5     CP15BEN        CP15 Barrier ENable
+#define SCTLR_CP15BEN_ENABLED     (1ULL << 5)

 
 

-// 4           SA0             Stack Alignment check for EL0
-#define SCTLR_SA0_ENABLED               (1 << 4)
+// 4     SA0            Stack Alignment check for EL0
+#define SCTLR_SA0_ENABLED         (1ULL << 4)

 
 

-// 3           SA              Stack Alignment check
-#define SCTLR_SA_ENABLED                (1 << 3)
+// 3     SA             Stack Alignment check
+#define SCTLR_SA_ENABLED          (1ULL << 3)

 
 

-// 2           C               Cache enable
-#define SCTLR_C_ENABLED                 (1 << 2)
+// 2     C              Cache enable
+#define SCTLR_C_ENABLED           (1ULL << 2)

 
 

-// 1           A               Alignment check
-#define SCTLR_A_ENABLED                 (1 << 1)
+// 1     A              Alignment check
+#define SCTLR_A_ENABLED           (1ULL << 1)

 
 

-// 0           M               MMU enable
-#define SCTLR_M_ENABLED                 (1 << 0)
+// 0     M              MMU enable
+#define SCTLR_M_ENABLED           (1ULL << 0)

 
 

-#define SCTLR_EL1_DEFAULT               (SCTLR_RESERVED | SCTLR_UCI_ENABLED | SCTLR_nTWE_WFE_ENABLED | SCTLR_DZE_ENABLED | \
-                                               SCTLR_I_ENABLED | SCTLR_SED_DISABLED | SCTLR_CP15BEN_ENABLED |             \
-                                               SCTLR_SA0_ENABLED | SCTLR_SA_ENABLED | SCTLR_C_ENABLED | SCTLR_M_ENABLED)
+#define SCTLR_EL1_DEFAULT \
+       (SCTLR_RESERVED | SCTLR_UCI_ENABLED | SCTLR_nTWE_WFE_ENABLED | SCTLR_DZE_ENABLED | \
+        SCTLR_I_ENABLED | SCTLR_SED_DISABLED | SCTLR_CP15BEN_ENABLED |                    \
+        SCTLR_SA0_ENABLED | SCTLR_SA_ENABLED | SCTLR_C_ENABLED | SCTLR_M_ENABLED)

 
 /*
  * Coprocessor Access Control Register (CPACR)
 
 /*
  * Coprocessor Access Control Register (CPACR)


@@ -285,15 +304,15 @@
  * +---+---+------+------+--------------------+
  *
  * where:
  * +---+---+------+------+--------------------+
  *
  * where:

- *     TTA             Trace trap
- *     FPEN    Floating point enable
+ *   TTA:  Trace trap
+ *   FPEN: Floating point enable

  */
  */

-#define CPACR_TTA_SHIFT                         28
-#define CPACR_TTA                                       (1 << CPACR_TTA_SHIFT)
+#define CPACR_TTA_SHIFT     28
+#define CPACR_TTA           (1 << CPACR_TTA_SHIFT)

 
 

-#define CPACR_FPEN_SHIFT                        20
-#define CPACR_FPEN_EL0_TRAP                     (0x1 << CPACR_FPEN_SHIFT)
-#define CPACR_FPEN_ENABLE                       (0x3 << CPACR_FPEN_SHIFT)
+#define CPACR_FPEN_SHIFT    20
+#define CPACR_FPEN_EL0_TRAP (0x1 << CPACR_FPEN_SHIFT)
+#define CPACR_FPEN_ENABLE   (0x3 << CPACR_FPEN_SHIFT)

 
 /*
  *  FPSR: Floating Point Status Register
 
 /*
  *  FPSR: Floating Point Status Register


@@ -304,37 +323,37 @@
  * +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+
  */
 
  * +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+
  */
 

-#define FPSR_N_SHIFT    31
-#define FPSR_Z_SHIFT    30
-#define FPSR_C_SHIFT    29
-#define FPSR_V_SHIFT    28
-#define FPSR_QC_SHIFT   27
-#define FPSR_IDC_SHIFT  7
-#define FPSR_IXC_SHIFT  4
-#define FPSR_UFC_SHIFT  3
-#define FPSR_OFC_SHIFT  2
-#define FPSR_DZC_SHIFT  1
-#define FPSR_IOC_SHIFT  0
-#define FPSR_N          (1 << FPSR_N_SHIFT)
-#define FPSR_Z          (1 << FPSR_Z_SHIFT)
-#define FPSR_C          (1 << FPSR_C_SHIFT)
-#define FPSR_V          (1 << FPSR_V_SHIFT)
-#define FPSR_QC         (1 << FPSR_QC_SHIFT)
-#define FPSR_IDC        (1 << FPSR_IDC_SHIFT)
-#define FPSR_IXC        (1 << FPSR_IXC_SHIFT)
-#define FPSR_UFC        (1 << FPSR_UFC_SHIFT)
-#define FPSR_OFC        (1 << FPSR_OFC_SHIFT)
-#define FPSR_DZC        (1 << FPSR_DZC_SHIFT)
-#define FPSR_IOC        (1 << FPSR_IOC_SHIFT)
+#define FPSR_N_SHIFT   31
+#define FPSR_Z_SHIFT   30
+#define FPSR_C_SHIFT   29
+#define FPSR_V_SHIFT   28
+#define FPSR_QC_SHIFT  27
+#define FPSR_IDC_SHIFT 7
+#define FPSR_IXC_SHIFT 4
+#define FPSR_UFC_SHIFT 3
+#define FPSR_OFC_SHIFT 2
+#define FPSR_DZC_SHIFT 1
+#define FPSR_IOC_SHIFT 0
+#define FPSR_N         (1 << FPSR_N_SHIFT)
+#define FPSR_Z         (1 << FPSR_Z_SHIFT)
+#define FPSR_C         (1 << FPSR_C_SHIFT)
+#define FPSR_V         (1 << FPSR_V_SHIFT)
+#define FPSR_QC        (1 << FPSR_QC_SHIFT)
+#define FPSR_IDC       (1 << FPSR_IDC_SHIFT)
+#define FPSR_IXC       (1 << FPSR_IXC_SHIFT)
+#define FPSR_UFC       (1 << FPSR_UFC_SHIFT)
+#define FPSR_OFC       (1 << FPSR_OFC_SHIFT)
+#define FPSR_DZC       (1 << FPSR_DZC_SHIFT)
+#define FPSR_IOC       (1 << FPSR_IOC_SHIFT)

 
 /*
  * A mask for all for all of the bits that are not RAZ for FPSR; this
  * is primarily for converting between a 32-bit view of NEON state
  * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR).
  */
 
 /*
  * A mask for all for all of the bits that are not RAZ for FPSR; this
  * is primarily for converting between a 32-bit view of NEON state
  * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR).
  */

-#define FPSR_MASK       (FPSR_N | FPSR_Z | FPSR_C | FPSR_V | FPSR_QC | \
-                        FPSR_IDC | FPSR_IXC | FPSR_UFC | FPSR_OFC | \
-                        FPSR_DZC | FPSR_IOC)
+#define FPSR_MASK \
+       (FPSR_N | FPSR_Z | FPSR_C | FPSR_V | FPSR_QC | FPSR_IDC | FPSR_IXC | \
+        FPSR_UFC | FPSR_OFC | FPSR_DZC | FPSR_IOC)

 
 /*
  *  FPCR: Floating Point Control Register
 
 /*
  *  FPCR: Floating Point Control Register


@@ -345,41 +364,41 @@
  * +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+
  */
 
  * +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+
  */
 

-#define FPCR_AHP_SHIFT          26
-#define FPCR_DN_SHIFT           25
-#define FPCR_FZ_SHIFT           24
-#define FPCR_RMODE_SHIFT        22
-#define FPCR_STRIDE_SHIFT       20
-#define FPCR_LEN_SHIFT          16
-#define FPCR_IDE_SHIFT          15
-#define FPCR_IXE_SHIFT          12
-#define FPCR_UFE_SHIFT          11
-#define FPCR_OFE_SHIFT          10
-#define FPCR_DZE_SHIFT          9
-#define FPCR_IOE_SHIFT          8
-#define FPCR_AHP                (1 << FPCR_AHP_SHIFT)
-#define FPCR_DN                 (1 << FPCR_DN_SHIFT)
-#define FPCR_FZ                 (1 << FPCR_FZ_SHIFT)
-#define FPCR_RMODE              (0x3 << FPCR_RMODE_SHIFT)
-#define FPCR_STRIDE             (0x3 << FPCR_STRIDE_SHIFT)
-#define FPCR_LEN                (0x7 << FPCR_LEN_SHIFT)
-#define FPCR_IDE                (1 << FPCR_IDE_SHIFT)
-#define FPCR_IXE                (1 << FPCR_IXE_SHIFT)
-#define FPCR_UFE                (1 << FPCR_UFE_SHIFT)
-#define FPCR_OFE                (1 << FPCR_OFE_SHIFT)
-#define FPCR_DZE                (1 << FPCR_DZE_SHIFT)
-#define FPCR_IOE                (1 << FPCR_IOE_SHIFT)
-#define FPCR_DEFAULT            (FPCR_DN)
-#define FPCR_DEFAULT_32         (FPCR_DN|FPCR_FZ)
+#define FPCR_AHP_SHIFT    26
+#define FPCR_DN_SHIFT     25
+#define FPCR_FZ_SHIFT     24
+#define FPCR_RMODE_SHIFT  22
+#define FPCR_STRIDE_SHIFT 20
+#define FPCR_LEN_SHIFT    16
+#define FPCR_IDE_SHIFT    15
+#define FPCR_IXE_SHIFT    12
+#define FPCR_UFE_SHIFT    11
+#define FPCR_OFE_SHIFT    10
+#define FPCR_DZE_SHIFT    9
+#define FPCR_IOE_SHIFT    8
+#define FPCR_AHP          (1 << FPCR_AHP_SHIFT)
+#define FPCR_DN           (1 << FPCR_DN_SHIFT)
+#define FPCR_FZ           (1 << FPCR_FZ_SHIFT)
+#define FPCR_RMODE        (0x3 << FPCR_RMODE_SHIFT)
+#define FPCR_STRIDE       (0x3 << FPCR_STRIDE_SHIFT)
+#define FPCR_LEN          (0x7 << FPCR_LEN_SHIFT)
+#define FPCR_IDE          (1 << FPCR_IDE_SHIFT)
+#define FPCR_IXE          (1 << FPCR_IXE_SHIFT)
+#define FPCR_UFE          (1 << FPCR_UFE_SHIFT)
+#define FPCR_OFE          (1 << FPCR_OFE_SHIFT)
+#define FPCR_DZE          (1 << FPCR_DZE_SHIFT)
+#define FPCR_IOE          (1 << FPCR_IOE_SHIFT)
+#define FPCR_DEFAULT      (FPCR_DN)
+#define FPCR_DEFAULT_32   (FPCR_DN|FPCR_FZ)

 
 /*
  * A mask for all for all of the bits that are not RAZ for FPCR; this
  * is primarily for converting between a 32-bit view of NEON state
  * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR).
  */
 
 /*
  * A mask for all for all of the bits that are not RAZ for FPCR; this
  * is primarily for converting between a 32-bit view of NEON state
  * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR).
  */

-#define FPCR_MASK               (FPCR_AHP | FPCR_DN | FPCR_FZ | FPCR_RMODE | \
-                                FPCR_STRIDE | FPCR_LEN | FPCR_IDE | FPCR_IXE | \
-                                FPCR_UFE | FPCR_OFE | FPCR_DZE | FPCR_IOE)
+#define FPCR_MASK \
+       (FPCR_AHP | FPCR_DN | FPCR_FZ | FPCR_RMODE | FPCR_STRIDE | FPCR_LEN | \
+        FPCR_IDE | FPCR_IXE | FPCR_UFE | FPCR_OFE | FPCR_DZE | FPCR_IOE)

 
 /*
  * Translation Control Register (TCR)
 
 /*
  * Translation Control Register (TCR)


@@ -398,102 +417,110 @@
  * | zero |TBI1|TBI0|AS|z| IPS | TG1 | SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ | TG0 | SH0 |ORGN0|IRGN0|EPD0|z|T0SZ|
  * +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+
  *
  * | zero |TBI1|TBI0|AS|z| IPS | TG1 | SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ | TG0 | SH0 |ORGN0|IRGN0|EPD0|z|T0SZ|
  * +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+
  *

- *     TBI1    Top Byte Ignored for TTBR1 region
- *     TBI0    Top Byte Ignored for TTBR0 region
- *     AS              ASID Size
- *     IPS             Physical Address Size limit
- *     TG1             Granule Size for TTBR1 region
- *     SH1             Shareability for TTBR1 region
- *  ORGN1      Outer Cacheability for TTBR1 region
- *  IRGN1      Inner Cacheability for TTBR1 region
- *     EPD1    Translation table walk disable for TTBR1
- *     A1              ASID selection from TTBR1 enable
- *     T1SZ    Virtual address size for TTBR1
- *     TG0             Granule Size for TTBR0 region
- *     SH0             Shareability for TTBR0 region
- *  ORGN0      Outer Cacheability for TTBR0 region
- *  IRGN0      Inner Cacheability for TTBR0 region
- *     T0SZ    Virtual address size for TTBR0
+ * TBI1:  Top Byte Ignored for TTBR1 region
+ * TBI0:  Top Byte Ignored for TTBR0 region
+ * AS:    ASID Size
+ * IPS:   Physical Address Size limit
+ * TG1:   Granule Size for TTBR1 region
+ * SH1:   Shareability for TTBR1 region
+ * ORGN1: Outer Cacheability for TTBR1 region
+ * IRGN1: Inner Cacheability for TTBR1 region
+ * EPD1:  Translation table walk disable for TTBR1
+ * A1:    ASID selection from TTBR1 enable
+ * T1SZ:  Virtual address size for TTBR1
+ * TG0:   Granule Size for TTBR0 region
+ * SH0:   Shareability for TTBR0 region
+ * ORGN0: Outer Cacheability for TTBR0 region
+ * IRGN0: Inner Cacheability for TTBR0 region
+ * T0SZ:  Virtual address size for TTBR0

  */
 
  */
 

-#define TCR_T0SZ_SHIFT                          0ULL
-#define TCR_TSZ_BITS                            6ULL
-#define TCR_TSZ_MASK                            ((1ULL << TCR_TSZ_BITS) - 1ULL)
+#define TCR_T0SZ_SHIFT          0ULL
+#define TCR_TSZ_BITS            6ULL
+#define TCR_TSZ_MASK            ((1ULL << TCR_TSZ_BITS) - 1ULL)

 
 

-#define TCR_IRGN0_SHIFT                         8ULL
-#define TCR_IRGN0_DISABLED                      (0ULL << TCR_IRGN0_SHIFT)
-#define TCR_IRGN0_WRITEBACK                     (1ULL << TCR_IRGN0_SHIFT)
-#define TCR_IRGN0_WRITETHRU                     (2ULL << TCR_IRGN0_SHIFT)
-#define TCR_IRGN0_WRITEBACKNO           (3ULL << TCR_IRGN0_SHIFT)
+#define TCR_IRGN0_SHIFT         8ULL
+#define TCR_IRGN0_DISABLED      (0ULL << TCR_IRGN0_SHIFT)
+#define TCR_IRGN0_WRITEBACK     (1ULL << TCR_IRGN0_SHIFT)
+#define TCR_IRGN0_WRITETHRU     (2ULL << TCR_IRGN0_SHIFT)
+#define TCR_IRGN0_WRITEBACKNO   (3ULL << TCR_IRGN0_SHIFT)

 
 

-#define TCR_ORGN0_SHIFT                         10ULL
-#define TCR_ORGN0_DISABLED                      (0ULL << TCR_ORGN0_SHIFT)
-#define TCR_ORGN0_WRITEBACK                     (1ULL << TCR_ORGN0_SHIFT)
-#define TCR_ORGN0_WRITETHRU                     (2ULL << TCR_ORGN0_SHIFT)
-#define TCR_ORGN0_WRITEBACKNO           (3ULL << TCR_ORGN0_SHIFT)
+#define TCR_ORGN0_SHIFT         10ULL
+#define TCR_ORGN0_DISABLED      (0ULL << TCR_ORGN0_SHIFT)
+#define TCR_ORGN0_WRITEBACK     (1ULL << TCR_ORGN0_SHIFT)
+#define TCR_ORGN0_WRITETHRU     (2ULL << TCR_ORGN0_SHIFT)
+#define TCR_ORGN0_WRITEBACKNO   (3ULL << TCR_ORGN0_SHIFT)

 
 

-#define TCR_SH0_SHIFT                           12ULL
-#define TCR_SH0_NONE                            (0ULL << TCR_SH0_SHIFT)
-#define TCR_SH0_OUTER                           (2ULL << TCR_SH0_SHIFT)
-#define TCR_SH0_INNER                           (3ULL << TCR_SH0_SHIFT)
+#define TCR_SH0_SHIFT           12ULL
+#define TCR_SH0_NONE            (0ULL << TCR_SH0_SHIFT)
+#define TCR_SH0_OUTER           (2ULL << TCR_SH0_SHIFT)
+#define TCR_SH0_INNER           (3ULL << TCR_SH0_SHIFT)

 
 

-#define TCR_TG0_GRANULE_SHIFT           (14ULL)
+#define TCR_TG0_GRANULE_SHIFT   (14ULL)

 
 

-#define TCR_TG0_GRANULE_4KB                     (0ULL << TCR_TG0_GRANULE_SHIFT)
-#define TCR_TG0_GRANULE_64KB            (1ULL << TCR_TG0_GRANULE_SHIFT)
-#define TCR_TG0_GRANULE_16KB            (2ULL << TCR_TG0_GRANULE_SHIFT)
+#define TCR_TG0_GRANULE_4KB     (0ULL << TCR_TG0_GRANULE_SHIFT)
+#define TCR_TG0_GRANULE_64KB    (1ULL << TCR_TG0_GRANULE_SHIFT)
+#define TCR_TG0_GRANULE_16KB    (2ULL << TCR_TG0_GRANULE_SHIFT)

 
 #if __ARM_16K_PG__
 
 #if __ARM_16K_PG__

-#define TCR_TG0_GRANULE_SIZE            (TCR_TG0_GRANULE_16KB)
+#define TCR_TG0_GRANULE_SIZE    (TCR_TG0_GRANULE_16KB)

 #else
 #else

-#define TCR_TG0_GRANULE_SIZE            (TCR_TG0_GRANULE_4KB)
+#define TCR_TG0_GRANULE_SIZE    (TCR_TG0_GRANULE_4KB)

 #endif
 
 #endif
 

-#define TCR_T1SZ_SHIFT                          16ULL
+#define TCR_T1SZ_SHIFT          16ULL

 
 

-#define TCR_A1_ASID1                            (1ULL << 22ULL)
-#define TCR_EPD1_TTBR1_DISABLED         (1ULL << 23ULL)
+#define TCR_A1_ASID1            (1ULL << 22ULL)
+#define TCR_EPD1_TTBR1_DISABLED (1ULL << 23ULL)

 
 

-#define TCR_IRGN1_SHIFT                         24ULL
-#define TCR_IRGN1_DISABLED                      (0ULL << TCR_IRGN1_SHIFT)
-#define TCR_IRGN1_WRITEBACK                     (1ULL << TCR_IRGN1_SHIFT)
-#define TCR_IRGN1_WRITETHRU                     (2ULL << TCR_IRGN1_SHIFT)
-#define TCR_IRGN1_WRITEBACKNO           (3ULL << TCR_IRGN1_SHIFT)
+#define TCR_IRGN1_SHIFT          24ULL
+#define TCR_IRGN1_DISABLED       (0ULL << TCR_IRGN1_SHIFT)
+#define TCR_IRGN1_WRITEBACK      (1ULL << TCR_IRGN1_SHIFT)
+#define TCR_IRGN1_WRITETHRU      (2ULL << TCR_IRGN1_SHIFT)
+#define TCR_IRGN1_WRITEBACKNO    (3ULL << TCR_IRGN1_SHIFT)

 
 

-#define TCR_ORGN1_SHIFT                         26ULL
-#define TCR_ORGN1_DISABLED                      (0ULL << TCR_ORGN1_SHIFT)
-#define TCR_ORGN1_WRITEBACK                     (1ULL << TCR_ORGN1_SHIFT)
-#define TCR_ORGN1_WRITETHRU                     (2ULL << TCR_ORGN1_SHIFT)
-#define TCR_ORGN1_WRITEBACKNO           (3ULL << TCR_ORGN1_SHIFT)
+#define TCR_ORGN1_SHIFT          26ULL
+#define TCR_ORGN1_DISABLED       (0ULL << TCR_ORGN1_SHIFT)
+#define TCR_ORGN1_WRITEBACK      (1ULL << TCR_ORGN1_SHIFT)
+#define TCR_ORGN1_WRITETHRU      (2ULL << TCR_ORGN1_SHIFT)
+#define TCR_ORGN1_WRITEBACKNO    (3ULL << TCR_ORGN1_SHIFT)

 
 

-#define TCR_SH1_SHIFT                           28ULL
-#define TCR_SH1_NONE                            (0ULL << TCR_SH1_SHIFT)
-#define TCR_SH1_OUTER                           (2ULL << TCR_SH1_SHIFT)
-#define TCR_SH1_INNER                           (3ULL << TCR_SH1_SHIFT)
+#define TCR_SH1_SHIFT            28ULL
+#define TCR_SH1_NONE             (0ULL << TCR_SH1_SHIFT)
+#define TCR_SH1_OUTER            (2ULL << TCR_SH1_SHIFT)
+#define TCR_SH1_INNER            (3ULL << TCR_SH1_SHIFT)

 
 

-#define TCR_TG1_GRANULE_SHIFT           30ULL
+#define TCR_TG1_GRANULE_SHIFT    30ULL

 
 

-#define TCR_TG1_GRANULE_16KB            (1ULL << TCR_TG1_GRANULE_SHIFT)
-#define TCR_TG1_GRANULE_4KB                     (2ULL << TCR_TG1_GRANULE_SHIFT)
-#define TCR_TG1_GRANULE_64KB            (3ULL << TCR_TG1_GRANULE_SHIFT)
+#define TCR_TG1_GRANULE_16KB     (1ULL << TCR_TG1_GRANULE_SHIFT)
+#define TCR_TG1_GRANULE_4KB      (2ULL << TCR_TG1_GRANULE_SHIFT)
+#define TCR_TG1_GRANULE_64KB     (3ULL << TCR_TG1_GRANULE_SHIFT)

 
 #if __ARM_16K_PG__
 
 #if __ARM_16K_PG__

-#define TCR_TG1_GRANULE_SIZE            (TCR_TG1_GRANULE_16KB)
+#define TCR_TG1_GRANULE_SIZE     (TCR_TG1_GRANULE_16KB)

 #else
 #else

-#define TCR_TG1_GRANULE_SIZE            (TCR_TG1_GRANULE_4KB)
+#define TCR_TG1_GRANULE_SIZE     (TCR_TG1_GRANULE_4KB)

 #endif
 
 #endif
 

-#define TCR_IPS_SHIFT                           32ULL
-#define TCR_IPS_32BITS                          (0ULL << TCR_IPS_SHIFT)
-#define TCR_IPS_36BITS                          (1ULL << TCR_IPS_SHIFT)
-#define TCR_IPS_40BITS                          (2ULL << TCR_IPS_SHIFT)
-#define TCR_IPS_42BITS                          (3ULL << TCR_IPS_SHIFT)
-#define TCR_IPS_44BITS                          (4ULL << TCR_IPS_SHIFT)
-#define TCR_IPS_48BITS                          (5ULL << TCR_IPS_SHIFT)
-
-#define TCR_AS_16BIT_ASID                       (1ULL << 36)
-#define TCR_TBI0_TOPBYTE_IGNORED        (1ULL << 37)
-#define TCR_TBI1_TOPBYTE_IGNORED        (1ULL << 38)
+#define TCR_IPS_SHIFT            32ULL
+#define TCR_IPS_32BITS           (0ULL << TCR_IPS_SHIFT)
+#define TCR_IPS_36BITS           (1ULL << TCR_IPS_SHIFT)
+#define TCR_IPS_40BITS           (2ULL << TCR_IPS_SHIFT)
+#define TCR_IPS_42BITS           (3ULL << TCR_IPS_SHIFT)
+#define TCR_IPS_44BITS           (4ULL << TCR_IPS_SHIFT)
+#define TCR_IPS_48BITS           (5ULL << TCR_IPS_SHIFT)
+
+#define TCR_AS_16BIT_ASID        (1ULL << 36)
+#define TCR_TBI0_TOPBYTE_IGNORED (1ULL << 37)
+#define TCR_TBI1_TOPBYTE_IGNORED (1ULL << 38)
+#define TCR_TBID0_TBI_DATA_ONLY  (1ULL << 51)
+#define TCR_TBID1_TBI_DATA_ONLY  (1ULL << 52)
+
+#if defined(HAS_APPLE_PAC)
+#define TCR_TBID0_ENABLE         TCR_TBID0_TBI_DATA_ONLY
+#else
+#define TCR_TBID0_ENABLE         0
+#endif

 
 /*
  * Multiprocessor Affinity Register (MPIDR_EL1)
 
 /*
  * Multiprocessor Affinity Register (MPIDR_EL1)


@@ -503,27 +530,21 @@
  * +---------------------------------+--+-----+--+-----+----+----+
  *
  * where
  * +---------------------------------+--+-----+--+-----+----+----+
  *
  * where

- *     U               Uniprocessor
- *     MT              Multi-threading at lowest affinity level
- *     Aff2    "1" - PCORE, "0" - ECORE
- *     Aff1    Cluster ID
- *     Aff0    CPU ID
+ *   U:    Uniprocessor
+ *   MT:   Multi-threading at lowest affinity level
+ *   Aff2: "1" - PCORE, "0" - ECORE
+ *   Aff1: Cluster ID
+ *   Aff0: CPU ID

  */
  */

-#define MPIDR_AFF0_MASK                         0xFF
-#define MPIDR_AFF1_MASK                         0xFF00
-#define MPIDR_AFF1_SHIFT                        8
-#define MPIDR_AFF2_MASK                         0xFF0000
-#define MPIDR_AFF2_SHIFT                        16
-
-/*
- * We currently use a 3 level page table (rather than the full 4
- * level page table).  As a result, we do not have the full 48-bits
- * of address space per TTBR (although the 16KB granule size lets us
- * get very close).
- */
-#if __ARM64_TWO_LEVEL_PMAP__ && !__ARM_16K_PG__
-#error ARM64 does not currently support a 2 level page table with 4KB pages
-#endif /* __ARM64_TWO_LEVEL_PMAP__ */
+#define MPIDR_AFF0_SHIFT 0
+#define MPIDR_AFF0_WIDTH 8
+#define MPIDR_AFF0_MASK  (((1 << MPIDR_AFF0_WIDTH) - 1) << MPIDR_AFF0_SHIFT)
+#define MPIDR_AFF1_SHIFT 8
+#define MPIDR_AFF1_WIDTH 8
+#define MPIDR_AFF1_MASK  (((1 << MPIDR_AFF1_WIDTH) - 1) << MPIDR_AFF1_SHIFT)
+#define MPIDR_AFF2_SHIFT 16
+#define MPIDR_AFF2_WIDTH 8
+#define MPIDR_AFF2_MASK  (((1 << MPIDR_AFF2_WIDTH) - 1) << MPIDR_AFF2_SHIFT)

 
 /*
  * TXSZ indicates the size of the range a TTBR covers.  Currently,
 
 /*
  * TXSZ indicates the size of the range a TTBR covers.  Currently,


@@ -544,56 +565,60 @@
  */
 #endif /* __ARM_KERNEL_PROTECT__ */
 #ifdef __ARM_16K_PG__
  */
 #endif /* __ARM_KERNEL_PROTECT__ */
 #ifdef __ARM_16K_PG__

-#if __ARM64_TWO_LEVEL_PMAP__
-#define T0SZ_BOOT                                               28ULL
-#elif __ARM64_PMAP_SUBPAGE_L1__
-#define T0SZ_BOOT                                               25ULL
-#else /* __ARM64_TWO_LEVEL_PMAP__ */
-#define T0SZ_BOOT                                               17ULL
-#endif /* __ARM64_TWO_LEVEL_PMAP__ */
+#if __ARM64_PMAP_SUBPAGE_L1__
+#define T0SZ_BOOT 25ULL
+#else /* !__ARM64_PMAP_SUBPAGE_L1__ */
+#define T0SZ_BOOT 17ULL
+#endif /* !__ARM64_PMAP_SUBPAGE_L1__ */

 #else /* __ARM_16K_PG__ */
 #if __ARM64_PMAP_SUBPAGE_L1__
 #else /* __ARM_16K_PG__ */
 #if __ARM64_PMAP_SUBPAGE_L1__

-#define T0SZ_BOOT                                               26ULL
+#define T0SZ_BOOT 26ULL

 #else /* __ARM64_PMAP_SUBPAGE_L1__ */
 #else /* __ARM64_PMAP_SUBPAGE_L1__ */

-#define T0SZ_BOOT                                               25ULL
+#define T0SZ_BOOT 25ULL

 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */
 #endif /* __ARM_16K_PG__ */
 
 #if defined(APPLE_ARM64_ARCH_FAMILY)
 /* T0SZ must be the same as T1SZ */
 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */
 #endif /* __ARM_16K_PG__ */
 
 #if defined(APPLE_ARM64_ARCH_FAMILY)
 /* T0SZ must be the same as T1SZ */

-#define T1SZ_BOOT                                               T0SZ_BOOT
+#define T1SZ_BOOT T0SZ_BOOT

 #else /* defined(APPLE_ARM64_ARCH_FAMILY) */
 #ifdef __ARM_16K_PG__
 #else /* defined(APPLE_ARM64_ARCH_FAMILY) */
 #ifdef __ARM_16K_PG__

-#if __ARM64_TWO_LEVEL_PMAP__
-#define T1SZ_BOOT                                               28ULL
-#elif __ARM64_PMAP_SUBPAGE_L1__
-#define T1SZ_BOOT                                               25ULL
-#else /* __ARM64_TWO_LEVEL_PMAP__ */
-#define T1SZ_BOOT                                               17ULL
-#endif /* __ARM64_TWO_LEVEL_PMAP__ */
+#if __ARM64_PMAP_SUBPAGE_L1__
+#define T1SZ_BOOT 25ULL
+#else /* !__ARM64_PMAP_SUBPAGE_L1__ */
+#define T1SZ_BOOT 17ULL
+#endif /* !__ARM64_PMAP_SUBPAGE_L1__ */

 #else /* __ARM_16K_PG__ */
 #if __ARM64_PMAP_SUBPAGE_L1__
 #else /* __ARM_16K_PG__ */
 #if __ARM64_PMAP_SUBPAGE_L1__

-#define T1SZ_BOOT                                               26ULL
+#define T1SZ_BOOT 26ULL

 #else /* __ARM64_PMAP_SUBPAGE_L1__ */
 #else /* __ARM64_PMAP_SUBPAGE_L1__ */

-#define T1SZ_BOOT                                               25ULL
+#define T1SZ_BOOT 25ULL

 #endif /*__ARM64_PMAP_SUBPAGE_L1__*/
 #endif /* __ARM_16K_PG__ */
 #endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
 
 #endif /*__ARM64_PMAP_SUBPAGE_L1__*/
 #endif /* __ARM_16K_PG__ */
 #endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
 

-#define TCR_EL1_BASE    (TCR_IPS_40BITS | \
-                                                TCR_SH0_OUTER | TCR_ORGN0_WRITEBACK |  TCR_IRGN0_WRITEBACK | (T0SZ_BOOT << TCR_T0SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE) |\
-                                                TCR_SH1_OUTER | TCR_ORGN1_WRITEBACK |  TCR_IRGN1_WRITEBACK | (TCR_TG1_GRANULE_SIZE))
+#if __ARM_42BIT_PA_SPACE__
+#define TCR_IPS_VALUE TCR_IPS_42BITS
+#else /* !__ARM_42BIT_PA_SPACE__ */
+#define TCR_IPS_VALUE TCR_IPS_40BITS
+#endif /* !__ARM_42BIT_PA_SPACE__ */
+
+#define TCR_EL1_BASE \
+       (TCR_IPS_VALUE | TCR_SH0_OUTER | TCR_ORGN0_WRITEBACK |         \
+        TCR_IRGN0_WRITEBACK | (T0SZ_BOOT << TCR_T0SZ_SHIFT) |          \
+        (TCR_TG0_GRANULE_SIZE) | TCR_SH1_OUTER | TCR_ORGN1_WRITEBACK | \
+        TCR_IRGN1_WRITEBACK | (TCR_TG1_GRANULE_SIZE) |                 \
+        TCR_TBI0_TOPBYTE_IGNORED | (TCR_TBID0_ENABLE))

 
 #if __ARM_KERNEL_PROTECT__
 
 #if __ARM_KERNEL_PROTECT__

-#define TCR_EL1_BOOT    (TCR_EL1_BASE | \
-                                                (T1SZ_BOOT << TCR_T1SZ_SHIFT) | TCR_TBI0_TOPBYTE_IGNORED)
-#define T1SZ_USER       (T1SZ_BOOT + 1)
-#define TCR_EL1_USER    (TCR_EL1_BASE | (T1SZ_USER << TCR_T1SZ_SHIFT) | TCR_TBI0_TOPBYTE_IGNORED)
+#define TCR_EL1_BOOT (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT))
+#define T1SZ_USER (T1SZ_BOOT + 1)
+#define TCR_EL1_USER (TCR_EL1_BASE | (T1SZ_USER << TCR_T1SZ_SHIFT))

 #else
 #else

-#define TCR_EL1_BOOT    (TCR_EL1_BASE | \
-                                                (T1SZ_BOOT << TCR_T1SZ_SHIFT))
+#define TCR_EL1_BOOT (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT))

 #endif /* __ARM_KERNEL_PROTECT__ */
 
 #endif /* __ARM_KERNEL_PROTECT__ */
 

+

 /*
  * Translation Table Base Register (TTBR)
  *
 /*
  * Translation Table Base Register (TTBR)
  *


@@ -603,10 +628,10 @@
  * +--------+------------------+------+
  *
  */
  * +--------+------------------+------+
  *
  */

-#define TTBR_ASID_SHIFT                 48
-#define TTBR_ASID_MASK                  0xffff000000000000
+#define TTBR_ASID_SHIFT 48
+#define TTBR_ASID_MASK  0xffff000000000000

 
 

-#define TTBR_BADDR_MASK                 0x0000ffffffffffff
+#define TTBR_BADDR_MASK 0x0000ffffffffffff

 
 /*
  * Memory Attribute Indirection Register
 
 /*
  * Memory Attribute Indirection Register


@@ -618,85 +643,93 @@
  *
  */
 
  *
  */
 

-#define MAIR_ATTR_SHIFT(x)                              (8*(x))
+#define MAIR_ATTR_SHIFT(x)          (8*(x))

 
 /* Strongly ordered or device memory attributes */
 
 /* Strongly ordered or device memory attributes */

-#define MAIR_OUTER_STRONGLY_ORDERED             0x0
-#define MAIR_OUTER_DEVICE                               0x0
+#define MAIR_OUTER_STRONGLY_ORDERED 0x0
+#define MAIR_OUTER_DEVICE           0x0

 
 

-#define MAIR_INNER_STRONGLY_ORDERED             0x0
-#define MAIR_INNER_DEVICE                               0x4
+#define MAIR_INNER_STRONGLY_ORDERED 0x0
+#define MAIR_INNER_DEVICE           0x4

 
 /* Normal memory attributes */
 
 /* Normal memory attributes */

-#define MAIR_OUTER_NON_CACHEABLE                0x40
-#define MAIR_OUTER_WRITE_THROUGH                0x80
-#define MAIR_OUTER_WRITE_BACK                   0xc0
+#define MAIR_OUTER_NON_CACHEABLE    0x40
+#define MAIR_OUTER_WRITE_THROUGH    0x80
+#define MAIR_OUTER_WRITE_BACK       0xc0

 
 

-#define MAIR_INNER_NON_CACHEABLE                0x4
-#define MAIR_INNER_WRITE_THROUGH                0x8
-#define MAIR_INNER_WRITE_BACK                   0xc
+#define MAIR_INNER_NON_CACHEABLE    0x4
+#define MAIR_INNER_WRITE_THROUGH    0x8
+#define MAIR_INNER_WRITE_BACK       0xc

 
 /* Allocate policy for cacheable memory */
 
 /* Allocate policy for cacheable memory */

-#define MAIR_OUTER_WRITE_ALLOCATE               0x10
-#define MAIR_OUTER_READ_ALLOCATE                0x20
+#define MAIR_OUTER_WRITE_ALLOCATE   0x10
+#define MAIR_OUTER_READ_ALLOCATE    0x20

 
 

-#define MAIR_INNER_WRITE_ALLOCATE               0x1
-#define MAIR_INNER_READ_ALLOCATE                0x2
+#define MAIR_INNER_WRITE_ALLOCATE   0x1
+#define MAIR_INNER_READ_ALLOCATE    0x2

 
 /* Memory Atribute Encoding */
 
 
 /* Memory Atribute Encoding */
 

-/* Device memory types:
- *  G (gathering): multiple reads/writes can be combined
- *  R (reordering): reads or writes may reach device out of program order
- *  E (early-acknowledge): writes may return immediately (e.g. PCIe posted writes)
+/*
+ * Device memory types:
+ * G (gathering): multiple reads/writes can be combined
+ * R (reordering): reads or writes may reach device out of program order
+ * E (early-acknowledge): writes may return immediately (e.g. PCIe posted writes)

  */
  */

-#define MAIR_DISABLE            0x00            /* Device Memory, nGnRnE (strongly ordered) */
-#define MAIR_POSTED             0x04            /* Device Memory, nGnRE (strongly ordered, posted writes) */
-#define MAIR_WRITECOMB          0x44            /* Normal Memory, Outer Non-Cacheable, Inner Non-Cacheable */
-#define MAIR_WRITETHRU          0xBB            /* Normal Memory, Outer Write-through, Inner Write-through */
-#define MAIR_WRITEBACK          0xFF            /* Normal Memory, Outer Write-back, Inner Write-back */
-#define MAIR_INNERWRITEBACK     0x4F            /* Normal Memory, Outer Non-Cacheable, Inner Write-back */
+#define MAIR_DISABLE                   0x00 /* Device Memory, nGnRnE (strongly ordered) */
+#define MAIR_POSTED                    0x04 /* Device Memory, nGnRE (strongly ordered, posted writes) */
+#define MAIR_POSTED_REORDERED          0x08 /* Device Memory, nGRE (reorderable, posted writes) */
+#define MAIR_POSTED_COMBINED_REORDERED 0x0C /* Device Memory, GRE (reorderable, gathered writes, posted writes) */
+#define MAIR_WRITECOMB                 0x44 /* Normal Memory, Outer Non-Cacheable, Inner Non-Cacheable */
+#define MAIR_WRITETHRU                 0xBB /* Normal Memory, Outer Write-through, Inner Write-through */
+#define MAIR_WRITEBACK                 0xFF /* Normal Memory, Outer Write-back, Inner Write-back */
+#define MAIR_INNERWRITEBACK            0x4F /* Normal Memory, Outer Non-Cacheable, Inner Write-back */

 
 
 /*
 
 
 /*

- *     ARM 4-level Page Table support - 2*1024TB (2^48) of address space
+ * ARM 4-level Page Table support - 2*1024TB (2^48) of address space

  */
 
 
 /*
  *  Memory Attribute Index
  */
  */
 
 
 /*
  *  Memory Attribute Index
  */

-#define CACHE_ATTRINDX_WRITEBACK                0x0     /* cache enabled, buffer enabled */
-#define CACHE_ATTRINDX_WRITECOMB                0x1     /* no cache, buffered writes */
-#define CACHE_ATTRINDX_WRITETHRU                0x2     /* cache enabled, buffer disabled */
-#define CACHE_ATTRINDX_DISABLE                  0x3     /* no cache, no buffer */
-#define CACHE_ATTRINDX_INNERWRITEBACK           0x4     /* inner cache enabled, buffer enabled, write allocate */
-#define CACHE_ATTRINDX_POSTED                   0x5     /* no cache, no buffer, posted writes */
-#define CACHE_ATTRINDX_DEFAULT                  CACHE_ATTRINDX_WRITEBACK
+#define CACHE_ATTRINDX_WRITEBACK                 0x0 /* cache enabled, buffer enabled  (normal memory) */
+#define CACHE_ATTRINDX_WRITECOMB                 0x1 /* no cache, buffered writes (normal memory) */
+#define CACHE_ATTRINDX_WRITETHRU                 0x2 /* cache enabled, buffer disabled (normal memory) */
+#define CACHE_ATTRINDX_DISABLE                   0x3 /* no cache, no buffer (device memory) */
+#define CACHE_ATTRINDX_INNERWRITEBACK            0x4 /* inner cache enabled, buffer enabled, write allocate (normal memory) */
+#define CACHE_ATTRINDX_POSTED                    0x5 /* no cache, no buffer, posted writes (device memory) */
+#define CACHE_ATTRINDX_POSTED_REORDERED          0x6 /* no cache, reorderable access, posted writes (device memory) */
+#define CACHE_ATTRINDX_POSTED_COMBINED_REORDERED 0x7 /* no cache, write gathering, reorderable access, posted writes (device memory) */
+#define CACHE_ATTRINDX_DEFAULT                   CACHE_ATTRINDX_WRITEBACK
+

 
 /*
 
 /*

- *      Access protection bit values (TTEs and PTEs)
+ * Access protection bit values (TTEs and PTEs), stage 1
+ *
+ * Bit 1 controls access type (1=RO, 0=RW), bit 0 controls user (1=access, 0=no access)

  */
  */

-#define AP_RWNA                                                 0x0     /* priv=read-write, user=no-access */
-#define AP_RWRW                                                 0x1     /* priv=read-write, user=read-write */
-#define AP_RONA                                                 0x2     /* priv=read-only, user=no-access */
-#define AP_RORO                                                 0x3     /* priv=read-only, user=read-only */
-#define AP_MASK                                                 0x3     /* mask to find ap bits */
+#define AP_RWNA 0x0 /* priv=read-write, user=no-access */
+#define AP_RWRW 0x1 /* priv=read-write, user=read-write */
+#define AP_RONA 0x2 /* priv=read-only, user=no-access */
+#define AP_RORO 0x3 /* priv=read-only, user=read-only */
+#define AP_MASK 0x3 /* mask to find ap bits */

 
 /*
  * Shareability attributes
  */
 
 /*
  * Shareability attributes
  */

-#define SH_NONE                                                 0x0     /* Non shareable  */
-#define SH_NONE                                                 0x0     /* Device shareable */
-#define SH_DEVICE                                               0x2     /* Normal memory Inner non shareable - Outer non shareable */
-#define SH_OUTER_MEMORY                                 0x2 /* Normal memory Inner shareable - Outer shareable */
-#define SH_INNER_MEMORY                                 0x3 /* Normal memory Inner shareable - Outer non shareable */
+#define SH_NONE         0x0 /* Non shareable  */
+#define SH_NONE         0x0 /* Device shareable */
+#define SH_DEVICE       0x2 /* Normal memory Inner non shareable - Outer non shareable */
+#define SH_OUTER_MEMORY 0x2 /* Normal memory Inner shareable - Outer shareable */
+#define SH_INNER_MEMORY 0x3 /* Normal memory Inner shareable - Outer non shareable */

 
 
 /*
  * ARM Page Granule
  */
 
 
 /*
  * ARM Page Granule
  */

-#ifdef  __ARM_16K_PG__
+#ifdef __ARM_16K_PG__

 #define ARM_PGSHIFT 14
 #else
 #define ARM_PGSHIFT 12
 #define ARM_PGSHIFT 14
 #else
 #define ARM_PGSHIFT 12


@@ -704,7 +737,6 @@
 #define ARM_PGBYTES (1 << ARM_PGSHIFT)
 #define ARM_PGMASK  (ARM_PGBYTES-1)
 
 #define ARM_PGBYTES (1 << ARM_PGSHIFT)
 #define ARM_PGMASK  (ARM_PGBYTES-1)
 

-

 /*
  *  L0 Translation table
  *
 /*
  *  L0 Translation table
  *


@@ -719,17 +751,17 @@
  *    Covers 256TB (2^48) of address space.
  */
 
  *    Covers 256TB (2^48) of address space.
  */
 

-#ifdef __ARM_16K_PG__
-#define ARM_TT_L0_SIZE                                  0x0000800000000000ULL           /* size of area covered by a tte */
-#define ARM_TT_L0_OFFMASK                               0x00007fffffffffffULL           /* offset within an L0 entry */
-#define ARM_TT_L0_SHIFT                                 47                                                      /* page descriptor shift */
-#define ARM_TT_L0_INDEX_MASK                    0x0000800000000000ULL           /* mask for getting index in L0 table from virtual address */
-#else
-#define ARM_TT_L0_SIZE                                  0x0000008000000000ULL           /* size of area covered by a tte */
-#define ARM_TT_L0_OFFMASK                               0x0000007fffffffffULL           /* offset within an L0 entry */
-#define ARM_TT_L0_SHIFT                                 39                                                      /* page descriptor shift */
-#define ARM_TT_L0_INDEX_MASK                    0x0000ff8000000000ULL           /* mask for getting index in L0 table from virtual address */
-#endif
+/* 16K L0 */
+#define ARM_16K_TT_L0_SIZE       0x0000800000000000ULL /* size of area covered by a tte */
+#define ARM_16K_TT_L0_OFFMASK    0x00007fffffffffffULL /* offset within an L0 entry */
+#define ARM_16K_TT_L0_SHIFT      47                    /* page descriptor shift */
+#define ARM_16K_TT_L0_INDEX_MASK 0x0000800000000000ULL /* mask for getting index in L0 table from virtual address */
+
+/* 4K L0 */
+#define ARM_4K_TT_L0_SIZE       0x0000008000000000ULL /* size of area covered by a tte */
+#define ARM_4K_TT_L0_OFFMASK    0x0000007fffffffffULL /* offset within an L0 entry */
+#define ARM_4K_TT_L0_SHIFT      39                    /* page descriptor shift */
+#define ARM_4K_TT_L0_INDEX_MASK 0x0000ff8000000000ULL /* mask for getting index in L0 table from virtual address */

 
 /*
  *  L1 Translation table
 
 /*
  *  L1 Translation table


@@ -745,27 +777,27 @@
  *    Covers 128TB (2^47) of address space.
  */
 
  *    Covers 128TB (2^47) of address space.
  */
 

-#ifdef __ARM_16K_PG__
-#define ARM_TT_L1_SIZE                                  0x0000001000000000ULL           /* size of area covered by a tte */
-#define ARM_TT_L1_OFFMASK                               0x0000000fffffffffULL           /* offset within an L1 entry */
-#define ARM_TT_L1_SHIFT                                 36                                                      /* page descriptor shift */
+/* 16K L1 */
+#define ARM_16K_TT_L1_SIZE       0x0000001000000000ULL /* size of area covered by a tte */
+#define ARM_16K_TT_L1_OFFMASK    0x0000000fffffffffULL /* offset within an L1 entry */
+#define ARM_16K_TT_L1_SHIFT      36                    /* page descriptor shift */

 #ifdef __ARM64_PMAP_SUBPAGE_L1__
 /* This config supports 512GB per TTBR. */
 #ifdef __ARM64_PMAP_SUBPAGE_L1__
 /* This config supports 512GB per TTBR. */

-#define ARM_TT_L1_INDEX_MASK                    0x0000007000000000ULL           /* mask for getting index into L1 table from virtual address */
+#define ARM_16K_TT_L1_INDEX_MASK 0x0000007000000000ULL /* mask for getting index into L1 table from virtual address */

 #else /* __ARM64_PMAP_SUBPAGE_L1__ */
 #else /* __ARM64_PMAP_SUBPAGE_L1__ */

-#define ARM_TT_L1_INDEX_MASK                    0x00007ff000000000ULL           /* mask for getting index into L1 table from virtual address */
+#define ARM_16K_TT_L1_INDEX_MASK 0x00007ff000000000ULL /* mask for getting index into L1 table from virtual address */

 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */
 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */

-#else /* __ARM_16K_PG__ */
-#define ARM_TT_L1_SIZE                                  0x0000000040000000ULL           /* size of area covered by a tte */
-#define ARM_TT_L1_OFFMASK                               0x000000003fffffffULL           /* offset within an L1 entry */
-#define ARM_TT_L1_SHIFT                                 30                                                      /* page descriptor shift */
+
+/* 4K L1 */
+#define ARM_4K_TT_L1_SIZE       0x0000000040000000ULL /* size of area covered by a tte */
+#define ARM_4K_TT_L1_OFFMASK    0x000000003fffffffULL /* offset within an L1 entry */
+#define ARM_4K_TT_L1_SHIFT      30                    /* page descriptor shift */

 #ifdef __ARM64_PMAP_SUBPAGE_L1__
 /* This config supports 256GB per TTBR. */
 #ifdef __ARM64_PMAP_SUBPAGE_L1__
 /* This config supports 256GB per TTBR. */

-#define ARM_TT_L1_INDEX_MASK                    0x0000003fc0000000ULL           /* mask for getting index into L1 table from virtual address */
+#define ARM_4K_TT_L1_INDEX_MASK 0x0000003fc0000000ULL /* mask for getting index into L1 table from virtual address */

 #else /* __ARM64_PMAP_SUBPAGE_L1__ */
 #else /* __ARM64_PMAP_SUBPAGE_L1__ */

-#define ARM_TT_L1_INDEX_MASK                    0x0000007fc0000000ULL           /* mask for getting index into L1 table from virtual address */
+#define ARM_4K_TT_L1_INDEX_MASK 0x0000007fc0000000ULL /* mask for getting index into L1 table from virtual address */

 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */
 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */

-#endif

 
 /* some sugar for getting pointers to page tables and entries */
 
 
 /* some sugar for getting pointers to page tables and entries */
 


@@ -773,7 +805,7 @@
 #define L2_TABLE_INDEX(va) (((va) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT)
 #define L3_TABLE_INDEX(va) (((va) & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT)
 
 #define L2_TABLE_INDEX(va) (((va) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT)
 #define L3_TABLE_INDEX(va) (((va) & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT)
 

-#define L2_TABLE_VA(tte) ((tt_entry_t*) phystokv((*(tte)) & ARM_TTE_TABLE_MASK))
+#define L2_TABLE_VA(tte)  ((tt_entry_t*) phystokv((*(tte)) & ARM_TTE_TABLE_MASK))

 #define L3_TABLE_VA(tte2) ((pt_entry_t*) phystokv((*(tte2)) & ARM_TTE_TABLE_MASK))
 
 /*
 #define L3_TABLE_VA(tte2) ((pt_entry_t*) phystokv((*(tte2)) & ARM_TTE_TABLE_MASK))
 
 /*


@@ -790,17 +822,17 @@
  *    Covers 64GB (2^36) of address space.
  */
 
  *    Covers 64GB (2^36) of address space.
  */
 

-#ifdef __ARM_16K_PG__
-#define ARM_TT_L2_SIZE                                  0x0000000002000000ULL           /* size of area covered by a tte */
-#define ARM_TT_L2_OFFMASK                               0x0000000001ffffffULL           /* offset within an L2 entry */
-#define ARM_TT_L2_SHIFT                                 25                                                      /* page descriptor shift */
-#define ARM_TT_L2_INDEX_MASK                    0x0000000ffe000000ULL           /* mask for getting index in L2 table from virtual address */
-#else
-#define ARM_TT_L2_SIZE                                  0x0000000000200000ULL           /* size of area covered by a tte */
-#define ARM_TT_L2_OFFMASK                               0x00000000001fffffULL           /* offset within an L2 entry */
-#define ARM_TT_L2_SHIFT                                 21                                                      /* page descriptor shift */
-#define ARM_TT_L2_INDEX_MASK                    0x000000003fe00000ULL           /* mask for getting index in L2 table from virtual address */
-#endif
+/* 16K L2 */
+#define ARM_16K_TT_L2_SIZE       0x0000000002000000ULL /* size of area covered by a tte */
+#define ARM_16K_TT_L2_OFFMASK    0x0000000001ffffffULL /* offset within an L2 entry */
+#define ARM_16K_TT_L2_SHIFT      25                    /* page descriptor shift */
+#define ARM_16K_TT_L2_INDEX_MASK 0x0000000ffe000000ULL /* mask for getting index in L2 table from virtual address */
+
+/* 4K L2 */
+#define ARM_4K_TT_L2_SIZE       0x0000000000200000ULL /* size of area covered by a tte */
+#define ARM_4K_TT_L2_OFFMASK    0x00000000001fffffULL /* offset within an L2 entry */
+#define ARM_4K_TT_L2_SHIFT      21                    /* page descriptor shift */
+#define ARM_4K_TT_L2_INDEX_MASK 0x000000003fe00000ULL /* mask for getting index in L2 table from virtual address */

 
 /*
  *  L3 Translation table
 
 /*
  *  L3 Translation table


@@ -816,17 +848,71 @@
  *    Covers 32MB (2^25) of address space.
  */
 
  *    Covers 32MB (2^25) of address space.
  */
 

+/* 16K L3 */
+#define ARM_16K_TT_L3_SIZE       0x0000000000004000ULL /* size of area covered by a tte */
+#define ARM_16K_TT_L3_OFFMASK    0x0000000000003fffULL /* offset within L3 PTE */
+#define ARM_16K_TT_L3_SHIFT      14                    /* page descriptor shift */
+#define ARM_16K_TT_L3_INDEX_MASK 0x0000000001ffc000ULL /* mask for page descriptor index */
+
+/* 4K L3 */
+#define ARM_4K_TT_L3_SIZE       0x0000000000001000ULL /* size of area covered by a tte */
+#define ARM_4K_TT_L3_OFFMASK    0x0000000000000fffULL /* offset within L3 PTE */
+#define ARM_4K_TT_L3_SHIFT      12                    /* page descriptor shift */
+#define ARM_4K_TT_L3_INDEX_MASK 0x00000000001ff000ULL /* mask for page descriptor index */
+

 #ifdef __ARM_16K_PG__
 #ifdef __ARM_16K_PG__

-#define ARM_TT_L3_SIZE                                  0x0000000000004000ULL           /* size of area covered by a tte */
-#define ARM_TT_L3_OFFMASK                               0x0000000000003fffULL           /* offset within L3 PTE */
-#define ARM_TT_L3_SHIFT                                 14                                                      /* page descriptor shift */
-#define ARM_TT_L3_INDEX_MASK                    0x0000000001ffc000ULL           /* mask for page descriptor index */
-#else
-#define ARM_TT_L3_SIZE                                  0x0000000000001000ULL           /* size of area covered by a tte */
-#define ARM_TT_L3_OFFMASK                               0x0000000000000fffULL           /* offset within L3 PTE */
-#define ARM_TT_L3_SHIFT                                 12                                                      /* page descriptor shift */
-#define ARM_TT_L3_INDEX_MASK                    0x00000000001ff000ULL           /* mask for page descriptor index */
-#endif
+
+/* Native L0 defines */
+#define ARM_TT_L0_SIZE       ARM_16K_TT_L0_SIZE
+#define ARM_TT_L0_OFFMASK    ARM_16K_TT_L0_OFFMASK
+#define ARM_TT_L0_SHIFT      ARM_16K_TT_L0_SHIFT
+#define ARM_TT_L0_INDEX_MASK ARM_16K_TT_L0_INDEX_MASK
+
+/* Native L1 defines */
+#define ARM_TT_L1_SIZE       ARM_16K_TT_L1_SIZE
+#define ARM_TT_L1_OFFMASK    ARM_16K_TT_L1_OFFMASK
+#define ARM_TT_L1_SHIFT      ARM_16K_TT_L1_SHIFT
+#define ARM_TT_L1_INDEX_MASK ARM_16K_TT_L1_INDEX_MASK
+
+/* Native L2 defines */
+#define ARM_TT_L2_SIZE       ARM_16K_TT_L2_SIZE
+#define ARM_TT_L2_OFFMASK    ARM_16K_TT_L2_OFFMASK
+#define ARM_TT_L2_SHIFT      ARM_16K_TT_L2_SHIFT
+#define ARM_TT_L2_INDEX_MASK ARM_16K_TT_L2_INDEX_MASK
+
+/* Native L3 defines */
+#define ARM_TT_L3_SIZE       ARM_16K_TT_L3_SIZE
+#define ARM_TT_L3_OFFMASK    ARM_16K_TT_L3_OFFMASK
+#define ARM_TT_L3_SHIFT      ARM_16K_TT_L3_SHIFT
+#define ARM_TT_L3_INDEX_MASK ARM_16K_TT_L3_INDEX_MASK
+
+#else /* !__ARM_16K_PG__ */
+
+/* Native L0 defines */
+#define ARM_TT_L0_SIZE       ARM_4K_TT_L0_SIZE
+#define ARM_TT_L0_OFFMASK    ARM_4K_TT_L0_OFFMASK
+#define ARM_TT_L0_SHIFT      ARM_4K_TT_L0_SHIFT
+#define ARM_TT_L0_INDEX_MASK ARM_4K_TT_L0_INDEX_MASK
+
+/* Native L1 defines */
+#define ARM_TT_L1_SIZE       ARM_4K_TT_L1_SIZE
+#define ARM_TT_L1_OFFMASK    ARM_4K_TT_L1_OFFMASK
+#define ARM_TT_L1_SHIFT      ARM_4K_TT_L1_SHIFT
+#define ARM_TT_L1_INDEX_MASK ARM_4K_TT_L1_INDEX_MASK
+
+/* Native L2 defines */
+#define ARM_TT_L2_SIZE       ARM_4K_TT_L2_SIZE
+#define ARM_TT_L2_OFFMASK    ARM_4K_TT_L2_OFFMASK
+#define ARM_TT_L2_SHIFT      ARM_4K_TT_L2_SHIFT
+#define ARM_TT_L2_INDEX_MASK ARM_4K_TT_L2_INDEX_MASK
+
+/* Native L3 defines */
+#define ARM_TT_L3_SIZE       ARM_4K_TT_L3_SIZE
+#define ARM_TT_L3_OFFMASK    ARM_4K_TT_L3_OFFMASK
+#define ARM_TT_L3_SHIFT      ARM_4K_TT_L3_SHIFT
+#define ARM_TT_L3_INDEX_MASK ARM_4K_TT_L3_INDEX_MASK
+
+#endif /* !__ARM_16K_PG__ */

 
 /*
  * Convenience definitions for:
 
 /*
  * Convenience definitions for:


@@ -836,27 +922,20 @@
  *
  *   My apologies to any botanists who may be reading this.
  */
  *
  *   My apologies to any botanists who may be reading this.
  */

-#define ARM_TT_LEAF_SIZE                                ARM_TT_L3_SIZE
-#define ARM_TT_LEAF_OFFMASK                             ARM_TT_L3_OFFMASK
-#define ARM_TT_LEAF_SHIFT                               ARM_TT_L3_SHIFT
-#define ARM_TT_LEAF_INDEX_MASK                  ARM_TT_L3_INDEX_MASK
-
-#define ARM_TT_TWIG_SIZE                                ARM_TT_L2_SIZE
-#define ARM_TT_TWIG_OFFMASK                             ARM_TT_L2_OFFMASK
-#define ARM_TT_TWIG_SHIFT                               ARM_TT_L2_SHIFT
-#define ARM_TT_TWIG_INDEX_MASK                  ARM_TT_L2_INDEX_MASK
-
-#if __ARM64_TWO_LEVEL_PMAP__
-#define ARM_TT_ROOT_SIZE                                ARM_TT_L2_SIZE
-#define ARM_TT_ROOT_OFFMASK                             ARM_TT_L2_OFFMASK
-#define ARM_TT_ROOT_SHIFT                               ARM_TT_L2_SHIFT
-#define ARM_TT_ROOT_INDEX_MASK                  ARM_TT_L2_INDEX_MASK
-#else
-#define ARM_TT_ROOT_SIZE                                ARM_TT_L1_SIZE
-#define ARM_TT_ROOT_OFFMASK                             ARM_TT_L1_OFFMASK
-#define ARM_TT_ROOT_SHIFT                               ARM_TT_L1_SHIFT
-#define ARM_TT_ROOT_INDEX_MASK                  ARM_TT_L1_INDEX_MASK
-#endif
+#define ARM_TT_LEAF_SIZE       ARM_TT_L3_SIZE
+#define ARM_TT_LEAF_OFFMASK    ARM_TT_L3_OFFMASK
+#define ARM_TT_LEAF_SHIFT      ARM_TT_L3_SHIFT
+#define ARM_TT_LEAF_INDEX_MASK ARM_TT_L3_INDEX_MASK
+
+#define ARM_TT_TWIG_SIZE       ARM_TT_L2_SIZE
+#define ARM_TT_TWIG_OFFMASK    ARM_TT_L2_OFFMASK
+#define ARM_TT_TWIG_SHIFT      ARM_TT_L2_SHIFT
+#define ARM_TT_TWIG_INDEX_MASK ARM_TT_L2_INDEX_MASK
+
+#define ARM_TT_ROOT_SIZE       ARM_TT_L1_SIZE
+#define ARM_TT_ROOT_OFFMASK    ARM_TT_L1_OFFMASK
+#define ARM_TT_ROOT_SHIFT      ARM_TT_L1_SHIFT
+#define ARM_TT_ROOT_INDEX_MASK ARM_TT_L1_INDEX_MASK

 
 /*
  * 4KB granule size:
 
 /*
  * 4KB granule size:


@@ -927,116 +1006,121 @@
  * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
  *
  * where:
  * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
  *
  * where:

- *     'nG'            notGlobal bit
- *     'SH'            Shareability field
- *     'AP'            access protection
- *     'XN'            eXecute Never bit
- *     'PXN'           Privilege eXecute Never bit
- *     'NS'            Non-Secure bit
- *     'HINT'          16 entry continuguous output hint
- *     'AttrIdx'       Memory Attribute Index
+ *   nG:      notGlobal bit
+ *   SH:      Shareability field
+ *   AP:      access protection
+ *   XN:      eXecute Never bit
+ *   PXN:     Privilege eXecute Never bit
+ *   NS:      Non-Secure bit
+ *   HINT:    16 entry continuguous output hint
+ *   AttrIdx: Memory Attribute Index

  */
 
  */
 

-#define TTE_SHIFT                                       3                                               /* shift width of a tte (sizeof(tte) == (1 << TTE_SHIFT)) */
+#define TTE_SHIFT                   3                              /* shift width of a tte (sizeof(tte) == (1 << TTE_SHIFT)) */

 #ifdef __ARM_16K_PG__
 #ifdef __ARM_16K_PG__

-#define TTE_PGENTRIES                           (16384 >> TTE_SHIFT)    /* number of ttes per page */
+#define TTE_PGENTRIES               (16384 >> TTE_SHIFT)           /* number of ttes per page */

 #else
 #else

-#define TTE_PGENTRIES                           (4096 >> TTE_SHIFT)             /* number of ttes per page */
+#define TTE_PGENTRIES               (4096 >> TTE_SHIFT)            /* number of ttes per page */

 #endif
 
 #endif
 

-#define ARM_TTE_MAX                                     (TTE_PGENTRIES)
+#define ARM_TTE_MAX                 (TTE_PGENTRIES)

 
 

-#define ARM_TTE_EMPTY                           0x0000000000000000ULL   /* unasigned - invalid entry */
-#define ARM_TTE_TYPE_FAULT                      0x0000000000000000ULL   /* unasigned - invalid entry */
+#define ARM_TTE_EMPTY               0x0000000000000000ULL          /* unasigned - invalid entry */
+#define ARM_TTE_TYPE_FAULT          0x0000000000000000ULL          /* unasigned - invalid entry */

 
 

-#define ARM_TTE_VALID                           0x0000000000000001ULL   /* valid entry */
+#define ARM_TTE_VALID               0x0000000000000001ULL          /* valid entry */

 
 

-#define ARM_TTE_TYPE_MASK                       0x0000000000000002ULL   /* mask for extracting the type */
-#define ARM_TTE_TYPE_TABLE                      0x0000000000000002ULL   /* page table type */
-#define ARM_TTE_TYPE_BLOCK                      0x0000000000000000ULL   /* block entry type */
-#define ARM_TTE_TYPE_L3BLOCK            0x0000000000000002ULL
-#define ARM_TTE_TYPE_MASK                       0x0000000000000002ULL   /* mask for extracting the type */
+#define ARM_TTE_TYPE_MASK           0x0000000000000002ULL          /* mask for extracting the type */
+#define ARM_TTE_TYPE_TABLE          0x0000000000000002ULL          /* page table type */
+#define ARM_TTE_TYPE_BLOCK          0x0000000000000000ULL          /* block entry type */
+#define ARM_TTE_TYPE_L3BLOCK        0x0000000000000002ULL
+#define ARM_TTE_TYPE_MASK           0x0000000000000002ULL          /* mask for extracting the type */

 
 #ifdef __ARM_16K_PG__
 
 #ifdef __ARM_16K_PG__

-/* Note that L0/L1 block entries are disallowed for the 16KB granule size; what are we doing with these? */
-#define ARM_TTE_BLOCK_SHIFT                     12                                              /* entry shift for a 16KB L3 TTE entry */
-#define ARM_TTE_BLOCK_L0_SHIFT          ARM_TT_L0_SHIFT                 /* block shift for 128TB section */
-#define ARM_TTE_BLOCK_L1_MASK           0x0000fff000000000ULL   /* mask to extract phys address from L1 block entry */
-#define ARM_TTE_BLOCK_L1_SHIFT          ARM_TT_L1_SHIFT                 /* block shift for 64GB section */
-#define ARM_TTE_BLOCK_L2_MASK           0x0000fffffe000000ULL   /* mask to extract phys address from Level 2 Translation Block entry */
-#define ARM_TTE_BLOCK_L2_SHIFT          ARM_TT_L2_SHIFT                 /* block shift for 32MB section */
+/*
+ * Note that L0/L1 block entries are disallowed for the 16KB granule size; what
+ * are we doing with these?
+ */
+#define ARM_TTE_BLOCK_SHIFT         12                             /* entry shift for a 16KB L3 TTE entry */
+#define ARM_TTE_BLOCK_L0_SHIFT      ARM_TT_L0_SHIFT                /* block shift for 128TB section */
+#define ARM_TTE_BLOCK_L1_MASK       0x0000fff000000000ULL          /* mask to extract phys address from L1 block entry */
+#define ARM_TTE_BLOCK_L1_SHIFT      ARM_TT_L1_SHIFT                /* block shift for 64GB section */
+#define ARM_TTE_BLOCK_L2_MASK       0x0000fffffe000000ULL          /* mask to extract phys address from Level 2 Translation Block entry */
+#define ARM_TTE_BLOCK_L2_SHIFT      ARM_TT_L2_SHIFT                /* block shift for 32MB section */

 #else
 #else

-#define ARM_TTE_BLOCK_SHIFT                     12                                              /* entry shift for a 4KB L3 TTE entry */
-#define ARM_TTE_BLOCK_L0_SHIFT          ARM_TT_L0_SHIFT                 /* block shift for 2048GB section */
-#define ARM_TTE_BLOCK_L1_MASK           0x0000ffffc0000000ULL   /* mask to extract phys address from L1 block entry */
-#define ARM_TTE_BLOCK_L1_SHIFT          ARM_TT_L1_SHIFT                 /* block shift for 1GB section */
-#define ARM_TTE_BLOCK_L2_MASK           0x0000ffffffe00000ULL   /* mask to extract phys address from Level 2 Translation Block entry */
-#define ARM_TTE_BLOCK_L2_SHIFT          ARM_TT_L2_SHIFT                 /* block shift for 2MB section */
+#define ARM_TTE_BLOCK_SHIFT         12                             /* entry shift for a 4KB L3 TTE entry */
+#define ARM_TTE_BLOCK_L0_SHIFT      ARM_TT_L0_SHIFT                /* block shift for 2048GB section */
+#define ARM_TTE_BLOCK_L1_MASK       0x0000ffffc0000000ULL          /* mask to extract phys address from L1 block entry */
+#define ARM_TTE_BLOCK_L1_SHIFT      ARM_TT_L1_SHIFT                /* block shift for 1GB section */
+#define ARM_TTE_BLOCK_L2_MASK       0x0000ffffffe00000ULL          /* mask to extract phys address from Level 2 Translation Block entry */
+#define ARM_TTE_BLOCK_L2_SHIFT      ARM_TT_L2_SHIFT                /* block shift for 2MB section */

 #endif
 
 #endif
 

-#define ARM_TTE_BLOCK_APSHIFT           6
-#define ARM_TTE_BLOCK_AP(x)                     ((x)<<ARM_TTE_BLOCK_APSHIFT) /* access protection */
-#define ARM_TTE_BLOCK_APMASK            (0x3 << ARM_TTE_BLOCK_APSHIFT)
+#define ARM_TTE_BLOCK_APSHIFT       6
+#define ARM_TTE_BLOCK_AP(x)         ((x)<<ARM_TTE_BLOCK_APSHIFT)   /* access protection */
+#define ARM_TTE_BLOCK_APMASK        (0x3 << ARM_TTE_BLOCK_APSHIFT)

 
 

-#define ARM_TTE_BLOCK_ATTRINDX(x)       ((x) << 2)                              /* memory attributes index */
-#define ARM_TTE_BLOCK_ATTRINDXMASK      (0x7ULL << 2)                   /* mask memory attributes index */
+#define ARM_TTE_BLOCK_ATTRINDX(x)   ((x) << 2)                     /* memory attributes index */
+#define ARM_TTE_BLOCK_ATTRINDXMASK  (0x7ULL << 2)                  /* mask memory attributes index */

 
 

-#define ARM_TTE_BLOCK_SH(x)                     ((x) << 8)                              /* access shared */
-#define ARM_TTE_BLOCK_SHMASK            (0x3ULL << 8)                   /* mask access shared */
+#define ARM_TTE_BLOCK_SH(x)         ((x) << 8)                     /* access shared */
+#define ARM_TTE_BLOCK_SHMASK        (0x3ULL << 8)                  /* mask access shared */

 
 

-#define ARM_TTE_BLOCK_AF                        0x0000000000000400ULL   /* value for access */
-#define ARM_TTE_BLOCK_AFMASK            0x0000000000000400ULL   /* access mask */
+#define ARM_TTE_BLOCK_AF            0x0000000000000400ULL          /* value for access */
+#define ARM_TTE_BLOCK_AFMASK        0x0000000000000400ULL          /* access mask */

 
 

-#define ARM_TTE_BLOCK_NG                        0x0000000000000800ULL   /* value for a global mapping */
-#define ARM_TTE_BLOCK_NG_MASK           0x0000000000000800ULL   /* notGlobal mapping mask */
+#define ARM_TTE_BLOCK_NG            0x0000000000000800ULL          /* value for a global mapping */
+#define ARM_TTE_BLOCK_NG_MASK       0x0000000000000800ULL          /* notGlobal mapping mask */

 
 

-#define ARM_TTE_BLOCK_NS                        0x0000000000000020ULL   /* value for a secure mapping */
-#define ARM_TTE_BLOCK_NS_MASK           0x0000000000000020ULL   /* notSecure mapping mask */
+#define ARM_TTE_BLOCK_NS            0x0000000000000020ULL          /* value for a secure mapping */
+#define ARM_TTE_BLOCK_NS_MASK       0x0000000000000020ULL          /* notSecure mapping mask */

 
 

-#define ARM_TTE_BLOCK_PNX                       0x0020000000000000ULL   /* value for privilege no execute bit */
-#define ARM_TTE_BLOCK_PNXMASK           0x0020000000000000ULL   /* privilege no execute mask */
+#define ARM_TTE_BLOCK_PNX           0x0020000000000000ULL          /* value for privilege no execute bit */
+#define ARM_TTE_BLOCK_PNXMASK       0x0020000000000000ULL          /* privilege no execute mask */

 
 

-#define ARM_TTE_BLOCK_NX                        0x0040000000000000ULL   /* value for no execute */
-#define ARM_TTE_BLOCK_NXMASK            0x0040000000000000ULL   /* no execute mask */
+#define ARM_TTE_BLOCK_NX            0x0040000000000000ULL          /* value for no execute */
+#define ARM_TTE_BLOCK_NXMASK        0x0040000000000000ULL          /* no execute mask */

 
 

-#define ARM_TTE_BLOCK_WIRED                     0x0080000000000000ULL   /* value for software wired bit */
-#define ARM_TTE_BLOCK_WIREDMASK         0x0080000000000000ULL   /* software wired mask */
+#define ARM_TTE_BLOCK_WIRED         0x0400000000000000ULL          /* value for software wired bit */
+#define ARM_TTE_BLOCK_WIREDMASK     0x0400000000000000ULL          /* software wired mask */

 
 

-#define ARM_TTE_BLOCK_WRITEABLE         0x0100000000000000ULL   /* value for software writeable bit */
-#define ARM_TTE_BLOCK_WRITEABLEMASK     0x0100000000000000ULL   /* software writeable mask */
+#define ARM_TTE_BLOCK_WRITEABLE     0x0800000000000000ULL          /* value for software writeable bit */
+#define ARM_TTE_BLOCK_WRITEABLEMASK 0x0800000000000000ULL          /* software writeable mask */

 
 #ifdef __ARM_16K_PG__
 /*
 
 #ifdef __ARM_16K_PG__
 /*

- * TODO: Do we care about the low bits being unused?  It should technically work either way, but masking them out should be future proof;
- * it is only a matter of time before someone wants to shove something into the free bits.
+ * TODO: Do we care about the low bits being unused?  It should technically
+ * work either way, but masking them out should be future proof; it is only a
+ * matter of time before someone wants to shove something into the free bits.

  */
  */

-#define ARM_TTE_TABLE_MASK                      (0x0000ffffffffc000ULL) /* mask for extracting pointer to next table (works at any level) */
+#define ARM_TTE_TABLE_MASK          (0x0000ffffffffc000ULL)        /* mask for extracting pointer to next table (works at any level) */

 #else
 #else

-#define ARM_TTE_TABLE_MASK                      (0x0000fffffffff000ULL) /* mask for extracting pointer to next table (works at any level) */
+#define ARM_TTE_TABLE_MASK          (0x0000fffffffff000ULL)        /* mask for extracting pointer to next table (works at any level) */

 #endif
 
 #endif
 

-#define ARM_TTE_TABLE_APSHIFT           61
-#define ARM_TTE_TABLE_AP(x)                     ((x)<<TTE_BLOCK_APSHIFT) /* access protection */
+#define ARM_TTE_TABLE_APSHIFT       61
+#define ARM_TTE_TABLE_AP(x)         ((x)<<TTE_BLOCK_APSHIFT)       /* access protection */

 
 

-#define ARM_TTE_TABLE_NS                        0x8000000000000020ULL   /* value for a secure mapping */
-#define ARM_TTE_TABLE_NS_MASK           0x8000000000000020ULL   /* notSecure mapping mask */
+#define ARM_TTE_TABLE_NS            0x8000000000000020ULL          /* value for a secure mapping */
+#define ARM_TTE_TABLE_NS_MASK       0x8000000000000020ULL          /* notSecure mapping mask */

 
 

-#define ARM_TTE_TABLE_XN                        0x1000000000000000ULL   /* value for no execute */
-#define ARM_TTE_TABLE_XNMASK            0x1000000000000000ULL   /* no execute mask */
+#define ARM_TTE_TABLE_XN            0x1000000000000000ULL          /* value for no execute */
+#define ARM_TTE_TABLE_XNMASK        0x1000000000000000ULL          /* no execute mask */

 
 

-#define ARM_TTE_TABLE_PXN                       0x0800000000000000ULL   /* value for privilege no execute bit */
-#define ARM_TTE_TABLE_PXNMASK           0x0800000000000000ULL   /* privilege execute mask */
+#define ARM_TTE_TABLE_PXN           0x0800000000000000ULL          /* value for privilege no execute bit */
+#define ARM_TTE_TABLE_PXNMASK       0x0800000000000000ULL          /* privilege execute mask */

 
 #if __ARM_KERNEL_PROTECT__
 
 #if __ARM_KERNEL_PROTECT__

-#define ARM_TTE_BOOT_BLOCK                      (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID |  ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY)        \
-                                                                        | ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF \
-                                                                        | ARM_TTE_BLOCK_NG)
+#define ARM_TTE_BOOT_BLOCK \
+       (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) | \
+        ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF | ARM_TTE_BLOCK_NG)

 #else /* __ARM_KERNEL_PROTECT__ */
 #else /* __ARM_KERNEL_PROTECT__ */

-#define ARM_TTE_BOOT_BLOCK                      (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID |  ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY)        \
-                                                                        | ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF)
+#define ARM_TTE_BOOT_BLOCK \
+       (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) | \
+        ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF)

 #endif /* __ARM_KERNEL_PROTECT__ */
 
 #endif /* __ARM_KERNEL_PROTECT__ */
 

-#define ARM_TTE_BOOT_TABLE                      (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID )
+#define ARM_TTE_BOOT_TABLE (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID )

 /*
  *  L3 Translation table
  *
 /*
  *  L3 Translation table
  *


@@ -1052,15 +1136,15 @@
  */
 
 #ifdef __ARM_16K_PG__
  */
 
 #ifdef __ARM_16K_PG__

-#define ARM_PTE_SIZE                            0x0000000000004000ULL           /* size of area covered by a tte */
-#define ARM_PTE_OFFMASK                         0x0000000000003fffULL           /* offset within pte area */
-#define ARM_PTE_SHIFT                           14                                                      /* page descriptor shift */
-#define ARM_PTE_MASK                            0x0000ffffffffc000ULL           /* mask for output address in PTE */
+#define ARM_PTE_SIZE    0x0000000000004000ULL /* size of area covered by a tte */
+#define ARM_PTE_OFFMASK 0x0000000000003fffULL /* offset within pte area */
+#define ARM_PTE_SHIFT   14                    /* page descriptor shift */
+#define ARM_PTE_MASK    0x0000ffffffffc000ULL /* mask for output address in PTE */

 #else
 #else

-#define ARM_PTE_SIZE                            0x0000000000001000ULL           /* size of area covered by a tte */
-#define ARM_PTE_OFFMASK                         0x0000000000000fffULL           /* offset within pte area */
-#define ARM_PTE_SHIFT                           12                                                      /* page descriptor shift */
-#define ARM_PTE_MASK                            0x0000fffffffff000ULL           /* mask for output address in PTE */
+#define ARM_PTE_SIZE    0x0000000000001000ULL /* size of area covered by a tte */
+#define ARM_PTE_OFFMASK 0x0000000000000fffULL /* offset within pte area */
+#define ARM_PTE_SHIFT   12                    /* page descriptor shift */
+#define ARM_PTE_MASK    0x0000fffffffff000ULL /* mask for output address in PTE */

 #endif
 
 /*
 #endif
 
 /*


@@ -1068,11 +1152,11 @@
  *
  * The following page table entry types are possible:
  *
  *
  * The following page table entry types are possible:
  *

- *     fault page entry
- *     63                            2  0
- *     +------------------------------+--+
- *     |    ignored                   |00|
- *     +------------------------------+--+
+ * fault page entry
+ *  63                            2  0
+ * +------------------------------+--+
+ * |    ignored                   |00|
+ * +------------------------------+--+

  *
  *
  *  63 59 58  55 54  53   52 51  48 47                  12 11 10 9  8 7  6  5 4     2 1 0
  *
  *
  *  63 59 58  55 54  53   52 51  48 47                  12 11 10 9  8 7  6  5 4     2 1 0


@@ -1081,121 +1165,130 @@
  * +-----+------+--+---+----+------+----------------------+--+--+----+----+--+-------+-+-+
  *
  * where:
  * +-----+------+--+---+----+------+----------------------+--+--+----+----+--+-------+-+-+
  *
  * where:

- *     'nG'            notGlobal bit
- *     'SH'            Shareability field
- *     'AP'            access protection
- *     'XN'            eXecute Never bit
- *     'PXN'           Privilege eXecute Never bit
- *     'NS'            Non-Secure bit
- *     'HINT'          16 entry continuguous output hint
- *     'AttrIdx'       Memory Attribute Index
+ *   nG:      notGlobal bit
+ *   SH:      Shareability field
+ *   AP:      access protection
+ *   XN:      eXecute Never bit
+ *   PXN:     Privilege eXecute Never bit
+ *   NS:      Non-Secure bit
+ *   HINT:    16 entry continuguous output hint
+ *   AttrIdx: Memory Attribute Index

  */
 
  */
 

-#define PTE_SHIFT                                       3                                               /* shift width of a pte (sizeof(pte) == (1 << PTE_SHIFT)) */
+#define PTE_SHIFT               3                     /* shift width of a pte (sizeof(pte) == (1 << PTE_SHIFT)) */

 #ifdef __ARM_16K_PG__
 #ifdef __ARM_16K_PG__

-#define PTE_PGENTRIES                           (16384 >> PTE_SHIFT)    /* number of ptes per page */
+#define PTE_PGENTRIES           (16384 >> PTE_SHIFT)  /* number of ptes per page */

 #else
 #else

-#define PTE_PGENTRIES                           (4096 >> PTE_SHIFT)             /* number of ptes per page */
+#define PTE_PGENTRIES           (4096 >> PTE_SHIFT)   /* number of ptes per page */

 #endif
 
 #endif
 

-#define ARM_PTE_EMPTY                           0x0000000000000000ULL   /* unasigned - invalid entry */
+#define ARM_PTE_EMPTY           0x0000000000000000ULL /* unassigned - invalid entry */

 
 /* markers for (invalid) PTE for a page sent to compressor */
 
 /* markers for (invalid) PTE for a page sent to compressor */

-#define ARM_PTE_COMPRESSED              0x8000000000000000ULL   /* compressed... */
-#define ARM_PTE_COMPRESSED_ALT          0x4000000000000000ULL   /* ... and was "alt_acct" */
-#define ARM_PTE_COMPRESSED_MASK         0xC000000000000000ULL
-#define ARM_PTE_IS_COMPRESSED(x)                                        \
-       ((((x) & 0x3) == 0) &&          /* PTE is not valid... */       \
-        ((x) & ARM_PTE_COMPRESSED) &&  /* ...has "compressed" marker" */ \
-        ((!((x) & ~ARM_PTE_COMPRESSED_MASK)) || /* ...no other bits */ \
-         (panic("compressed PTE %p 0x%llx has extra bits 0x%llx: corrupted?", \
-                &(x), (x), (x) & ~ARM_PTE_COMPRESSED_MASK), FALSE)))
-
-#define ARM_PTE_TYPE                            0x0000000000000003ULL   /* valid L3 entry: includes bit #1 (counterintuitively) */
-#define ARM_PTE_TYPE_VALID                      0x0000000000000003ULL   /* valid L3 entry: includes bit #1 (counterintuitively) */
-#define ARM_PTE_TYPE_FAULT                      0x0000000000000000ULL   /* invalid L3 entry */
-#define ARM_PTE_TYPE_MASK                       0x0000000000000002ULL   /* mask to get pte type */
+#define ARM_PTE_COMPRESSED      0x8000000000000000ULL /* compressed... */
+#define ARM_PTE_COMPRESSED_ALT  0x4000000000000000ULL /* ... and was "alt_acct" */
+#define ARM_PTE_COMPRESSED_MASK 0xC000000000000000ULL
+
+#define ARM_PTE_IS_COMPRESSED(x, p) \
+       ((((x) & 0x3) == 0) && /* PTE is not valid... */                      \
+        ((x) & ARM_PTE_COMPRESSED) && /* ...has "compressed" marker" */      \
+        ((!((x) & ~ARM_PTE_COMPRESSED_MASK)) || /* ...no other bits */       \
+        (panic("compressed PTE %p 0x%llx has extra bits 0x%llx: corrupted?", \
+               (p), (x), (x) & ~ARM_PTE_COMPRESSED_MASK), FALSE)))
+
+#define ARM_PTE_TYPE               0x0000000000000003ULL /* valid L3 entry: includes bit #1 (counterintuitively) */
+#define ARM_PTE_TYPE_VALID         0x0000000000000003ULL /* valid L3 entry: includes bit #1 (counterintuitively) */
+#define ARM_PTE_TYPE_FAULT         0x0000000000000000ULL /* invalid L3 entry */
+#define ARM_PTE_TYPE_MASK          0x0000000000000002ULL /* mask to get pte type */

 
 #ifdef __ARM_16K_PG__
 /* TODO: What does the shift mean here? */
 
 #ifdef __ARM_16K_PG__
 /* TODO: What does the shift mean here? */

-#define ARM_PTE_PAGE_MASK                       0x0000FFFFFFFFC000ULL   /* mask for 16KB page */
+#define ARM_PTE_PAGE_MASK          0x0000FFFFFFFFC000ULL /* mask for 16KB page */

 #else
 #else

-#define ARM_PTE_PAGE_MASK                       0x0000FFFFFFFFF000ULL   /* mask for  4KB page */
-#define ARM_PTE_PAGE_SHIFT                      12                                              /* page shift for 4KB page */
+#define ARM_PTE_PAGE_MASK          0x0000FFFFFFFFF000ULL /* mask for  4KB page */
+#define ARM_PTE_PAGE_SHIFT         12                    /* page shift for 4KB page */

 #endif
 
 #endif
 

-#define ARM_PTE_AP(x)                           ((x) << 6)                              /* access protections */
-#define ARM_PTE_APMASK                          (0x3ULL << 6)                   /* mask access protections */
-#define ARM_PTE_EXTRACT_AP(x)           (((x) >> 6) & 0x3ULL)   /* extract access protections from PTE */
+#define ARM_PTE_AP(x)              ((x) << 6)            /* access protections */
+#define ARM_PTE_APMASK             (0x3ULL << 6)         /* mask access protections */
+#define ARM_PTE_EXTRACT_AP(x)      (((x) >> 6) & 0x3ULL) /* extract access protections from PTE */

 
 

-#define ARM_PTE_ATTRINDX(x)                     ((x) << 2)                              /* memory attributes index */
-#define ARM_PTE_ATTRINDXMASK            (0x7ULL << 2)                   /* mask memory attributes index */
+#define ARM_PTE_ATTRINDX(x)        ((x) << 2)            /* memory attributes index */
+#define ARM_PTE_ATTRINDXMASK       (0x7ULL << 2)         /* mask memory attributes index */

 
 

-#define ARM_PTE_SH(x)                           ((x) << 8)                              /* access shared */
-#define ARM_PTE_SHMASK                          (0x3ULL << 8)                   /* mask access shared */
+#define ARM_PTE_SH(x)              ((x) << 8)            /* access shared */
+#define ARM_PTE_SHMASK             (0x3ULL << 8)         /* mask access shared */

 
 

-#define ARM_PTE_AF                                      0x0000000000000400ULL   /* value for access */
-#define ARM_PTE_AFMASK                          0x0000000000000400ULL   /* access mask */
+#define ARM_PTE_AF                 0x0000000000000400ULL /* value for access */
+#define ARM_PTE_AFMASK             0x0000000000000400ULL /* access mask */

 
 

-#define ARM_PTE_NG                                      0x0000000000000800ULL   /* value for a global mapping */
-#define ARM_PTE_NG_MASK                         0x0000000000000800ULL   /* notGlobal mapping mask */
+#define ARM_PTE_NG                 0x0000000000000800ULL /* value for a global mapping */
+#define ARM_PTE_NG_MASK            0x0000000000000800ULL /* notGlobal mapping mask */

 
 

-#define ARM_PTE_NS                                      0x0000000000000020ULL   /* value for a secure mapping */
-#define ARM_PTE_NS_MASK                         0x0000000000000020ULL   /* notSecure mapping mask */
+#define ARM_PTE_NS                 0x0000000000000020ULL /* value for a secure mapping */
+#define ARM_PTE_NS_MASK            0x0000000000000020ULL /* notSecure mapping mask */

 
 

-#define ARM_PTE_HINT                            0x0010000000000000ULL   /* value for contiguous entries hint */
-#define ARM_PTE_HINT_MASK                       0x0010000000000000ULL   /* mask for contiguous entries hint */
+#define ARM_PTE_HINT               0x0010000000000000ULL /* value for contiguous entries hint */
+#define ARM_PTE_HINT_MASK          0x0010000000000000ULL /* mask for contiguous entries hint */

 
 #if __ARM_16K_PG__
 
 #if __ARM_16K_PG__

-#define ARM_PTE_HINT_ENTRIES            128ULL                                  /* number of entries the hint covers */
-#define ARM_PTE_HINT_ENTRIES_SHIFT      7ULL                                    /* shift to construct the number of entries */
-#define ARM_PTE_HINT_ADDR_MASK          0x0000FFFFFFE00000ULL                   /* mask to extract the starting hint address */
-#define ARM_PTE_HINT_ADDR_SHIFT         21                                      /* shift for the hint address */
-#define ARM_KVA_HINT_ADDR_MASK          0xFFFFFFFFFFE00000ULL                   /* mask to extract the starting hint address */
+#define ARM_PTE_HINT_ENTRIES       128ULL                /* number of entries the hint covers */
+#define ARM_PTE_HINT_ENTRIES_SHIFT 7ULL                  /* shift to construct the number of entries */
+#define ARM_PTE_HINT_ADDR_MASK     0x0000FFFFFFE00000ULL /* mask to extract the starting hint address */
+#define ARM_PTE_HINT_ADDR_SHIFT    21                    /* shift for the hint address */
+#define ARM_KVA_HINT_ADDR_MASK     0xFFFFFFFFFFE00000ULL /* mask to extract the starting hint address */

 #else
 #else

-#define ARM_PTE_HINT_ENTRIES            16ULL                                   /* number of entries the hint covers */
-#define ARM_PTE_HINT_ENTRIES_SHIFT      4ULL                                    /* shift to construct the number of entries */
-#define ARM_PTE_HINT_ADDR_MASK          0x0000FFFFFFFF0000ULL                   /* mask to extract the starting hint address */
-#define ARM_PTE_HINT_ADDR_SHIFT         16                                      /* shift for the hint address */
-#define ARM_KVA_HINT_ADDR_MASK          0xFFFFFFFFFFFF0000ULL                   /* mask to extract the starting hint address */
+#define ARM_PTE_HINT_ENTRIES       16ULL                 /* number of entries the hint covers */
+#define ARM_PTE_HINT_ENTRIES_SHIFT 4ULL                  /* shift to construct the number of entries */
+#define ARM_PTE_HINT_ADDR_MASK     0x0000FFFFFFFF0000ULL /* mask to extract the starting hint address */
+#define ARM_PTE_HINT_ADDR_SHIFT    16                    /* shift for the hint address */
+#define ARM_KVA_HINT_ADDR_MASK     0xFFFFFFFFFFFF0000ULL /* mask to extract the starting hint address */

 #endif
 
 #endif
 

-#define ARM_PTE_PNX                                     0x0020000000000000ULL   /* value for privilege no execute bit */
-#define ARM_PTE_PNXMASK                         0x0020000000000000ULL   /* privilege no execute mask */
+#define ARM_PTE_PNX                0x0020000000000000ULL /* value for privilege no execute bit */
+#define ARM_PTE_PNXMASK            0x0020000000000000ULL /* privilege no execute mask */

 
 

-#define ARM_PTE_NX                                      0x0040000000000000ULL   /* value for no execute bit */
-#define ARM_PTE_NXMASK                          0x0040000000000000ULL   /* no execute mask */
+#define ARM_PTE_NX                 0x0040000000000000ULL /* value for no execute bit */
+#define ARM_PTE_NXMASK             0x0040000000000000ULL /* no execute mask */

 
 

-#define ARM_PTE_WIRED                           0x0080000000000000ULL   /* value for software wired bit */
-#define ARM_PTE_WIRED_MASK                      0x0080000000000000ULL   /* software wired mask */
+#define ARM_PTE_WIRED              0x0400000000000000ULL /* value for software wired bit */
+#define ARM_PTE_WIRED_MASK         0x0400000000000000ULL /* software wired mask */

 
 

-#define ARM_PTE_WRITEABLE                       0x0100000000000000ULL   /* value for software writeable bit */
-#define ARM_PTE_WRITEABLE_MASK          0x0100000000000000ULL   /* software writeable mask */
+#define ARM_PTE_WRITEABLE          0x0800000000000000ULL /* value for software writeable bit */
+#define ARM_PTE_WRITEABLE_MASK     0x0800000000000000ULL /* software writeable mask */

 
 #if CONFIG_PGTRACE
 
 #if CONFIG_PGTRACE

-#define ARM_PTE_PGTRACE             0x0200000000000000ULL   /* value for software trace bit */
-#define ARM_PTE_PGTRACE_MASK        0x0200000000000000ULL   /* software trace mask */
+#define ARM_PTE_PGTRACE            0x0200000000000000ULL /* value for software trace bit */
+#define ARM_PTE_PGTRACE_MASK       0x0200000000000000ULL /* software trace mask */

 #endif
 
 #endif
 

-#define ARM_PTE_BOOT_PAGE_BASE                  (ARM_PTE_TYPE_VALID |  ARM_PTE_SH(SH_OUTER_MEMORY) \
-                                                                        | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_PTE_AF)
+#define ARM_PTE_BOOT_PAGE_BASE \
+       (ARM_PTE_TYPE_VALID | ARM_PTE_SH(SH_OUTER_MEMORY) |       \
+        ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_PTE_AF)

 
 #if __ARM_KERNEL_PROTECT__
 
 #if __ARM_KERNEL_PROTECT__

-#define ARM_PTE_BOOT_PAGE                       (ARM_PTE_BOOT_PAGE_BASE | ARM_PTE_NG)
+#define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE | ARM_PTE_NG)

 #else /* __ARM_KERNEL_PROTECT__ */
 #else /* __ARM_KERNEL_PROTECT__ */

-#define ARM_PTE_BOOT_PAGE                       (ARM_PTE_BOOT_PAGE_BASE)
+#define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE)

 #endif /* __ARM_KERNEL_PROTECT__ */
 
 /*
  * TLBI appers to only deal in 4KB page addresses, so give
  * it an explicit shift of 12.
  */
 #endif /* __ARM_KERNEL_PROTECT__ */
 
 /*
  * TLBI appers to only deal in 4KB page addresses, so give
  * it an explicit shift of 12.
  */

+#define TLBI_ADDR_SHIFT (0)

 #define TLBI_ADDR_SIZE  (44)
 #define TLBI_ADDR_MASK  ((1ULL << TLBI_ADDR_SIZE) - 1)
 #define TLBI_ADDR_SIZE  (44)
 #define TLBI_ADDR_MASK  ((1ULL << TLBI_ADDR_SIZE) - 1)

-#define TLBI_ADDR_SHIFT (12)

 #define TLBI_ASID_SHIFT (48)
 #define TLBI_ASID_SIZE  (16)
 #define TLBI_ASID_SHIFT (48)
 #define TLBI_ASID_SIZE  (16)

-#define TLBI_ASID_MASK  (((1ULL << TLBI_ASID_SIZE) - 1) << TLBI_ASID_SHIFT)
+#define TLBI_ASID_MASK  (((1ULL << TLBI_ASID_SIZE) - 1))
+
+#define RTLBI_ADDR_SIZE (37)
+#define RTLBI_ADDR_MASK ((1ULL << RTLBI_ADDR_SIZE) - 1)
+#define RTLBI_ADDR_SHIFT ARM_TT_L3_SHIFT
+#define RTLBI_TG ((uint64_t)(((ARM_TT_L3_SHIFT - 12) >> 1) + 1) << 46)
+#define RTLBI_SCALE_SHIFT (44)
+#define RTLBI_NUM_SHIFT (39)

 
 /*
  * Exception Syndrome Register
 
 /*
  * Exception Syndrome Register


@@ -1205,85 +1298,85 @@
  * |  EC  |IL|       ISS        |
  * +------+--+------------------+
  *
  * |  EC  |IL|       ISS        |
  * +------+--+------------------+
  *

- *     EC - Exception Class
- *     IL - Instruction Length
- *  ISS- Instruction Specific Syndrome
+ * EC  - Exception Class
+ * IL  - Instruction Length
+ * ISS - Instruction Specific Syndrome

  *
  * Note: The ISS can have many forms. These are defined separately below.
  */
 
  *
  * Note: The ISS can have many forms. These are defined separately below.
  */
 

-#define ESR_EC_SHIFT                            26
-#define ESR_EC_MASK                                     (0x3F << ESR_EC_SHIFT)
-#define ESR_EC(x)                                       ((x & ESR_EC_MASK) >> ESR_EC_SHIFT)
+#define ESR_EC_SHIFT           26
+#define ESR_EC_MASK            (0x3FULL << ESR_EC_SHIFT)
+#define ESR_EC(x)              ((x & ESR_EC_MASK) >> ESR_EC_SHIFT)

 
 

-#define ESR_IL_SHIFT                            25
-#define ESR_IL                                          (1 << ESR_IL_SHIFT)
+#define ESR_IL_SHIFT           25
+#define ESR_IL                 (1 << ESR_IL_SHIFT)

 
 

-#define ESR_INSTR_IS_2BYTES(x)          (!(x & ESR_IL))
+#define ESR_INSTR_IS_2BYTES(x) (!(x & ESR_IL))

 
 

-#define ESR_ISS_MASK                            0x01FFFFFF
-#define ESR_ISS(x)                                      (x & ESR_ISS_MASK)
+#define ESR_ISS_MASK           0x01FFFFFF
+#define ESR_ISS(x)             (x & ESR_ISS_MASK)

 
 #ifdef __ASSEMBLER__
 /* Define only the classes we need to test in the exception vectors. */
 
 #ifdef __ASSEMBLER__
 /* Define only the classes we need to test in the exception vectors. */

-#define ESR_EC_IABORT_EL1                       0x21
-#define ESR_EC_DABORT_EL1                       0x25
-#define ESR_EC_SP_ALIGN                         0x26
+#define ESR_EC_IABORT_EL1      0x21
+#define ESR_EC_DABORT_EL1      0x25
+#define ESR_EC_SP_ALIGN        0x26

 #else
 typedef enum {
 #else
 typedef enum {

-       ESR_EC_UNCATEGORIZED                    = 0x00,
-       ESR_EC_WFI_WFE                                  = 0x01,
-       ESR_EC_MCR_MRC_CP15_TRAP                = 0x03,
-       ESR_EC_MCRR_MRRC_CP15_TRAP              = 0x04,
-       ESR_EC_MCR_MRC_CP14_TRAP                = 0x05,
-       ESR_EC_LDC_STC_CP14_TRAP                = 0x06,
-       ESR_EC_TRAP_SIMD_FP                             = 0x07,
-       ESR_EC_MCRR_MRRC_CP14_TRAP              = 0x0c,
-       ESR_EC_ILLEGAL_INSTR_SET                = 0x0e,
-       ESR_EC_SVC_32                                   = 0x11,
-       ESR_EC_SVC_64                                   = 0x15,
-       ESR_EC_MSR_TRAP                                 = 0x18,
-       ESR_EC_IABORT_EL0                               = 0x20,
-       ESR_EC_IABORT_EL1                               = 0x21,
-       ESR_EC_PC_ALIGN                                 = 0x22,
-       ESR_EC_DABORT_EL0                               = 0x24,
-       ESR_EC_DABORT_EL1                               = 0x25,
-       ESR_EC_SP_ALIGN                                 = 0x26,
-       ESR_EC_FLOATING_POINT_32                = 0x28,
-       ESR_EC_FLOATING_POINT_64                = 0x2C,
-       ESR_EC_BKPT_REG_MATCH_EL0               = 0x30, // Breakpoint Debug event taken to the EL from a lower EL.
-       ESR_EC_BKPT_REG_MATCH_EL1               = 0x31, // Breakpoint Debug event taken to the EL from the EL.
-       ESR_EC_SW_STEP_DEBUG_EL0                = 0x32, // Software Step Debug event taken to the EL from a lower EL.
-       ESR_EC_SW_STEP_DEBUG_EL1                = 0x33, // Software Step Debug event taken to the EL from the EL.
-       ESR_EC_WATCHPT_MATCH_EL0                = 0x34, // Watchpoint Debug event taken to the EL from a lower EL.
-       ESR_EC_WATCHPT_MATCH_EL1                = 0x35, // Watchpoint Debug event taken to the EL from the EL.
-       ESR_EC_BKPT_AARCH32                             = 0x38,
-       ESR_EC_BRK_AARCH64                              = 0x3C
+       ESR_EC_UNCATEGORIZED       = 0x00,
+       ESR_EC_WFI_WFE             = 0x01,
+       ESR_EC_MCR_MRC_CP15_TRAP   = 0x03,
+       ESR_EC_MCRR_MRRC_CP15_TRAP = 0x04,
+       ESR_EC_MCR_MRC_CP14_TRAP   = 0x05,
+       ESR_EC_LDC_STC_CP14_TRAP   = 0x06,
+       ESR_EC_TRAP_SIMD_FP        = 0x07,
+       ESR_EC_MCRR_MRRC_CP14_TRAP = 0x0c,
+       ESR_EC_ILLEGAL_INSTR_SET   = 0x0e,
+       ESR_EC_SVC_32              = 0x11,
+       ESR_EC_SVC_64              = 0x15,
+       ESR_EC_MSR_TRAP            = 0x18,
+       ESR_EC_IABORT_EL0          = 0x20,
+       ESR_EC_IABORT_EL1          = 0x21,
+       ESR_EC_PC_ALIGN            = 0x22,
+       ESR_EC_DABORT_EL0          = 0x24,
+       ESR_EC_DABORT_EL1          = 0x25,
+       ESR_EC_SP_ALIGN            = 0x26,
+       ESR_EC_FLOATING_POINT_32   = 0x28,
+       ESR_EC_FLOATING_POINT_64   = 0x2C,
+       ESR_EC_BKPT_REG_MATCH_EL0  = 0x30, // Breakpoint Debug event taken to the EL from a lower EL.
+       ESR_EC_BKPT_REG_MATCH_EL1  = 0x31, // Breakpoint Debug event taken to the EL from the EL.
+       ESR_EC_SW_STEP_DEBUG_EL0   = 0x32, // Software Step Debug event taken to the EL from a lower EL.
+       ESR_EC_SW_STEP_DEBUG_EL1   = 0x33, // Software Step Debug event taken to the EL from the EL.
+       ESR_EC_WATCHPT_MATCH_EL0   = 0x34, // Watchpoint Debug event taken to the EL from a lower EL.
+       ESR_EC_WATCHPT_MATCH_EL1   = 0x35, // Watchpoint Debug event taken to the EL from the EL.
+       ESR_EC_BKPT_AARCH32        = 0x38,
+       ESR_EC_BRK_AARCH64         = 0x3C,

 } esr_exception_class_t;
 
 typedef enum {
 } esr_exception_class_t;
 
 typedef enum {

-       FSC_TRANSLATION_FAULT_L0                = 0x04,
-       FSC_TRANSLATION_FAULT_L1                = 0x05,
-       FSC_TRANSLATION_FAULT_L2                = 0x06,
-       FSC_TRANSLATION_FAULT_L3                = 0x07,
-       FSC_ACCESS_FLAG_FAULT_L1                = 0x09,
-       FSC_ACCESS_FLAG_FAULT_L2                = 0x0A,
-       FSC_ACCESS_FLAG_FAULT_L3                = 0x0B,
-       FSC_PERMISSION_FAULT_L1                 = 0x0D,
-       FSC_PERMISSION_FAULT_L2                 = 0x0E,
-       FSC_PERMISSION_FAULT_L3                 = 0x0F,
-       FSC_SYNC_EXT_ABORT                              = 0x10,
-       FSC_ASYNC_EXT_ABORT                             = 0x11,
-       FSC_SYNC_EXT_ABORT_TT_L1                = 0x15,
-       FSC_SYNC_EXT_ABORT_TT_L2                = 0x16,
-       FSC_SYNC_EXT_ABORT_TT_L3                = 0x17,
-       FSC_SYNC_PARITY                                 = 0x18,
-       FSC_ASYNC_PARITY                                = 0x19,
-       FSC_SYNC_PARITY_TT_L1                   = 0x1D,
-       FSC_SYNC_PARITY_TT_L2                   = 0x1E,
-       FSC_SYNC_PARITY_TT_L3                   = 0x1F,
-       FSC_ALIGNMENT_FAULT                             = 0x21,
-       FSC_DEBUG_FAULT                                 = 0x22
+       FSC_TRANSLATION_FAULT_L0   = 0x04,
+       FSC_TRANSLATION_FAULT_L1   = 0x05,
+       FSC_TRANSLATION_FAULT_L2   = 0x06,
+       FSC_TRANSLATION_FAULT_L3   = 0x07,
+       FSC_ACCESS_FLAG_FAULT_L1   = 0x09,
+       FSC_ACCESS_FLAG_FAULT_L2   = 0x0A,
+       FSC_ACCESS_FLAG_FAULT_L3   = 0x0B,
+       FSC_PERMISSION_FAULT_L1    = 0x0D,
+       FSC_PERMISSION_FAULT_L2    = 0x0E,
+       FSC_PERMISSION_FAULT_L3    = 0x0F,
+       FSC_SYNC_EXT_ABORT         = 0x10,
+       FSC_ASYNC_EXT_ABORT        = 0x11,
+       FSC_SYNC_EXT_ABORT_TT_L1   = 0x15,
+       FSC_SYNC_EXT_ABORT_TT_L2   = 0x16,
+       FSC_SYNC_EXT_ABORT_TT_L3   = 0x17,
+       FSC_SYNC_PARITY            = 0x18,
+       FSC_ASYNC_PARITY           = 0x19,
+       FSC_SYNC_PARITY_TT_L1      = 0x1D,
+       FSC_SYNC_PARITY_TT_L2      = 0x1E,
+       FSC_SYNC_PARITY_TT_L3      = 0x1F,
+       FSC_ALIGNMENT_FAULT        = 0x21,
+       FSC_DEBUG_FAULT            = 0x22

 } fault_status_t;
 #endif /* ASSEMBLER */
 
 } fault_status_t;
 #endif /* ASSEMBLER */
 


@@ -1295,19 +1388,19 @@ typedef enum {
  * +---+-----------------+--+------+
  *
  * where:
  * +---+-----------------+--+------+
  *
  * where:

- *     ISV             Instruction syndrome valid
- *     EX              Exclusive access
- *     IFSC    Instruction Fault Status Code
+ *   ISV:  Instruction syndrome valid
+ *   EX:   Exclusive access
+ *   IFSC: Instruction Fault Status Code

  */
 
  */
 

-#define ISS_SSDE_ISV_SHIFT                      24
-#define ISS_SSDE_ISV                            (0x1 << ISS_SSDE_ISV_SHIFT)
+#define ISS_SSDE_ISV_SHIFT 24
+#define ISS_SSDE_ISV       (0x1 << ISS_SSDE_ISV_SHIFT)

 
 

-#define ISS_SSDE_EX_SHIFT                       6
-#define ISS_SSDE_EX                                     (0x1 << ISS_SSDE_EX_SHIFT)
+#define ISS_SSDE_EX_SHIFT  6
+#define ISS_SSDE_EX        (0x1 << ISS_SSDE_EX_SHIFT)

 
 

-#define ISS_SSDE_FSC_MASK                       0x3F
-#define ISS_SSDE_FSC(x)                         (x & ISS_SSDE_FSC_MASK)
+#define ISS_SSDE_FSC_MASK  0x3F
+#define ISS_SSDE_FSC(x)    (x & ISS_SSDE_FSC_MASK)

 
 /*
  * Instruction Abort ISS (EL1)
 
 /*
  * Instruction Abort ISS (EL1)


@@ -1317,15 +1410,15 @@ typedef enum {
  * +---------------+--+---+------+
  *
  * where:
  * +---------------+--+---+------+
  *
  * where:

- *     EA              External Abort type
- *     IFSC    Instruction Fault Status Code
+ *   EA:   External Abort type
+ *   IFSC: Instruction Fault Status Code

  */
 
  */
 

-#define ISS_IA_EA_SHIFT                         9
-#define ISS_IA_EA                                       (0x1 << ISS_IA_EA_SHIFT)
+#define ISS_IA_EA_SHIFT 9
+#define ISS_IA_EA       (0x1 << ISS_IA_EA_SHIFT)

 
 

-#define ISS_IA_FSC_MASK                         0x3F
-#define ISS_IA_FSC(x)                           (x & ISS_IA_FSC_MASK)
+#define ISS_IA_FSC_MASK 0x3F
+#define ISS_IA_FSC(x)   (x & ISS_IA_FSC_MASK)

 
 
 /*
 
 
 /*


@@ -1337,59 +1430,98 @@ typedef enum {
  * +---------------+--+--+-+---+----+
  *
  * where:
  * +---------------+--+--+-+---+----+
  *
  * where:

- *     EA              External Abort type
- *     CM              Cache Maintenance operation
- *     WnR             Write not Read
- *     DFSC    Data Fault Status Code
+ *   EA:   External Abort type
+ *   CM:   Cache Maintenance operation
+ *   WnR:  Write not Read
+ *   DFSC: Data Fault Status Code

  */
  */

-#define ISS_DA_EA_SHIFT                         9
-#define ISS_DA_EA                                       (0x1 << ISS_DA_EA_SHIFT)
+#define ISS_DA_EA_SHIFT  9
+#define ISS_DA_EA        (0x1 << ISS_DA_EA_SHIFT)

 
 

-#define ISS_DA_CM_SHIFT                         8
-#define ISS_DA_CM                                       (0x1 << ISS_DA_CM_SHIFT)
+#define ISS_DA_CM_SHIFT  8
+#define ISS_DA_CM        (0x1 << ISS_DA_CM_SHIFT)

 
 

-#define ISS_DA_WNR_SHIFT                        6
-#define ISS_DA_WNR                                      (0x1 << ISS_DA_WNR_SHIFT)
+#define ISS_DA_WNR_SHIFT 6
+#define ISS_DA_WNR       (0x1 << ISS_DA_WNR_SHIFT)
+
+#define ISS_DA_FSC_MASK  0x3F
+#define ISS_DA_FSC(x)    (x & ISS_DA_FSC_MASK)
+
+/*
+ * Floating Point Exception ISS (EL1)
+ *
+ * 24  23 22            8  7      4   3   2   1   0
+ * +-+---+---------------+---+--+---+---+---+---+---+
+ * |0|TFV|000000000000000|IDF|00|IXF|UFF|OFF|DZF|IOF|
+ * +-+---+---------------+---+--+---+---+---+---+---+
+ *
+ * where:
+ *   TFV: Trapped Fault Valid
+ *   IDF: Input Denormal Exception
+ *   IXF: Input Inexact Exception
+ *   UFF: Underflow Exception
+ *   OFF: Overflow Exception
+ *   DZF: Divide by Zero Exception
+ *   IOF: Invalid Operation Exception
+ */
+#define ISS_FP_TFV_SHIFT 23
+#define ISS_FP_TFV       (0x1 << ISS_FP_TFV_SHIFT)
+
+#define ISS_FP_IDF_SHIFT 7
+#define ISS_FP_IDF       (0x1 << ISS_FP_IDF_SHIFT)
+
+#define ISS_FP_IXF_SHIFT 4
+#define ISS_FP_IXF       (0x1 << ISS_FP_IXF_SHIFT)
+
+#define ISS_FP_UFF_SHIFT 3
+#define ISS_FP_UFF       (0x1 << ISS_FP_UFF_SHIFT)
+
+#define ISS_FP_OFF_SHIFT 2
+#define ISS_FP_OFF       (0x1 << ISS_FP_OFF_SHIFT)
+
+#define ISS_FP_DZF_SHIFT 1
+#define ISS_FP_DZF       (0x1 << ISS_FP_DZF_SHIFT)
+
+#define ISS_FP_IOF_SHIFT 0
+#define ISS_FP_IOF       (0x1 << ISS_FP_IOF_SHIFT)

 
 

-#define ISS_DA_FSC_MASK                         0x3F
-#define ISS_DA_FSC(x)                           (x & ISS_DA_FSC_MASK)

 
 /*
  * Physical Address Register (EL1)
  */
 
 /*
  * Physical Address Register (EL1)
  */

-#define PAR_F_SHIFT                                     0
-#define PAR_F                                           (0x1 << PAR_F_SHIFT)
+#define PAR_F_SHIFT 0
+#define PAR_F       (0x1 << PAR_F_SHIFT)

 
 

-#define PLATFORM_SYSCALL_TRAP_NO                0x80000000
+#define PLATFORM_SYSCALL_TRAP_NO 0x80000000

 
 

-#define ARM64_SYSCALL_CODE_REG_NUM                      (16)
+#define ARM64_SYSCALL_CODE_REG_NUM (16)

 
 

-#define ARM64_CLINE_SHIFT                       6
+#define ARM64_CLINE_SHIFT 6

 
 #if defined(APPLE_ARM64_ARCH_FAMILY)
 
 #if defined(APPLE_ARM64_ARCH_FAMILY)

-#define L2CERRSTS_DATSBEESV     (1ULL << 2)     /* L2C data single bit ECC error */
-#define L2CERRSTS_DATDBEESV     (1ULL << 4)     /* L2C data double bit ECC error */
+#define L2CERRSTS_DATSBEESV (1ULL << 2) /* L2C data single bit ECC error */
+#define L2CERRSTS_DATDBEESV (1ULL << 4) /* L2C data double bit ECC error */

 #endif
 
 /*
  * Timer definitions.
  */
 #endif
 
 /*
  * Timer definitions.
  */

-#define CNTKCTL_EL1_PL0PTEN                             (0x1 << 9)              /* 1: EL0 access to physical timer regs permitted */
-#define CNTKCTL_EL1_PL0VTEN                             (0x1 << 8)              /* 1: EL0 access to virtual timer regs permitted */
-#define CNTKCTL_EL1_EVENTI_MASK                 (0x000000f0)    /* Mask for bits describing which bit to use for triggering event stream */
-#define CNTKCTL_EL1_EVENTI_SHIFT                (0x4)                   /* Shift for same */
-#define CNTKCTL_EL1_EVENTDIR                    (0x1 << 3)              /* 1: one-to-zero transition of specified bit causes event */
-#define CNTKCTL_EL1_EVNTEN                              (0x1 << 2)              /* 1: enable event stream */
-#define CNTKCTL_EL1_PL0VCTEN                    (0x1 << 1)              /* 1: EL0 access to physical timebase + frequency reg enabled */
-#define CNTKCTL_EL1_PL0PCTEN                    (0x1 << 0)              /* 1: EL0 access to virtual timebase + frequency reg enabled */
-
-#define CNTV_CTL_EL0_ISTATUS            (0x1 << 2)              /* (read only): whether interrupt asserted */
-#define CNTV_CTL_EL0_IMASKED            (0x1 << 1)              /* 1: interrupt masked */
-#define CNTV_CTL_EL0_ENABLE                     (0x1 << 0)              /* 1: virtual timer enabled */
-
-#define CNTP_CTL_EL0_ISTATUS            CNTV_CTL_EL0_ISTATUS
-#define CNTP_CTL_EL0_IMASKED            CNTV_CTL_EL0_IMASKED
-#define CNTP_CTL_EL0_ENABLE                     CNTV_CTL_EL0_ENABLE
+#define CNTKCTL_EL1_PL0PTEN      (0x1 << 9)           /* 1: EL0 access to physical timer regs permitted */
+#define CNTKCTL_EL1_PL0VTEN      (0x1 << 8)           /* 1: EL0 access to virtual timer regs permitted */
+#define CNTKCTL_EL1_EVENTI_MASK  (0x000000f0)         /* Mask for bits describing which bit to use for triggering event stream */
+#define CNTKCTL_EL1_EVENTI_SHIFT (0x4)                /* Shift for same */
+#define CNTKCTL_EL1_EVENTDIR     (0x1 << 3)           /* 1: one-to-zero transition of specified bit causes event */
+#define CNTKCTL_EL1_EVNTEN       (0x1 << 2)           /* 1: enable event stream */
+#define CNTKCTL_EL1_PL0VCTEN     (0x1 << 1)           /* 1: EL0 access to physical timebase + frequency reg enabled */
+#define CNTKCTL_EL1_PL0PCTEN     (0x1 << 0)           /* 1: EL0 access to virtual timebase + frequency reg enabled */
+
+#define CNTV_CTL_EL0_ISTATUS     (0x1 << 2)           /* (read only): whether interrupt asserted */
+#define CNTV_CTL_EL0_IMASKED     (0x1 << 1)           /* 1: interrupt masked */
+#define CNTV_CTL_EL0_ENABLE      (0x1 << 0)           /* 1: virtual timer enabled */
+
+#define CNTP_CTL_EL0_ISTATUS     CNTV_CTL_EL0_ISTATUS
+#define CNTP_CTL_EL0_IMASKED     CNTV_CTL_EL0_IMASKED
+#define CNTP_CTL_EL0_ENABLE      CNTV_CTL_EL0_ENABLE

 
 /*
  * At present all other uses of ARM_DBG_* are shared bit compatibly with the 32bit definitons.
 
 /*
  * At present all other uses of ARM_DBG_* are shared bit compatibly with the 32bit definitons.


@@ -1397,28 +1529,36 @@ typedef enum {
  */
 #define ARM_DBG_VR_ADDRESS_MASK64 0xFFFFFFFFFFFFFFFCull /* BVR & WVR */
 
  */
 #define ARM_DBG_VR_ADDRESS_MASK64 0xFFFFFFFFFFFFFFFCull /* BVR & WVR */
 

-#define MIDR_EL1_REV_SHIFT                      0
-#define MIDR_EL1_REV_MASK                       (0xf << MIDR_EL1_REV_SHIFT)
-#define MIDR_EL1_PNUM_SHIFT                     4
-#define MIDR_EL1_PNUM_MASK                      (0xfff << MIDR_EL1_PNUM_SHIFT)
-#define MIDR_EL1_ARCH_SHIFT                     16
-#define MIDR_EL1_ARCH_MASK                      (0xf << MIDR_EL1_ARCH_SHIFT)
-#define MIDR_EL1_VAR_SHIFT                      20
-#define MIDR_EL1_VAR_MASK                       (0xf << MIDR_EL1_VAR_SHIFT)
-#define MIDR_EL1_IMP_SHIFT                      24
-#define MIDR_EL1_IMP_MASK                       (0xff << MIDR_EL1_IMP_SHIFT)
+#define MIDR_EL1_REV_SHIFT  0
+#define MIDR_EL1_REV_MASK   (0xf << MIDR_EL1_REV_SHIFT)
+#define MIDR_EL1_PNUM_SHIFT 4
+#define MIDR_EL1_PNUM_MASK  (0xfff << MIDR_EL1_PNUM_SHIFT)
+#define MIDR_EL1_ARCH_SHIFT 16
+#define MIDR_EL1_ARCH_MASK  (0xf << MIDR_EL1_ARCH_SHIFT)
+#define MIDR_EL1_VAR_SHIFT  20
+#define MIDR_EL1_VAR_MASK   (0xf << MIDR_EL1_VAR_SHIFT)
+#define MIDR_EL1_IMP_SHIFT  24
+#define MIDR_EL1_IMP_MASK   (0xff << MIDR_EL1_IMP_SHIFT)

 
 /*
  * CoreSight debug registers
  */
 
 /*
  * CoreSight debug registers
  */

-#define CORESIGHT_ED    0
-#define CORESIGHT_CTI   1
-#define CORESIGHT_PMU   2
-#define CORESIGHT_UTT   3 /* Not truly a coresight thing, but at a fixed convenient location right after the coresight region */
+#define CORESIGHT_ED  0
+#define CORESIGHT_CTI 1
+#define CORESIGHT_PMU 2
+#define CORESIGHT_UTT 3 /* Not truly a coresight thing, but at a fixed convenient location right after the coresight region */
+
+#define CORESIGHT_OFFSET(x) ((x) * 0x10000)
+#define CORESIGHT_REGIONS   4
+#define CORESIGHT_SIZE      0x1000
+
+
+
+
+
+
+

 
 

-#define CORESIGHT_OFFSET(x)     ((x) * 0x10000)
-#define CORESIGHT_REGIONS       4
-#define CORESIGHT_SIZE          0x1000

 
 
 /*
 
 
 /*


@@ -1430,30 +1570,78 @@ typedef enum {
  * +----------+--------+------+------+------+-----+------+
  */
 
  * +----------+--------+------+------+------+-----+------+
  */
 

-#define ID_AA64ISAR0_EL1_ATOMIC_OFFSET  20
-#define ID_AA64ISAR0_EL1_ATOMIC_MASK    (0xfull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET)
-#define ID_AA64ISAR0_EL1_ATOMIC_8_1     (2ull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET)
+#define ID_AA64ISAR0_EL1_FHM_OFFSET    48
+#define ID_AA64ISAR0_EL1_FHM_MASK      (0xfull << ID_AA64ISAR0_EL1_FHM_OFFSET)
+#define ID_AA64ISAR0_EL1_FHM_8_2       (1ull << ID_AA64ISAR0_EL1_FHM_OFFSET)
+
+#define ID_AA64ISAR0_EL1_ATOMIC_OFFSET 20
+#define ID_AA64ISAR0_EL1_ATOMIC_MASK   (0xfull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET)
+#define ID_AA64ISAR0_EL1_ATOMIC_8_1    (2ull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET)
+
+#define ID_AA64ISAR0_EL1_CRC32_OFFSET  16
+#define ID_AA64ISAR0_EL1_CRC32_MASK    (0xfull << ID_AA64ISAR0_EL1_CRC32_OFFSET)
+#define ID_AA64ISAR0_EL1_CRC32_EN      (1ull << ID_AA64ISAR0_EL1_CRC32_OFFSET)
+
+#define ID_AA64ISAR0_EL1_SHA2_OFFSET   12
+#define ID_AA64ISAR0_EL1_SHA2_MASK     (0xfull << ID_AA64ISAR0_EL1_SHA2_OFFSET)
+#define ID_AA64ISAR0_EL1_SHA2_EN       (1ull << ID_AA64ISAR0_EL1_SHA2_OFFSET)

 
 

-#define ID_AA64ISAR0_EL1_CRC32_OFFSET   16
-#define ID_AA64ISAR0_EL1_CRC32_MASK     (0xfull << ID_AA64ISAR0_EL1_CRC32_OFFSET)
-#define ID_AA64ISAR0_EL1_CRC32_EN       (1ull << ID_AA64ISAR0_EL1_CRC32_OFFSET)
+#define ID_AA64ISAR0_EL1_SHA1_OFFSET   8
+#define ID_AA64ISAR0_EL1_SHA1_MASK     (0xfull << ID_AA64ISAR0_EL1_SHA1_OFFSET)
+#define ID_AA64ISAR0_EL1_SHA1_EN       (1ull << ID_AA64ISAR0_EL1_SHA1_OFFSET)

 
 

-#define ID_AA64ISAR0_EL1_SHA2_OFFSET    12
-#define ID_AA64ISAR0_EL1_SHA2_MASK      (0xfull << ID_AA64ISAR0_EL1_SHA2_OFFSET)
-#define ID_AA64ISAR0_EL1_SHA2_EN        (1ull << ID_AA64ISAR0_EL1_SHA2_OFFSET)
+#define ID_AA64ISAR0_EL1_AES_OFFSET    4
+#define ID_AA64ISAR0_EL1_AES_MASK      (0xfull << ID_AA64ISAR0_EL1_AES_OFFSET)
+#define ID_AA64ISAR0_EL1_AES_EN        (1ull << ID_AA64ISAR0_EL1_AES_OFFSET)
+#define ID_AA64ISAR0_EL1_AES_PMULL_EN  (2ull << ID_AA64ISAR0_EL1_AES_OFFSET)

 
 

-#define ID_AA64ISAR0_EL1_SHA1_OFFSET    8
-#define ID_AA64ISAR0_EL1_SHA1_MASK      (0xfull << ID_AA64ISAR0_EL1_SHA1_OFFSET)
-#define ID_AA64ISAR0_EL1_SHA1_EN        (1ull << ID_AA64ISAR0_EL1_SHA1_OFFSET)

 
 

-#define ID_AA64ISAR0_EL1_AES_OFFSET     4
-#define ID_AA64ISAR0_EL1_AES_MASK       (0xfull << ID_AA64ISAR0_EL1_AES_OFFSET)
-#define ID_AA64ISAR0_EL1_AES_EN         (1ull << ID_AA64ISAR0_EL1_AES_OFFSET)
-#define ID_AA64ISAR0_EL1_AES_PMULL_EN   (2ull << ID_AA64ISAR0_EL1_AES_OFFSET)
+#if __APCFG_SUPPORTED__
+/*
+ * APCFG_EL1
+ *
+ *  63       2 1 0
+ * +----------+-+-+
+ * | reserved |K|R|
+ * +----------+-+-+
+ *
+ * where:
+ *   R: Reserved
+ *   K: ElXEnKey - Enable ARMV8.3 defined {IA,IB,DA,DB} keys when CPU is
+ *                 operating in EL1 (or higher) and when under Apple-Mode
+ */

 
 

+#define APCFG_EL1_ELXENKEY_OFFSET      1
+#define APCFG_EL1_ELXENKEY_MASK        (0x1ULL << APCFG_EL1_ELXENKEY_OFFSET)
+#define APCFG_EL1_ELXENKEY             APCFG_EL1_ELXENKEY_MASK
+#endif /* __APCFG_SUPPORTED__ */
+
+#define APSTATE_G_SHIFT  (0)
+#define APSTATE_P_SHIFT  (1)
+#define APSTATE_A_SHIFT  (2)
+
+#ifdef __APSTS_SUPPORTED__
+#define APCTL_EL1_AppleMode  (1ULL << 0)
+#define APCTL_EL1_KernKeyEn  (1ULL << 1)
+#define APCTL_EL1_EnAPKey0   (1ULL << 2)
+#define APCTL_EL1_EnAPKey1   (1ULL << 3)
+#define APSTS_EL1_MKEYVld    (1ULL << 0)
+#else
+#define APCTL_EL1_AppleMode  (1ULL << 0)
+#define APCTL_EL1_MKEYVld    (1ULL << 1)
+#define APCTL_EL1_KernKeyEn  (1ULL << 2)
+#endif

 
 

+#define ACTLR_EL1_DisHWP_OFFSET  3
+#define ACTLR_EL1_DisHWP_MASK    (1ULL << ACTLR_EL1_DisHWP_OFFSET)
+#define ACTLR_EL1_DisHWP         ACTLR_EL1_DisHWP_MASK

 
 
 
 

+#if defined(HAS_APPLE_PAC)
+// The value of ptrauth_string_discriminator("recover"), hardcoded so it can be used from assembly code
+#define PAC_DISCRIMINATOR_RECOVER    0x1e02
+#endif
+

 #ifdef __ASSEMBLER__
 
 /*
 #ifdef __ASSEMBLER__
 
 /*


@@ -1463,14 +1651,14 @@ typedef enum {
  * Where the "variant" is the major number and the "revision" is the minor number.
  *
  * For example:
  * Where the "variant" is the major number and the "revision" is the minor number.
  *
  * For example:

- *      Cyclone A0 is variant 0, revision 0, i.e. 0.
- *     Cyclone B0 is variant 1, revision 0, i.e. 0x10
+ *   Cyclone A0 is variant 0, revision 0, i.e. 0.
+ *   Cyclone B0 is variant 1, revision 0, i.e. 0x10

  * $0 - register to place value in
  */
 .macro GET_MIDR_CPU_VERSION
  * $0 - register to place value in
  */
 .macro GET_MIDR_CPU_VERSION

-mrs     $0, MIDR_EL1                                            // Read MIDR_EL1 for CPUID
-bfi             $0, $0, #(MIDR_EL1_VAR_SHIFT - 4), #4           // move bits 3:0 (revision) to 19:16 (below variant) to get values adjacent
-ubfx    $0, $0, #(MIDR_EL1_VAR_SHIFT - 4), #8           // And extract the concatenated bitstring to beginning of register
+mrs  $0, MIDR_EL1                                  // Read MIDR_EL1 for CPUID
+bfi  $0, $0, #(MIDR_EL1_VAR_SHIFT - 4), #4         // move bits 3:0 (revision) to 19:16 (below variant) to get values adjacent
+ubfx $0, $0, #(MIDR_EL1_VAR_SHIFT - 4), #8         // And extract the concatenated bitstring to beginning of register

 .endmacro
 
 /*
 .endmacro
 
 /*


@@ -1483,8 +1671,8 @@ ubfx    $0, $0, #(MIDR_EL1_VAR_SHIFT - 4), #8           // And extract the conca
  */
 .macro SKIP_IF_CPU_VERSION_GREATER_OR_EQUAL
 GET_MIDR_CPU_VERSION $0
  */
 .macro SKIP_IF_CPU_VERSION_GREATER_OR_EQUAL
 GET_MIDR_CPU_VERSION $0

-cmp     $0, $1
-b.pl    $2                      // Unsigned "greater or equal"
+cmp  $0, $1
+b.pl $2                         // Unsigned "greater or equal"

 .endmacro
 
 /*
 .endmacro
 
 /*


@@ -1497,8 +1685,8 @@ b.pl    $2                      // Unsigned "greater or equal"
  */
 .macro SKIP_IF_CPU_VERSION_LESS_THAN
 GET_MIDR_CPU_VERSION $0
  */
 .macro SKIP_IF_CPU_VERSION_LESS_THAN
 GET_MIDR_CPU_VERSION $0

-cmp     $0, $1
-b.mi    $2                      // Unsigned "strictly less than"
+cmp  $0, $1
+b.mi $2                         // Unsigned "strictly less than"

 .endmacro
 
 #endif /* __ASSEMBLER__ */
 .endmacro
 
 #endif /* __ASSEMBLER__ */