xnu-6153.121.1.tar.gz

[apple/xnu.git] / osfmk / arm64 / machine_routines_asm.s

diff --git a/osfmk/arm64/machine_routines_asm.s b/osfmk/arm64/machine_routines_asm.s
index 5dc6cde731106afb85392ade5d0c675874c826f0..191997c1353bbcb0a276eb04425d998910388c10 100644 (file)
--- a/osfmk/arm64/machine_routines_asm.s
+++ b/osfmk/arm64/machine_routines_asm.s
@@ -58,7 +58,62 @@ LEXT(ml_set_kernelkey_enabled)
 
 #endif /* defined(HAS_APPLE_PAC) */
 
 
 #endif /* defined(HAS_APPLE_PAC) */
 

+#if HAS_BP_RET

 
 

+/*
+ * void set_bp_ret(void)
+ * Helper function to enable branch predictor state retention
+ * across ACC sleep
+ */
+
+       .align 2
+       .globl EXT(set_bp_ret)
+LEXT(set_bp_ret)
+       // Load bpret boot-arg
+       adrp            x14, EXT(bp_ret)@page
+       add             x14, x14, EXT(bp_ret)@pageoff
+       ldr             w14, [x14]
+
+       mrs             x13, ARM64_REG_ACC_CFG
+       and             x13, x13, (~(ARM64_REG_ACC_CFG_bpSlp_mask << ARM64_REG_ACC_CFG_bpSlp_shift))
+       and             x14, x14, #(ARM64_REG_ACC_CFG_bpSlp_mask)
+       orr             x13, x13, x14, lsl #(ARM64_REG_ACC_CFG_bpSlp_shift)
+       msr             ARM64_REG_ACC_CFG, x13
+
+       ret
+#endif // HAS_BP_RET
+
+#if HAS_NEX_PG
+       .align 2
+       .globl EXT(set_nex_pg)
+LEXT(set_nex_pg)
+       mrs             x14, MPIDR_EL1
+       // Skip if this isn't a p-core; NEX powergating isn't available for e-cores
+       and             x14, x14, #(MPIDR_PNE)
+       cbz             x14, Lnex_pg_done
+
+       // Set the SEG-recommended value of 12 additional reset cycles
+       mrs             x14, ARM64_REG_HID13
+       and             x14, x14, (~ARM64_REG_HID13_RstCyc_mask)
+       orr             x14, x14, ARM64_REG_HID13_RstCyc_val
+       msr             ARM64_REG_HID13, x14
+
+       // Load nexpg boot-arg
+       adrp            x14, EXT(nex_pg)@page
+       add             x14, x14, EXT(nex_pg)@pageoff
+       ldr             w14, [x14]
+
+       mrs             x13, ARM64_REG_HID14
+       and             x13, x13, (~ARM64_REG_HID14_NexPwgEn)
+       cbz             w14, Lset_nex_pg
+       orr             x13, x13, ARM64_REG_HID14_NexPwgEn
+Lset_nex_pg:
+       msr             ARM64_REG_HID14, x13
+
+Lnex_pg_done:
+       ret
+
+#endif // HAS_NEX_PG

 
 /*     uint32_t get_fpscr(void):
  *             Returns (FPSR | FPCR).
 
 /*     uint32_t get_fpscr(void):
  *             Returns (FPSR | FPCR).


@@ -168,12 +223,21 @@ LEXT(set_mmu_ttb_alternate)
        bl              EXT(pinst_set_ttbr1)
        mov             lr, x1
 #else
        bl              EXT(pinst_set_ttbr1)
        mov             lr, x1
 #else

+#if defined(HAS_VMSA_LOCK)
+       mrs             x1, ARM64_REG_VMSA_LOCK_EL1
+       and             x1, x1, #(VMSA_LOCK_TTBR1_EL1)
+       cbnz            x1, L_set_locked_reg_panic
+#endif /* defined(HAS_VMSA_LOCK) */

        msr             TTBR1_EL1, x0
 #endif /* defined(KERNEL_INTEGRITY_KTRR) */
        isb             sy
        ret
 
        msr             TTBR1_EL1, x0
 #endif /* defined(KERNEL_INTEGRITY_KTRR) */
        isb             sy
        ret
 

+#if XNU_MONITOR
+       .section __PPLTEXT,__text,regular,pure_instructions
+#else

        .text
        .text

+#endif

        .align 2
        .globl EXT(set_mmu_ttb)
 LEXT(set_mmu_ttb)
        .align 2
        .globl EXT(set_mmu_ttb)
 LEXT(set_mmu_ttb)


@@ -211,6 +275,19 @@ LEXT(set_vbar_el1)
 #endif
 #endif /* __ARM_KERNEL_PROTECT__ */
 
 #endif
 #endif /* __ARM_KERNEL_PROTECT__ */
 

+#if defined(HAS_VMSA_LOCK)
+       .text
+       .align 2
+       .globl EXT(vmsa_lock)
+LEXT(vmsa_lock)
+       isb sy
+       mov x1, #(VMSA_LOCK_SCTLR_M_BIT)
+       mov x0, #(VMSA_LOCK_TTBR1_EL1 | VMSA_LOCK_TCR_EL1 | VMSA_LOCK_VBAR_EL1)
+       orr x0, x0, x1
+       msr ARM64_REG_VMSA_LOCK_EL1, x0
+       isb sy
+       ret
+#endif /* defined(HAS_VMSA_LOCK) */

 
 /*
  *     set translation control register
 
 /*
  *     set translation control register


@@ -229,6 +306,12 @@ LEXT(set_tcr)
        bl              EXT(pinst_set_tcr)
        mov             lr, x1
 #else
        bl              EXT(pinst_set_tcr)
        mov             lr, x1
 #else

+#if defined(HAS_VMSA_LOCK)
+       // assert TCR unlocked
+       mrs             x1, ARM64_REG_VMSA_LOCK_EL1
+       and             x1, x1, #(VMSA_LOCK_TCR_EL1)
+       cbnz            x1, L_set_locked_reg_panic
+#endif /* defined(HAS_VMSA_LOCK) */

        msr             TCR_EL1, x0
 #endif /* defined(KERNEL_INTRITY_KTRR) */
        isb             sy
        msr             TCR_EL1, x0
 #endif /* defined(KERNEL_INTRITY_KTRR) */
        isb             sy


@@ -256,7 +339,7 @@ L_set_tcr_panic_str:
 L_set_locked_reg_panic_str:
        .asciz  "attempt to set locked register: (%llx)\n"
 #else
 L_set_locked_reg_panic_str:
        .asciz  "attempt to set locked register: (%llx)\n"
 #else

-#if defined(KERNEL_INTEGRITY_KTRR)
+#if defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR)

        mov             x1, lr
        bl              EXT(pinst_set_tcr)
        mov             lr, x1
        mov             x1, lr
        bl              EXT(pinst_set_tcr)
        mov             lr, x1


@@ -332,25 +415,32 @@ L_mmu_kvtop_wpreflight_invalid:
 /*
  * SET_RECOVERY_HANDLER
  *
 /*
  * SET_RECOVERY_HANDLER
  *

- *     Sets up a page fault recovery handler
+ *     Sets up a page fault recovery handler.  This macro clobbers x16 and x17.

  *
  *

- *     arg0 - persisted thread pointer
- *     arg1 - persisted recovery handler
- *     arg2 - scratch reg
- *     arg3 - recovery label
+ *     label - recovery label
+ *     tpidr - persisted thread pointer
+ *     old_handler - persisted recovery handler
+ *     label_in_adr_range - whether \label is within 1 MB of PC

  */
  */

-.macro SET_RECOVERY_HANDLER
-       mrs             $0, TPIDR_EL1                                   // Load thread pointer
-       adrp    $2, $3@page                                             // Load the recovery handler address
-       add             $2, $2, $3@pageoff
+.macro SET_RECOVERY_HANDLER    label, tpidr=x16, old_handler=x10, label_in_adr_range=0
+       // Note: x16 and x17 are designated for use as temporaries in
+       // interruptible PAC routines.  DO NOT CHANGE THESE REGISTER ASSIGNMENTS.
+.if \label_in_adr_range==1                                             // Load the recovery handler address
+       adr             x17, \label
+.else
+       adrp    x17, \label@page
+       add             x17, x17, \label@pageoff
+.endif

 #if defined(HAS_APPLE_PAC)
 #if defined(HAS_APPLE_PAC)

-       add             $1, $0, TH_RECOVER
-       movk    $1, #PAC_DISCRIMINATOR_RECOVER, lsl 48
-       pacia   $2, $1                                                  // Sign with IAKey + blended discriminator
+       mrs             x16, TPIDR_EL1
+       add             x16, x16, TH_RECOVER
+       movk    x16, #PAC_DISCRIMINATOR_RECOVER, lsl 48
+       pacia   x17, x16                                                        // Sign with IAKey + blended discriminator

 #endif
 
 #endif
 

-       ldr             $1, [$0, TH_RECOVER]                    // Save previous recovery handler
-       str             $2, [$0, TH_RECOVER]                    // Set new signed recovery handler
+       mrs             \tpidr, TPIDR_EL1                                       // Load thread pointer
+       ldr             \old_handler, [\tpidr, TH_RECOVER]      // Save previous recovery handler
+       str             x17, [\tpidr, TH_RECOVER]                       // Set new signed recovery handler

 .endmacro
 
 /*
 .endmacro
 
 /*


@@ -358,18 +448,18 @@ L_mmu_kvtop_wpreflight_invalid:
  *
  *     Clears page fault handler set by SET_RECOVERY_HANDLER
  *
  *
  *     Clears page fault handler set by SET_RECOVERY_HANDLER
  *

- *     arg0 - thread pointer saved by SET_RECOVERY_HANDLER
- *     arg1 - old recovery handler saved by SET_RECOVERY_HANDLER
+ *     tpidr - thread pointer saved by SET_RECOVERY_HANDLER
+ *     old_handler - old recovery handler saved by SET_RECOVERY_HANDLER

  */
  */

-.macro CLEAR_RECOVERY_HANDLER
-       str             $1, [$0, TH_RECOVER]            // Restore the previous recovery handler
+.macro CLEAR_RECOVERY_HANDLER  tpidr=x16, old_handler=x10
+       str             \old_handler, [\tpidr, TH_RECOVER]      // Restore the previous recovery handler

 .endmacro
 
 
        .text
        .align 2
 copyio_error:
 .endmacro
 
 
        .text
        .align 2
 copyio_error:

-       CLEAR_RECOVERY_HANDLER x10, x11
+       CLEAR_RECOVERY_HANDLER

        mov             x0, #EFAULT                                     // Return an EFAULT error
        POP_FRAME
        ARM64_STACK_EPILOG
        mov             x0, #EFAULT                                     // Return an EFAULT error
        POP_FRAME
        ARM64_STACK_EPILOG


@@ -383,7 +473,7 @@ copyio_error:
 LEXT(_bcopyin)
        ARM64_STACK_PROLOG
        PUSH_FRAME
 LEXT(_bcopyin)
        ARM64_STACK_PROLOG
        PUSH_FRAME

-       SET_RECOVERY_HANDLER x10, x11, x3, copyio_error
+       SET_RECOVERY_HANDLER copyio_error

        /* If len is less than 16 bytes, just do a bytewise copy */
        cmp             x2, #16
        b.lt    2f
        /* If len is less than 16 bytes, just do a bytewise copy */
        cmp             x2, #16
        b.lt    2f


@@ -403,7 +493,7 @@ LEXT(_bcopyin)
        strb    w3, [x1], #1
        b.hi    2b
 3:
        strb    w3, [x1], #1
        b.hi    2b
 3:

-       CLEAR_RECOVERY_HANDLER x10, x11
+       CLEAR_RECOVERY_HANDLER

        mov             x0, #0
        POP_FRAME
        ARM64_STACK_EPILOG
        mov             x0, #0
        POP_FRAME
        ARM64_STACK_EPILOG


@@ -417,11 +507,11 @@ LEXT(_bcopyin)
 LEXT(_copyin_atomic32)
        ARM64_STACK_PROLOG
        PUSH_FRAME
 LEXT(_copyin_atomic32)
        ARM64_STACK_PROLOG
        PUSH_FRAME

-       SET_RECOVERY_HANDLER x10, x11, x3, copyio_error
+       SET_RECOVERY_HANDLER copyio_error

        ldr             w8, [x0]
        str             w8, [x1]
        mov             x0, #0
        ldr             w8, [x0]
        str             w8, [x1]
        mov             x0, #0

-       CLEAR_RECOVERY_HANDLER x10, x11
+       CLEAR_RECOVERY_HANDLER

        POP_FRAME
        ARM64_STACK_EPILOG
 
        POP_FRAME
        ARM64_STACK_EPILOG
 


@@ -434,7 +524,7 @@ LEXT(_copyin_atomic32)
 LEXT(_copyin_atomic32_wait_if_equals)
        ARM64_STACK_PROLOG
        PUSH_FRAME
 LEXT(_copyin_atomic32_wait_if_equals)
        ARM64_STACK_PROLOG
        PUSH_FRAME

-       SET_RECOVERY_HANDLER x10, x11, x3, copyio_error
+       SET_RECOVERY_HANDLER copyio_error

        ldxr            w8, [x0]
        cmp             w8, w1
        mov             x0, ESTALE
        ldxr            w8, [x0]
        cmp             w8, w1
        mov             x0, ESTALE


@@ -443,7 +533,7 @@ LEXT(_copyin_atomic32_wait_if_equals)
        wfe
 1:
        clrex
        wfe
 1:
        clrex

-       CLEAR_RECOVERY_HANDLER x10, x11
+       CLEAR_RECOVERY_HANDLER

        POP_FRAME
        ARM64_STACK_EPILOG
 
        POP_FRAME
        ARM64_STACK_EPILOG
 


@@ -456,11 +546,11 @@ LEXT(_copyin_atomic32_wait_if_equals)
 LEXT(_copyin_atomic64)
        ARM64_STACK_PROLOG
        PUSH_FRAME
 LEXT(_copyin_atomic64)
        ARM64_STACK_PROLOG
        PUSH_FRAME

-       SET_RECOVERY_HANDLER x10, x11, x3, copyio_error
+       SET_RECOVERY_HANDLER copyio_error

        ldr             x8, [x0]
        str             x8, [x1]
        mov             x0, #0
        ldr             x8, [x0]
        str             x8, [x1]
        mov             x0, #0

-       CLEAR_RECOVERY_HANDLER x10, x11
+       CLEAR_RECOVERY_HANDLER

        POP_FRAME
        ARM64_STACK_EPILOG
 
        POP_FRAME
        ARM64_STACK_EPILOG
 


@@ -474,10 +564,10 @@ LEXT(_copyin_atomic64)
 LEXT(_copyout_atomic32)
        ARM64_STACK_PROLOG
        PUSH_FRAME
 LEXT(_copyout_atomic32)
        ARM64_STACK_PROLOG
        PUSH_FRAME

-       SET_RECOVERY_HANDLER x10, x11, x3, copyio_error
+       SET_RECOVERY_HANDLER copyio_error

        str             w0, [x1]
        mov             x0, #0
        str             w0, [x1]
        mov             x0, #0

-       CLEAR_RECOVERY_HANDLER x10, x11
+       CLEAR_RECOVERY_HANDLER

        POP_FRAME
        ARM64_STACK_EPILOG
 
        POP_FRAME
        ARM64_STACK_EPILOG
 


@@ -490,10 +580,10 @@ LEXT(_copyout_atomic32)
 LEXT(_copyout_atomic64)
        ARM64_STACK_PROLOG
        PUSH_FRAME
 LEXT(_copyout_atomic64)
        ARM64_STACK_PROLOG
        PUSH_FRAME

-       SET_RECOVERY_HANDLER x10, x11, x3, copyio_error
+       SET_RECOVERY_HANDLER copyio_error

        str             x0, [x1]
        mov             x0, #0
        str             x0, [x1]
        mov             x0, #0

-       CLEAR_RECOVERY_HANDLER x10, x11
+       CLEAR_RECOVERY_HANDLER

        POP_FRAME
        ARM64_STACK_EPILOG
 
        POP_FRAME
        ARM64_STACK_EPILOG
 


@@ -507,7 +597,7 @@ LEXT(_copyout_atomic64)
 LEXT(_bcopyout)
        ARM64_STACK_PROLOG
        PUSH_FRAME
 LEXT(_bcopyout)
        ARM64_STACK_PROLOG
        PUSH_FRAME

-       SET_RECOVERY_HANDLER x10, x11, x3, copyio_error
+       SET_RECOVERY_HANDLER copyio_error

        /* If len is less than 16 bytes, just do a bytewise copy */
        cmp             x2, #16
        b.lt    2f
        /* If len is less than 16 bytes, just do a bytewise copy */
        cmp             x2, #16
        b.lt    2f


@@ -527,7 +617,7 @@ LEXT(_bcopyout)
        strb    w3, [x1], #1
        b.hi    2b
 3:
        strb    w3, [x1], #1
        b.hi    2b
 3:

-       CLEAR_RECOVERY_HANDLER x10, x11
+       CLEAR_RECOVERY_HANDLER

        mov             x0, #0
        POP_FRAME
        ARM64_STACK_EPILOG
        mov             x0, #0
        POP_FRAME
        ARM64_STACK_EPILOG


@@ -545,17 +635,7 @@ LEXT(_bcopyout)
 LEXT(_bcopyinstr)
        ARM64_STACK_PROLOG
        PUSH_FRAME
 LEXT(_bcopyinstr)
        ARM64_STACK_PROLOG
        PUSH_FRAME

-       adr             x4, Lcopyinstr_error            // Get address for recover
-       mrs             x10, TPIDR_EL1                          // Get thread pointer
-       ldr             x11, [x10, TH_RECOVER]          // Save previous recover
-
-#if defined(HAS_APPLE_PAC)
-       add             x5, x10, TH_RECOVER             // Sign new pointer with IAKey + blended discriminator
-       movk    x5, #PAC_DISCRIMINATOR_RECOVER, lsl 48
-       pacia   x4, x5
-#endif
-       str             x4, [x10, TH_RECOVER]           // Store new recover
-
+       SET_RECOVERY_HANDLER Lcopyinstr_error, label_in_adr_range=1

        mov             x4, #0                                          // x4 - total bytes copied
 Lcopyinstr_loop:
        ldrb    w5, [x0], #1                                    // Load a byte from the user source
        mov             x4, #0                                          // x4 - total bytes copied
 Lcopyinstr_loop:
        ldrb    w5, [x0], #1                                    // Load a byte from the user source


@@ -573,7 +653,7 @@ Lcopyinstr_done:
 Lcopyinstr_error:
        mov             x0, #EFAULT                                     // Return EFAULT on error
 Lcopyinstr_exit:
 Lcopyinstr_error:
        mov             x0, #EFAULT                                     // Return EFAULT on error
 Lcopyinstr_exit:

-       str             x11, [x10, TH_RECOVER]          // Restore old recover
+       CLEAR_RECOVERY_HANDLER

        POP_FRAME
        ARM64_STACK_EPILOG
 
        POP_FRAME
        ARM64_STACK_EPILOG
 


@@ -589,9 +669,9 @@ Lcopyinstr_exit:
  *     x3 : temp
  *     x5 : temp (kernel virtual base)
  *     x9 : temp
  *     x3 : temp
  *     x5 : temp (kernel virtual base)
  *     x9 : temp

- *     x10 : thread pointer (set by SET_RECOVERY_HANDLER)
- *     x11 : old recovery function (set by SET_RECOVERY_HANDLER)
+ *     x10 : old recovery function (set by SET_RECOVERY_HANDLER)

  *     x12, x13 : backtrace data
  *     x12, x13 : backtrace data

+ *     x16 : thread pointer (set by SET_RECOVERY_HANDLER)

  *
  */
        .text
  *
  */
        .text


@@ -600,7 +680,7 @@ Lcopyinstr_exit:
 LEXT(copyinframe)
        ARM64_STACK_PROLOG
        PUSH_FRAME
 LEXT(copyinframe)
        ARM64_STACK_PROLOG
        PUSH_FRAME

-       SET_RECOVERY_HANDLER x10, x11, x3, copyio_error
+       SET_RECOVERY_HANDLER copyio_error

        cbnz    w2, Lcopyinframe64              // Check frame size
        adrp    x5, EXT(gVirtBase)@page // For 32-bit frame, make sure we're not trying to copy from kernel
        add             x5, x5, EXT(gVirtBase)@pageoff
        cbnz    w2, Lcopyinframe64              // Check frame size
        adrp    x5, EXT(gVirtBase)@page // For 32-bit frame, make sure we're not trying to copy from kernel
        add             x5, x5, EXT(gVirtBase)@pageoff


@@ -631,7 +711,7 @@ Lcopyinframe_valid:
        mov     w0, #0                                  // Success
 
 Lcopyinframe_done:
        mov     w0, #0                                  // Success
 
 Lcopyinframe_done:

-       CLEAR_RECOVERY_HANDLER x10, x11
+       CLEAR_RECOVERY_HANDLER

        POP_FRAME
        ARM64_STACK_EPILOG
 
        POP_FRAME
        ARM64_STACK_EPILOG
 


@@ -690,6 +770,9 @@ LEXT(arm64_prepare_for_sleep)
        orr             x1, x1, #(  ARM64_REG_ACC_OVRD_ok2TrDnLnk_deepsleep)
        and             x1, x1, #(~(ARM64_REG_ACC_OVRD_ok2PwrDnCPM_mask))
        orr             x1, x1, #(  ARM64_REG_ACC_OVRD_ok2PwrDnCPM_deepsleep)
        orr             x1, x1, #(  ARM64_REG_ACC_OVRD_ok2TrDnLnk_deepsleep)
        and             x1, x1, #(~(ARM64_REG_ACC_OVRD_ok2PwrDnCPM_mask))
        orr             x1, x1, #(  ARM64_REG_ACC_OVRD_ok2PwrDnCPM_deepsleep)

+#if HAS_RETENTION_STATE
+       orr             x1, x1, #(ARM64_REG_ACC_OVRD_disPioOnWfiCpu)
+#endif

        msr             ARM64_REG_ACC_OVRD, x1
 
 
        msr             ARM64_REG_ACC_OVRD, x1
 
 


@@ -701,9 +784,12 @@ LEXT(arm64_prepare_for_sleep)
        // Set "OK to power down" (<rdar://problem/12390433>)
        mrs             x0, ARM64_REG_CYC_OVRD
        orr             x0, x0, #(ARM64_REG_CYC_OVRD_ok2pwrdn_force_down)
        // Set "OK to power down" (<rdar://problem/12390433>)
        mrs             x0, ARM64_REG_CYC_OVRD
        orr             x0, x0, #(ARM64_REG_CYC_OVRD_ok2pwrdn_force_down)

+#if HAS_RETENTION_STATE
+       orr             x0, x0, #(ARM64_REG_CYC_OVRD_disWfiRetn)
+#endif

        msr             ARM64_REG_CYC_OVRD, x0
 
        msr             ARM64_REG_CYC_OVRD, x0
 

-#if defined(APPLEMONSOON)
+#if defined(APPLEMONSOON) || defined(APPLEVORTEX)

        ARM64_IS_PCORE x0
        cbz             x0, Lwfi_inst // skip if not p-core 
 
        ARM64_IS_PCORE x0
        cbz             x0, Lwfi_inst // skip if not p-core 
 


@@ -717,6 +803,12 @@ LEXT(arm64_prepare_for_sleep)
         * and re-enabling GUPS, which forces the prefetch queue to
         * drain.  This should be done as close to wfi as possible, i.e.
         * at the very end of arm64_prepare_for_sleep(). */
         * and re-enabling GUPS, which forces the prefetch queue to
         * drain.  This should be done as close to wfi as possible, i.e.
         * at the very end of arm64_prepare_for_sleep(). */

+#if defined(APPLEVORTEX)
+       /* <rdar://problem/32821461>: Cyprus A0/A1 parts have a similar
+        * bug in the HSP prefetcher that can be worked around through
+        * the same method mentioned above for Skye. */
+       SKIP_IF_CPU_VERSION_GREATER_OR_EQUAL x0, VORTEX_CPU_VERSION_B0, Lwfi_inst
+#endif

        mrs             x0, ARM64_REG_HID10
        orr             x0, x0, #(ARM64_REG_HID10_DisHwpGups)
        msr             ARM64_REG_HID10, x0
        mrs             x0, ARM64_REG_HID10
        orr             x0, x0, #(ARM64_REG_HID10_DisHwpGups)
        msr             ARM64_REG_HID10, x0


@@ -750,6 +842,21 @@ LEXT(arm64_force_wfi_clock_gate)
        ARM64_STACK_EPILOG
 
 
        ARM64_STACK_EPILOG
 
 

+#if HAS_RETENTION_STATE
+       .text
+       .align 2
+       .globl EXT(arm64_retention_wfi)
+LEXT(arm64_retention_wfi)
+       wfi
+       cbz             lr, Lwfi_retention      // If lr is 0, we entered retention state and lost all GPRs except sp and pc
+       ret                                     // Otherwise just return to cpu_idle()
+Lwfi_retention:
+       mov             x0, #1
+       bl              EXT(ClearIdlePop)
+       mov             x0, #0 
+       bl              EXT(cpu_idle_exit)      // cpu_idle_exit(from_reset = FALSE)
+       b               .                       // cpu_idle_exit() should never return
+#endif

 
 #if defined(APPLETYPHOON)
 
 
 #if defined(APPLETYPHOON)
 


@@ -931,7 +1038,7 @@ LEXT(arm64_replace_bootstack)
        mrs             x4, DAIF                                        // Load current DAIF; use x4 as pinst may trash x1-x3
        msr             DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF | DAIFSC_ASYNCF)           // Disable IRQ/FIQ/serror
        // Set SP_EL1 to exception stack
        mrs             x4, DAIF                                        // Load current DAIF; use x4 as pinst may trash x1-x3
        msr             DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF | DAIFSC_ASYNCF)           // Disable IRQ/FIQ/serror
        // Set SP_EL1 to exception stack

-#if defined(KERNEL_INTEGRITY_KTRR)
+#if defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR)

        mov             x1, lr
        bl              EXT(pinst_spsel_1)
        mov             lr, x1
        mov             x1, lr
        bl              EXT(pinst_spsel_1)
        mov             lr, x1


@@ -1014,6 +1121,24 @@ Lcheck_hash_panic:
        CALL_EXTERN panic_with_thread_kernel_state
 Lcheck_hash_str:
        .asciz "JOP Hash Mismatch Detected (PC, CPSR, or LR corruption)"
        CALL_EXTERN panic_with_thread_kernel_state
 Lcheck_hash_str:
        .asciz "JOP Hash Mismatch Detected (PC, CPSR, or LR corruption)"

+
+/**
+ * void ml_auth_thread_state_invalid_cpsr(arm_saved_state_t *ss)
+ *
+ * Panics due to an invalid CPSR value in ss.
+ */
+       .text
+       .align 2
+       .globl EXT(ml_auth_thread_state_invalid_cpsr)
+LEXT(ml_auth_thread_state_invalid_cpsr)
+       ARM64_STACK_PROLOG
+       PUSH_FRAME
+       mov             x1, x0
+       adr             x0, Linvalid_cpsr_str
+       CALL_EXTERN panic_with_thread_kernel_state
+
+Linvalid_cpsr_str:
+       .asciz "Thread state corruption detected (PE mode == 0)"

 #endif /* HAS_APPLE_PAC */
 
        .text
 #endif /* HAS_APPLE_PAC */
 
        .text