X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/94ff46dc2849db4d43eaaf144872decc522aafb4..bca245acd4c03fd752d1a45f011ad495e60fe53d:/osfmk/arm64/machine_routines_asm.s diff --git a/osfmk/arm64/machine_routines_asm.s b/osfmk/arm64/machine_routines_asm.s index 5dc6cde73..191997c13 100644 --- a/osfmk/arm64/machine_routines_asm.s +++ b/osfmk/arm64/machine_routines_asm.s @@ -58,7 +58,62 @@ LEXT(ml_set_kernelkey_enabled) #endif /* defined(HAS_APPLE_PAC) */ +#if HAS_BP_RET +/* + * void set_bp_ret(void) + * Helper function to enable branch predictor state retention + * across ACC sleep + */ + + .align 2 + .globl EXT(set_bp_ret) +LEXT(set_bp_ret) + // Load bpret boot-arg + adrp x14, EXT(bp_ret)@page + add x14, x14, EXT(bp_ret)@pageoff + ldr w14, [x14] + + mrs x13, ARM64_REG_ACC_CFG + and x13, x13, (~(ARM64_REG_ACC_CFG_bpSlp_mask << ARM64_REG_ACC_CFG_bpSlp_shift)) + and x14, x14, #(ARM64_REG_ACC_CFG_bpSlp_mask) + orr x13, x13, x14, lsl #(ARM64_REG_ACC_CFG_bpSlp_shift) + msr ARM64_REG_ACC_CFG, x13 + + ret +#endif // HAS_BP_RET + +#if HAS_NEX_PG + .align 2 + .globl EXT(set_nex_pg) +LEXT(set_nex_pg) + mrs x14, MPIDR_EL1 + // Skip if this isn't a p-core; NEX powergating isn't available for e-cores + and x14, x14, #(MPIDR_PNE) + cbz x14, Lnex_pg_done + + // Set the SEG-recommended value of 12 additional reset cycles + mrs x14, ARM64_REG_HID13 + and x14, x14, (~ARM64_REG_HID13_RstCyc_mask) + orr x14, x14, ARM64_REG_HID13_RstCyc_val + msr ARM64_REG_HID13, x14 + + // Load nexpg boot-arg + adrp x14, EXT(nex_pg)@page + add x14, x14, EXT(nex_pg)@pageoff + ldr w14, [x14] + + mrs x13, ARM64_REG_HID14 + and x13, x13, (~ARM64_REG_HID14_NexPwgEn) + cbz w14, Lset_nex_pg + orr x13, x13, ARM64_REG_HID14_NexPwgEn +Lset_nex_pg: + msr ARM64_REG_HID14, x13 + +Lnex_pg_done: + ret + +#endif // HAS_NEX_PG /* uint32_t get_fpscr(void): * Returns (FPSR | FPCR). @@ -168,12 +223,21 @@ LEXT(set_mmu_ttb_alternate) bl EXT(pinst_set_ttbr1) mov lr, x1 #else +#if defined(HAS_VMSA_LOCK) + mrs x1, ARM64_REG_VMSA_LOCK_EL1 + and x1, x1, #(VMSA_LOCK_TTBR1_EL1) + cbnz x1, L_set_locked_reg_panic +#endif /* defined(HAS_VMSA_LOCK) */ msr TTBR1_EL1, x0 #endif /* defined(KERNEL_INTEGRITY_KTRR) */ isb sy ret +#if XNU_MONITOR + .section __PPLTEXT,__text,regular,pure_instructions +#else .text +#endif .align 2 .globl EXT(set_mmu_ttb) LEXT(set_mmu_ttb) @@ -211,6 +275,19 @@ LEXT(set_vbar_el1) #endif #endif /* __ARM_KERNEL_PROTECT__ */ +#if defined(HAS_VMSA_LOCK) + .text + .align 2 + .globl EXT(vmsa_lock) +LEXT(vmsa_lock) + isb sy + mov x1, #(VMSA_LOCK_SCTLR_M_BIT) + mov x0, #(VMSA_LOCK_TTBR1_EL1 | VMSA_LOCK_TCR_EL1 | VMSA_LOCK_VBAR_EL1) + orr x0, x0, x1 + msr ARM64_REG_VMSA_LOCK_EL1, x0 + isb sy + ret +#endif /* defined(HAS_VMSA_LOCK) */ /* * set translation control register @@ -229,6 +306,12 @@ LEXT(set_tcr) bl EXT(pinst_set_tcr) mov lr, x1 #else +#if defined(HAS_VMSA_LOCK) + // assert TCR unlocked + mrs x1, ARM64_REG_VMSA_LOCK_EL1 + and x1, x1, #(VMSA_LOCK_TCR_EL1) + cbnz x1, L_set_locked_reg_panic +#endif /* defined(HAS_VMSA_LOCK) */ msr TCR_EL1, x0 #endif /* defined(KERNEL_INTRITY_KTRR) */ isb sy @@ -256,7 +339,7 @@ L_set_tcr_panic_str: L_set_locked_reg_panic_str: .asciz "attempt to set locked register: (%llx)\n" #else -#if defined(KERNEL_INTEGRITY_KTRR) +#if defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR) mov x1, lr bl EXT(pinst_set_tcr) mov lr, x1 @@ -332,25 +415,32 @@ L_mmu_kvtop_wpreflight_invalid: /* * SET_RECOVERY_HANDLER * - * Sets up a page fault recovery handler + * Sets up a page fault recovery handler. This macro clobbers x16 and x17. * - * arg0 - persisted thread pointer - * arg1 - persisted recovery handler - * arg2 - scratch reg - * arg3 - recovery label + * label - recovery label + * tpidr - persisted thread pointer + * old_handler - persisted recovery handler + * label_in_adr_range - whether \label is within 1 MB of PC */ -.macro SET_RECOVERY_HANDLER - mrs $0, TPIDR_EL1 // Load thread pointer - adrp $2, $3@page // Load the recovery handler address - add $2, $2, $3@pageoff +.macro SET_RECOVERY_HANDLER label, tpidr=x16, old_handler=x10, label_in_adr_range=0 + // Note: x16 and x17 are designated for use as temporaries in + // interruptible PAC routines. DO NOT CHANGE THESE REGISTER ASSIGNMENTS. +.if \label_in_adr_range==1 // Load the recovery handler address + adr x17, \label +.else + adrp x17, \label@page + add x17, x17, \label@pageoff +.endif #if defined(HAS_APPLE_PAC) - add $1, $0, TH_RECOVER - movk $1, #PAC_DISCRIMINATOR_RECOVER, lsl 48 - pacia $2, $1 // Sign with IAKey + blended discriminator + mrs x16, TPIDR_EL1 + add x16, x16, TH_RECOVER + movk x16, #PAC_DISCRIMINATOR_RECOVER, lsl 48 + pacia x17, x16 // Sign with IAKey + blended discriminator #endif - ldr $1, [$0, TH_RECOVER] // Save previous recovery handler - str $2, [$0, TH_RECOVER] // Set new signed recovery handler + mrs \tpidr, TPIDR_EL1 // Load thread pointer + ldr \old_handler, [\tpidr, TH_RECOVER] // Save previous recovery handler + str x17, [\tpidr, TH_RECOVER] // Set new signed recovery handler .endmacro /* @@ -358,18 +448,18 @@ L_mmu_kvtop_wpreflight_invalid: * * Clears page fault handler set by SET_RECOVERY_HANDLER * - * arg0 - thread pointer saved by SET_RECOVERY_HANDLER - * arg1 - old recovery handler saved by SET_RECOVERY_HANDLER + * tpidr - thread pointer saved by SET_RECOVERY_HANDLER + * old_handler - old recovery handler saved by SET_RECOVERY_HANDLER */ -.macro CLEAR_RECOVERY_HANDLER - str $1, [$0, TH_RECOVER] // Restore the previous recovery handler +.macro CLEAR_RECOVERY_HANDLER tpidr=x16, old_handler=x10 + str \old_handler, [\tpidr, TH_RECOVER] // Restore the previous recovery handler .endmacro .text .align 2 copyio_error: - CLEAR_RECOVERY_HANDLER x10, x11 + CLEAR_RECOVERY_HANDLER mov x0, #EFAULT // Return an EFAULT error POP_FRAME ARM64_STACK_EPILOG @@ -383,7 +473,7 @@ copyio_error: LEXT(_bcopyin) ARM64_STACK_PROLOG PUSH_FRAME - SET_RECOVERY_HANDLER x10, x11, x3, copyio_error + SET_RECOVERY_HANDLER copyio_error /* If len is less than 16 bytes, just do a bytewise copy */ cmp x2, #16 b.lt 2f @@ -403,7 +493,7 @@ LEXT(_bcopyin) strb w3, [x1], #1 b.hi 2b 3: - CLEAR_RECOVERY_HANDLER x10, x11 + CLEAR_RECOVERY_HANDLER mov x0, #0 POP_FRAME ARM64_STACK_EPILOG @@ -417,11 +507,11 @@ LEXT(_bcopyin) LEXT(_copyin_atomic32) ARM64_STACK_PROLOG PUSH_FRAME - SET_RECOVERY_HANDLER x10, x11, x3, copyio_error + SET_RECOVERY_HANDLER copyio_error ldr w8, [x0] str w8, [x1] mov x0, #0 - CLEAR_RECOVERY_HANDLER x10, x11 + CLEAR_RECOVERY_HANDLER POP_FRAME ARM64_STACK_EPILOG @@ -434,7 +524,7 @@ LEXT(_copyin_atomic32) LEXT(_copyin_atomic32_wait_if_equals) ARM64_STACK_PROLOG PUSH_FRAME - SET_RECOVERY_HANDLER x10, x11, x3, copyio_error + SET_RECOVERY_HANDLER copyio_error ldxr w8, [x0] cmp w8, w1 mov x0, ESTALE @@ -443,7 +533,7 @@ LEXT(_copyin_atomic32_wait_if_equals) wfe 1: clrex - CLEAR_RECOVERY_HANDLER x10, x11 + CLEAR_RECOVERY_HANDLER POP_FRAME ARM64_STACK_EPILOG @@ -456,11 +546,11 @@ LEXT(_copyin_atomic32_wait_if_equals) LEXT(_copyin_atomic64) ARM64_STACK_PROLOG PUSH_FRAME - SET_RECOVERY_HANDLER x10, x11, x3, copyio_error + SET_RECOVERY_HANDLER copyio_error ldr x8, [x0] str x8, [x1] mov x0, #0 - CLEAR_RECOVERY_HANDLER x10, x11 + CLEAR_RECOVERY_HANDLER POP_FRAME ARM64_STACK_EPILOG @@ -474,10 +564,10 @@ LEXT(_copyin_atomic64) LEXT(_copyout_atomic32) ARM64_STACK_PROLOG PUSH_FRAME - SET_RECOVERY_HANDLER x10, x11, x3, copyio_error + SET_RECOVERY_HANDLER copyio_error str w0, [x1] mov x0, #0 - CLEAR_RECOVERY_HANDLER x10, x11 + CLEAR_RECOVERY_HANDLER POP_FRAME ARM64_STACK_EPILOG @@ -490,10 +580,10 @@ LEXT(_copyout_atomic32) LEXT(_copyout_atomic64) ARM64_STACK_PROLOG PUSH_FRAME - SET_RECOVERY_HANDLER x10, x11, x3, copyio_error + SET_RECOVERY_HANDLER copyio_error str x0, [x1] mov x0, #0 - CLEAR_RECOVERY_HANDLER x10, x11 + CLEAR_RECOVERY_HANDLER POP_FRAME ARM64_STACK_EPILOG @@ -507,7 +597,7 @@ LEXT(_copyout_atomic64) LEXT(_bcopyout) ARM64_STACK_PROLOG PUSH_FRAME - SET_RECOVERY_HANDLER x10, x11, x3, copyio_error + SET_RECOVERY_HANDLER copyio_error /* If len is less than 16 bytes, just do a bytewise copy */ cmp x2, #16 b.lt 2f @@ -527,7 +617,7 @@ LEXT(_bcopyout) strb w3, [x1], #1 b.hi 2b 3: - CLEAR_RECOVERY_HANDLER x10, x11 + CLEAR_RECOVERY_HANDLER mov x0, #0 POP_FRAME ARM64_STACK_EPILOG @@ -545,17 +635,7 @@ LEXT(_bcopyout) LEXT(_bcopyinstr) ARM64_STACK_PROLOG PUSH_FRAME - adr x4, Lcopyinstr_error // Get address for recover - mrs x10, TPIDR_EL1 // Get thread pointer - ldr x11, [x10, TH_RECOVER] // Save previous recover - -#if defined(HAS_APPLE_PAC) - add x5, x10, TH_RECOVER // Sign new pointer with IAKey + blended discriminator - movk x5, #PAC_DISCRIMINATOR_RECOVER, lsl 48 - pacia x4, x5 -#endif - str x4, [x10, TH_RECOVER] // Store new recover - + SET_RECOVERY_HANDLER Lcopyinstr_error, label_in_adr_range=1 mov x4, #0 // x4 - total bytes copied Lcopyinstr_loop: ldrb w5, [x0], #1 // Load a byte from the user source @@ -573,7 +653,7 @@ Lcopyinstr_done: Lcopyinstr_error: mov x0, #EFAULT // Return EFAULT on error Lcopyinstr_exit: - str x11, [x10, TH_RECOVER] // Restore old recover + CLEAR_RECOVERY_HANDLER POP_FRAME ARM64_STACK_EPILOG @@ -589,9 +669,9 @@ Lcopyinstr_exit: * x3 : temp * x5 : temp (kernel virtual base) * x9 : temp - * x10 : thread pointer (set by SET_RECOVERY_HANDLER) - * x11 : old recovery function (set by SET_RECOVERY_HANDLER) + * x10 : old recovery function (set by SET_RECOVERY_HANDLER) * x12, x13 : backtrace data + * x16 : thread pointer (set by SET_RECOVERY_HANDLER) * */ .text @@ -600,7 +680,7 @@ Lcopyinstr_exit: LEXT(copyinframe) ARM64_STACK_PROLOG PUSH_FRAME - SET_RECOVERY_HANDLER x10, x11, x3, copyio_error + SET_RECOVERY_HANDLER copyio_error cbnz w2, Lcopyinframe64 // Check frame size adrp x5, EXT(gVirtBase)@page // For 32-bit frame, make sure we're not trying to copy from kernel add x5, x5, EXT(gVirtBase)@pageoff @@ -631,7 +711,7 @@ Lcopyinframe_valid: mov w0, #0 // Success Lcopyinframe_done: - CLEAR_RECOVERY_HANDLER x10, x11 + CLEAR_RECOVERY_HANDLER POP_FRAME ARM64_STACK_EPILOG @@ -690,6 +770,9 @@ LEXT(arm64_prepare_for_sleep) orr x1, x1, #( ARM64_REG_ACC_OVRD_ok2TrDnLnk_deepsleep) and x1, x1, #(~(ARM64_REG_ACC_OVRD_ok2PwrDnCPM_mask)) orr x1, x1, #( ARM64_REG_ACC_OVRD_ok2PwrDnCPM_deepsleep) +#if HAS_RETENTION_STATE + orr x1, x1, #(ARM64_REG_ACC_OVRD_disPioOnWfiCpu) +#endif msr ARM64_REG_ACC_OVRD, x1 @@ -701,9 +784,12 @@ LEXT(arm64_prepare_for_sleep) // Set "OK to power down" () mrs x0, ARM64_REG_CYC_OVRD orr x0, x0, #(ARM64_REG_CYC_OVRD_ok2pwrdn_force_down) +#if HAS_RETENTION_STATE + orr x0, x0, #(ARM64_REG_CYC_OVRD_disWfiRetn) +#endif msr ARM64_REG_CYC_OVRD, x0 -#if defined(APPLEMONSOON) +#if defined(APPLEMONSOON) || defined(APPLEVORTEX) ARM64_IS_PCORE x0 cbz x0, Lwfi_inst // skip if not p-core @@ -717,6 +803,12 @@ LEXT(arm64_prepare_for_sleep) * and re-enabling GUPS, which forces the prefetch queue to * drain. This should be done as close to wfi as possible, i.e. * at the very end of arm64_prepare_for_sleep(). */ +#if defined(APPLEVORTEX) + /* : Cyprus A0/A1 parts have a similar + * bug in the HSP prefetcher that can be worked around through + * the same method mentioned above for Skye. */ + SKIP_IF_CPU_VERSION_GREATER_OR_EQUAL x0, VORTEX_CPU_VERSION_B0, Lwfi_inst +#endif mrs x0, ARM64_REG_HID10 orr x0, x0, #(ARM64_REG_HID10_DisHwpGups) msr ARM64_REG_HID10, x0 @@ -750,6 +842,21 @@ LEXT(arm64_force_wfi_clock_gate) ARM64_STACK_EPILOG +#if HAS_RETENTION_STATE + .text + .align 2 + .globl EXT(arm64_retention_wfi) +LEXT(arm64_retention_wfi) + wfi + cbz lr, Lwfi_retention // If lr is 0, we entered retention state and lost all GPRs except sp and pc + ret // Otherwise just return to cpu_idle() +Lwfi_retention: + mov x0, #1 + bl EXT(ClearIdlePop) + mov x0, #0 + bl EXT(cpu_idle_exit) // cpu_idle_exit(from_reset = FALSE) + b . // cpu_idle_exit() should never return +#endif #if defined(APPLETYPHOON) @@ -931,7 +1038,7 @@ LEXT(arm64_replace_bootstack) mrs x4, DAIF // Load current DAIF; use x4 as pinst may trash x1-x3 msr DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF | DAIFSC_ASYNCF) // Disable IRQ/FIQ/serror // Set SP_EL1 to exception stack -#if defined(KERNEL_INTEGRITY_KTRR) +#if defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR) mov x1, lr bl EXT(pinst_spsel_1) mov lr, x1 @@ -1014,6 +1121,24 @@ Lcheck_hash_panic: CALL_EXTERN panic_with_thread_kernel_state Lcheck_hash_str: .asciz "JOP Hash Mismatch Detected (PC, CPSR, or LR corruption)" + +/** + * void ml_auth_thread_state_invalid_cpsr(arm_saved_state_t *ss) + * + * Panics due to an invalid CPSR value in ss. + */ + .text + .align 2 + .globl EXT(ml_auth_thread_state_invalid_cpsr) +LEXT(ml_auth_thread_state_invalid_cpsr) + ARM64_STACK_PROLOG + PUSH_FRAME + mov x1, x0 + adr x0, Linvalid_cpsr_str + CALL_EXTERN panic_with_thread_kernel_state + +Linvalid_cpsr_str: + .asciz "Thread state corruption detected (PE mode == 0)" #endif /* HAS_APPLE_PAC */ .text