xnu-6153.121.1.tar.gz

[apple/xnu.git] / osfmk / arm64 / cswitch.s

diff --git a/osfmk/arm64/cswitch.s b/osfmk/arm64/cswitch.s
index e3a0cb317dc8e819e7279c2f21cd7fcca72965e8..c9b1893a1f8d63c4b922fe76ab0e7dd7e2b7bb22 100644 (file)
--- a/osfmk/arm64/cswitch.s
+++ b/osfmk/arm64/cswitch.s
@@ -27,6 +27,7 @@
  */
 #include <machine/asm.h>
 #include <arm64/machine_machdep.h>
  */
 #include <machine/asm.h>
 #include <arm64/machine_machdep.h>

+#include <arm64/machine_routines_asm.h>

 #include <arm64/proc_reg.h>
 #include "assym.s"
 
 #include <arm64/proc_reg.h>
 #include "assym.s"
 


@@ -51,6 +52,30 @@
        stp             x25, x26, [$0, SS64_X25]
        stp             x27, x28, [$0, SS64_X27]
        stp             fp, lr, [$0, SS64_FP]
        stp             x25, x26, [$0, SS64_X25]
        stp             x27, x28, [$0, SS64_X27]
        stp             fp, lr, [$0, SS64_FP]

+#ifdef HAS_APPLE_PAC
+       stp             x0, x1, [sp, #-16]!
+       stp             x2, x3, [sp, #-16]!
+       stp             x4, x5, [sp, #-16]!
+
+       /*
+        * Arg0: The ARM context pointer
+        * Arg1: PC value to sign
+        * Arg2: CPSR value to sign
+        * Arg3: LR to sign
+        */
+       mov             x0, $0
+       ldr             x1, [x0, SS64_PC]
+       ldr             w2, [x0, SS64_CPSR]
+       mov             x3, lr
+       mov             x4, x16
+       mov             x5, x17
+       bl              EXT(ml_sign_thread_state)
+
+       ldp             x4, x5, [sp], #16
+       ldp             x2, x3, [sp], #16
+       ldp             x0, x1, [sp], #16
+       ldp             fp, lr, [$0, SS64_FP]
+#endif /* defined(HAS_APPLE_PAC) */

        mov             $1, sp
        str             $1, [$0, SS64_SP]
 
        mov             $1, sp
        str             $1, [$0, SS64_SP]
 


@@ -78,13 +103,25 @@
  *   arg1 - Scratch register
  */
 .macro load_general_registers
  *   arg1 - Scratch register
  */
 .macro load_general_registers

-       ldp             x16, x17, [$0, SS64_X16]
+       mov             x20, x0
+       mov             x21, x1
+       mov             x22, x2
+
+       mov             x0, $0
+       AUTH_THREAD_STATE_IN_X0 x23, x24, x25, x26, x27
+
+       mov             x0, x20
+       mov             x1, x21
+       mov             x2, x22
+
+       // Skip x16, x17 - already loaded + authed by AUTH_THREAD_STATE_IN_X0

        ldp             x19, x20, [$0, SS64_X19]
        ldp             x21, x22, [$0, SS64_X21]
        ldp             x23, x24, [$0, SS64_X23]
        ldp             x25, x26, [$0, SS64_X25]
        ldp             x27, x28, [$0, SS64_X27]
        ldp             x19, x20, [$0, SS64_X19]
        ldp             x21, x22, [$0, SS64_X21]
        ldp             x23, x24, [$0, SS64_X23]
        ldp             x25, x26, [$0, SS64_X25]
        ldp             x27, x28, [$0, SS64_X27]

-       ldp             fp, lr, [$0, SS64_FP]
+       ldr             fp, [$0, SS64_FP]
+       // Skip lr - already loaded + authed by AUTH_THREAD_STATE_IN_X0

        ldr             $1, [$0, SS64_SP]
        mov             sp, $1
 
        ldr             $1, [$0, SS64_SP]
        mov             sp, $1
 


@@ -98,6 +135,7 @@
        ldr             d15,[$0, NS64_D15]
 .endmacro
 
        ldr             d15,[$0, NS64_D15]
 .endmacro
 

+

 /*
  * set_thread_registers
  *
 /*
  * set_thread_registers
  *


@@ -108,17 +146,48 @@
  */
 .macro set_thread_registers
        msr             TPIDR_EL1, $0                                           // Write new thread pointer to TPIDR_EL1
  */
 .macro set_thread_registers
        msr             TPIDR_EL1, $0                                           // Write new thread pointer to TPIDR_EL1

+       ldr             $1, [$0, ACT_CPUDATAP]
+       str             $0, [$1, CPU_ACTIVE_THREAD]

        ldr             $1, [$0, TH_CTH_SELF]                           // Get cthread pointer
        mrs             $2, TPIDRRO_EL0                                         // Extract cpu number from TPIDRRO_EL0
        and             $2, $2, #(MACHDEP_CPUNUM_MASK)
        orr             $2, $1, $2                                                      // Save new cthread/cpu to TPIDRRO_EL0
        msr             TPIDRRO_EL0, $2
        ldr             $1, [$0, TH_CTH_SELF]                           // Get cthread pointer
        mrs             $2, TPIDRRO_EL0                                         // Extract cpu number from TPIDRRO_EL0
        and             $2, $2, #(MACHDEP_CPUNUM_MASK)
        orr             $2, $1, $2                                                      // Save new cthread/cpu to TPIDRRO_EL0
        msr             TPIDRRO_EL0, $2

-       ldr             $1, [$0, TH_CTH_DATA]                           // Get new cthread data pointer
-       msr             TPIDR_EL0, $1                                           // Save data pointer to TPIDRRW_EL0
+       msr             TPIDR_EL0, xzr

        /* ARM64_TODO Reserve x18 until we decide what to do with it */
        mov             x18, $1                                                         // ... and trash reserved x18
 .endmacro
 
        /* ARM64_TODO Reserve x18 until we decide what to do with it */
        mov             x18, $1                                                         // ... and trash reserved x18
 .endmacro
 

+#if defined(HAS_APPLE_PAC)
+/*
+ * set_process_dependent_keys
+ *
+ * Updates process dependent keys during context switch if necessary
+ *  Per CPU Data rop_key is initialized in arm_init() for bootstrap processor
+ *  and in cpu_data_init for slave processors
+ *
+ *  arg0 - New thread pointer/Current CPU key
+ *  arg1 - Scratch register: New Thread Key
+ *  arg2 - Scratch register: Current CPU Data pointer
+ */
+.macro set_process_dependent_keys
+       ldr             $1, [$0, TH_ROP_PID]
+       ldr             $2, [$0, ACT_CPUDATAP]
+       ldr             $0, [$2, CPU_ROP_KEY]
+       cmp             $0, $1
+       b.eq    1f
+       str             $1, [$2, CPU_ROP_KEY]
+       msr             APIBKeyLo_EL1, $1
+       add             $1, $1, #1
+       msr             APIBKeyHi_EL1, $1
+       add             $1, $1, #1
+       msr             APDBKeyLo_EL1, $1
+       add             $1, $1, #1
+       msr             APDBKeyHi_EL1, $1
+       isb     sy
+1:
+.endmacro
+#endif /* defined(HAS_APPLE_PAC) */

 
 /*
  * void     machine_load_context(thread_t        thread)
 
 /*
  * void     machine_load_context(thread_t        thread)


@@ -134,14 +203,19 @@ LEXT(machine_load_context)
        set_thread_registers    x0, x1, x2
        ldr             x1, [x0, TH_KSTACKPTR]                          // Get top of kernel stack
        load_general_registers  x1, x2
        set_thread_registers    x0, x1, x2
        ldr             x1, [x0, TH_KSTACKPTR]                          // Get top of kernel stack
        load_general_registers  x1, x2

-       mov             x0, xzr                                                         // Clear argument to thread_continue
+#ifdef HAS_APPLE_PAC
+       set_process_dependent_keys      x0, x1, x2
+#endif
+       mov             x0, #0                                                          // Clear argument to thread_continue

        ret
 
 /*
        ret
 
 /*

- *     void Call_continuation( void (*continuation)(void), 
- *                             void *param, 
- *                             wait_result_t wresult, 
- *                             vm_offset_t stack_ptr)
+ *  typedef void (*thread_continue_t)(void *param, wait_result_t)
+ *
+ *     void Call_continuation( thread_continue_t continuation,
+ *                                     void *param,
+ *                                         wait_result_t wresult,
+ *                          bool enable interrupts)

  */
        .text
        .align  5
  */
        .text
        .align  5


@@ -153,12 +227,28 @@ LEXT(Call_continuation)
        /* ARM64_TODO arm loads the kstack top instead of arg4. What should we use? */
        ldr             x5, [x4, TH_KSTACKPTR]                          // Get the top of the kernel stack
        mov             sp, x5                                                          // Set stack pointer
        /* ARM64_TODO arm loads the kstack top instead of arg4. What should we use? */
        ldr             x5, [x4, TH_KSTACKPTR]                          // Get the top of the kernel stack
        mov             sp, x5                                                          // Set stack pointer

+       mov             fp, #0                                                          // Clear the frame pointer
+
+#if defined(HAS_APPLE_PAC)
+       set_process_dependent_keys      x4, x5, x6
+#endif

 
 

-       mov             fp, xzr                                                         // Clear the frame pointer
-       mov             x4, x0                                                          // Load the continuation
-       mov             x0, x1                                                          // Set the first parameter
-       mov             x1, x2                                                          // Set the wait result arg
-       blr             x4                                                                      // Branch to the continuation
+    mov x20, x0  //continuation
+    mov x21, x1  //continuation parameter
+    mov x22, x2  //wait result
+
+    cbz x3, 1f
+    mov x0, #1
+    bl EXT(ml_set_interrupts_enabled)
+1:
+
+       mov             x0, x21                                                         // Set the first parameter
+       mov             x1, x22                                                         // Set the wait result arg
+#ifdef HAS_APPLE_PAC
+       blraaz  x20                                                                     // Branch to the continuation
+#else
+       blr             x20                                                                     // Branch to the continuation
+#endif

        mrs             x0, TPIDR_EL1                                           // Get the current thread pointer
        b               EXT(thread_terminate)                           // Kill the thread
 
        mrs             x0, TPIDR_EL1                                           // Get the current thread pointer
        b               EXT(thread_terminate)                           // Kill the thread
 


@@ -180,6 +270,9 @@ Lswitch_threads:
        set_thread_registers    x2, x3, x4
        ldr             x3, [x2, TH_KSTACKPTR]
        load_general_registers  x3, x4
        set_thread_registers    x2, x3, x4
        ldr             x3, [x2, TH_KSTACKPTR]
        load_general_registers  x3, x4

+#if defined(HAS_APPLE_PAC)
+       set_process_dependent_keys      x2, x3, x4
+#endif

        ret
 
 /*
        ret
 
 /*


@@ -200,7 +293,6 @@ LEXT(Shutdown_context)
        mov             sp, x12
        b               EXT(cpu_doshutdown)
 
        mov             sp, x12
        b               EXT(cpu_doshutdown)
 

-

 /*
  *     thread_t Idle_context(void)
  *
 /*
  *     thread_t Idle_context(void)
  *


@@ -230,6 +322,9 @@ LEXT(Idle_load_context)
        mrs             x0, TPIDR_EL1                                           // Get thread pointer
        ldr             x1, [x0, TH_KSTACKPTR]                          // Get the top of the kernel stack
        load_general_registers  x1, x2
        mrs             x0, TPIDR_EL1                                           // Get thread pointer
        ldr             x1, [x0, TH_KSTACKPTR]                          // Get the top of the kernel stack
        load_general_registers  x1, x2

+#ifdef HAS_APPLE_PAC
+       set_process_dependent_keys      x0, x1, x2
+#endif

        ret
 
        .align  2
        ret
 
        .align  2


@@ -237,3 +332,5 @@ LEXT(Idle_load_context)
 LEXT(machine_set_current_thread)
        set_thread_registers x0, x1, x2
        ret
 LEXT(machine_set_current_thread)
        set_thread_registers x0, x1, x2
        ret

+
+