osfmk/arm64/cswitch.s

   1 /*
   2  * Copyright (c) 2007 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 #include <machine/asm.h>
  29 #include <arm64/machine_machdep.h>
  30 #include <arm64/machine_routines_asm.h>
  31 #include <arm64/proc_reg.h>
  32 #include "assym.s"
  33
  34 /*
  35  * save_general_registers
  36  *
  37  * Saves variable registers to kernel PCB.
  38  *   arg0 - thread_kernel_state pointer
  39  *   arg1 - Scratch register
  40  */
  41
  42 .macro  save_general_registers
  43 /* AAPCS-64 Page 14
  44  *
  45  * A subroutine invocation must preserve the contents of the registers r19-r29
  46  * and SP. We also save IP0 and IP1, as machine_idle uses IP0 for saving the LR.
  47  */
  48         stp             x16, x17, [$0, SS64_X16]
  49         stp             x19, x20, [$0, SS64_X19]
  50         stp             x21, x22, [$0, SS64_X21]
  51         stp             x23, x24, [$0, SS64_X23]
  52         stp             x25, x26, [$0, SS64_X25]
  53         stp             x27, x28, [$0, SS64_X27]
  54         stp             fp, lr, [$0, SS64_FP]
  55 #ifdef HAS_APPLE_PAC
  56         stp             x0, x1, [sp, #-16]!
  57         stp             x2, x3, [sp, #-16]!
  58         stp             x4, x5, [sp, #-16]!
  59
  60         /*
  61          * Arg0: The ARM context pointer
  62          * Arg1: PC value to sign
  63          * Arg2: CPSR value to sign
  64          * Arg3: LR to sign
  65          */
  66         mov             x0, $0
  67         ldr             x1, [x0, SS64_PC]
  68         ldr             w2, [x0, SS64_CPSR]
  69         mov             x3, lr
  70         mov             x4, x16
  71         mov             x5, x17
  72         bl              EXT(ml_sign_thread_state)
  73
  74         ldp             x4, x5, [sp], #16
  75         ldp             x2, x3, [sp], #16
  76         ldp             x0, x1, [sp], #16
  77         ldp             fp, lr, [$0, SS64_FP]
  78 #endif /* defined(HAS_APPLE_PAC) */
  79         mov             $1, sp
  80         str             $1, [$0, SS64_SP]
  81
  82 /* AAPCS-64 Page 14
  83  *
  84  * Registers d8-d15 (s8-s15) must be preserved by a callee across subroutine
  85  * calls; the remaining registers (v0-v7, v16-v31) do not need to be preserved
  86  * (or should be preserved by the caller).
  87  */
  88         str             d8,     [$0, NS64_D8]
  89         str             d9,     [$0, NS64_D9]
  90         str             d10,[$0, NS64_D10]
  91         str             d11,[$0, NS64_D11]
  92         str             d12,[$0, NS64_D12]
  93         str             d13,[$0, NS64_D13]
  94         str             d14,[$0, NS64_D14]
  95         str             d15,[$0, NS64_D15]
  96 .endmacro
  97
  98 /*
  99  * load_general_registers
 100  *
 101  * Loads variable registers from kernel PCB.
 102  *   arg0 - thread_kernel_state pointer
 103  *   arg1 - Scratch register
 104  */
 105 .macro  load_general_registers
 106         mov             x20, x0
 107         mov             x21, x1
 108         mov             x22, x2
 109
 110         mov             x0, $0
 111         AUTH_THREAD_STATE_IN_X0 x23, x24, x25, x26, x27
 112
 113         mov             x0, x20
 114         mov             x1, x21
 115         mov             x2, x22
 116
 117         // Skip x16, x17 - already loaded + authed by AUTH_THREAD_STATE_IN_X0
 118         ldp             x19, x20, [$0, SS64_X19]
 119         ldp             x21, x22, [$0, SS64_X21]
 120         ldp             x23, x24, [$0, SS64_X23]
 121         ldp             x25, x26, [$0, SS64_X25]
 122         ldp             x27, x28, [$0, SS64_X27]
 123         ldr             fp, [$0, SS64_FP]
 124         // Skip lr - already loaded + authed by AUTH_THREAD_STATE_IN_X0
 125         ldr             $1, [$0, SS64_SP]
 126         mov             sp, $1
 127
 128         ldr             d8,     [$0, NS64_D8]
 129         ldr             d9,     [$0, NS64_D9]
 130         ldr             d10,[$0, NS64_D10]
 131         ldr             d11,[$0, NS64_D11]
 132         ldr             d12,[$0, NS64_D12]
 133         ldr             d13,[$0, NS64_D13]
 134         ldr             d14,[$0, NS64_D14]
 135         ldr             d15,[$0, NS64_D15]
 136 .endmacro
 137
 138
 139 /*
 140  * set_thread_registers
 141  *
 142  * Updates thread registers during context switch
 143  *  arg0 - New thread pointer
 144  *  arg1 - Scratch register
 145  *  arg2 - Scratch register
 146  */
 147 .macro  set_thread_registers
 148         msr             TPIDR_EL1, $0                                           // Write new thread pointer to TPIDR_EL1
 149         ldr             $1, [$0, TH_CTH_SELF]                           // Get cthread pointer
 150         mrs             $2, TPIDRRO_EL0                                         // Extract cpu number from TPIDRRO_EL0
 151         and             $2, $2, #(MACHDEP_CPUNUM_MASK)
 152         orr             $2, $1, $2                                                      // Save new cthread/cpu to TPIDRRO_EL0
 153         msr             TPIDRRO_EL0, $2
 154         ldr             $1, [$0, TH_CTH_DATA]                           // Get new cthread data pointer
 155         msr             TPIDR_EL0, $1                                           // Save data pointer to TPIDRRW_EL0
 156         /* ARM64_TODO Reserve x18 until we decide what to do with it */
 157         mov             x18, $1                                                         // ... and trash reserved x18
 158 .endmacro
 159
 160 #if defined(HAS_APPLE_PAC)
 161 /*
 162  * set_process_dependent_keys
 163  *
 164  * Updates process dependent keys during context switch if necessary
 165  *  Per CPU Data rop_key is initialized in arm_init() for bootstrap processor
 166  *  and in cpu_data_init for slave processors
 167  *
 168  *  arg0 - New thread pointer/Current CPU key
 169  *  arg1 - Scratch register: New Thread Key
 170  *  arg2 - Scratch register: Current CPU Data pointer
 171  */
 172 .macro set_process_dependent_keys
 173         ldr             $1, [$0, TH_ROP_PID]
 174         ldr             $2, [$0, ACT_CPUDATAP]
 175         ldr             $0, [$2, CPU_ROP_KEY]
 176         cmp             $0, $1
 177         b.eq    1f
 178         str             $1, [$2, CPU_ROP_KEY]
 179         msr             APIBKeyLo_EL1, $1
 180         add             $1, $1, #1
 181         msr             APIBKeyHi_EL1, $1
 182         add             $1, $1, #1
 183         msr             APDBKeyLo_EL1, $1
 184         add             $1, $1, #1
 185         msr             APDBKeyHi_EL1, $1
 186         isb     sy
 187 1:
 188 .endmacro
 189 #endif /* defined(HAS_APPLE_PAC) */
 190
 191 /*
 192  * void     machine_load_context(thread_t        thread)
 193  *
 194  * Load the context for the first thread to run on a
 195  * cpu, and go.
 196  */
 197         .text
 198         .align 2
 199         .globl  EXT(machine_load_context)
 200
 201 LEXT(machine_load_context)
 202         set_thread_registers    x0, x1, x2
 203         ldr             x1, [x0, TH_KSTACKPTR]                          // Get top of kernel stack
 204         load_general_registers  x1, x2
 205 #ifdef HAS_APPLE_PAC
 206         set_process_dependent_keys      x0, x1, x2
 207 #endif
 208         mov             x0, #0                                                          // Clear argument to thread_continue
 209         ret
 210
 211 /*
 212  *  typedef void (*thread_continue_t)(void *param, wait_result_t)
 213  *
 214  *      void Call_continuation( thread_continue_t continuation,
 215  *                                      void *param,
 216  *                                          wait_result_t wresult,
 217  *                          bool enable interrupts)
 218  */
 219         .text
 220         .align  5
 221         .globl  EXT(Call_continuation)
 222
 223 LEXT(Call_continuation)
 224         mrs             x4, TPIDR_EL1                                           // Get the current thread pointer
 225
 226         /* ARM64_TODO arm loads the kstack top instead of arg4. What should we use? */
 227         ldr             x5, [x4, TH_KSTACKPTR]                          // Get the top of the kernel stack
 228         mov             sp, x5                                                          // Set stack pointer
 229         mov             fp, #0                                                          // Clear the frame pointer
 230
 231 #if defined(HAS_APPLE_PAC)
 232         set_process_dependent_keys      x4, x5, x6
 233 #endif
 234
 235     mov x20, x0  //continuation
 236     mov x21, x1  //continuation parameter
 237     mov x22, x2  //wait result
 238
 239     cbz x3, 1f
 240     mov x0, #1
 241     bl EXT(ml_set_interrupts_enabled)
 242 1:
 243
 244         mov             x0, x21                                                         // Set the first parameter
 245         mov             x1, x22                                                         // Set the wait result arg
 246 #ifdef HAS_APPLE_PAC
 247         blraaz  x20                                                                     // Branch to the continuation
 248 #else
 249         blr             x20                                                                     // Branch to the continuation
 250 #endif
 251         mrs             x0, TPIDR_EL1                                           // Get the current thread pointer
 252         b               EXT(thread_terminate)                           // Kill the thread
 253
 254
 255 /*
 256  *      thread_t Switch_context(thread_t        old,
 257  *                              void            (*cont)(void),
 258  *                              thread_t        new)
 259  */
 260         .text
 261         .align 5
 262         .globl  EXT(Switch_context)
 263
 264 LEXT(Switch_context)
 265         cbnz    x1, Lswitch_threads                                     // Skip saving old state if blocking on continuation
 266         ldr             x3, [x0, TH_KSTACKPTR]                          // Get the old kernel stack top
 267         save_general_registers  x3, x4
 268 Lswitch_threads:
 269         set_thread_registers    x2, x3, x4
 270         ldr             x3, [x2, TH_KSTACKPTR]
 271         load_general_registers  x3, x4
 272 #if defined(HAS_APPLE_PAC)
 273         set_process_dependent_keys      x2, x3, x4
 274 #endif
 275         ret
 276
 277 /*
 278  *      thread_t Shutdown_context(void (*doshutdown)(processor_t), processor_t processor)
 279  *
 280  */
 281         .text
 282         .align 5
 283         .globl  EXT(Shutdown_context)
 284
 285 LEXT(Shutdown_context)
 286         mrs             x10, TPIDR_EL1                                                  // Get thread pointer
 287         ldr             x11, [x10, TH_KSTACKPTR]                                // Get the top of the kernel stack
 288         save_general_registers  x11, x12
 289         msr             DAIFSet, #(DAIFSC_FIQF | DAIFSC_IRQF)   // Disable interrupts
 290         ldr             x11, [x10, ACT_CPUDATAP]                                // Get current cpu
 291         ldr             x12, [x11, CPU_ISTACKPTR]                               // Switch to interrupt stack
 292         mov             sp, x12
 293         b               EXT(cpu_doshutdown)
 294
 295 /*
 296  *      thread_t Idle_context(void)
 297  *
 298  */
 299         .text
 300         .align 5
 301         .globl  EXT(Idle_context)
 302
 303 LEXT(Idle_context)
 304         mrs             x0, TPIDR_EL1                                           // Get thread pointer
 305         ldr             x1, [x0, TH_KSTACKPTR]                          // Get the top of the kernel stack
 306         save_general_registers  x1, x2
 307         ldr             x1, [x0, ACT_CPUDATAP]                          // Get current cpu
 308         ldr             x2, [x1, CPU_ISTACKPTR]                         // Switch to interrupt stack
 309         mov             sp, x2
 310         b               EXT(cpu_idle)
 311
 312 /*
 313  *      thread_t Idle_context(void)
 314  *
 315  */
 316         .text
 317         .align 5
 318         .globl  EXT(Idle_load_context)
 319
 320 LEXT(Idle_load_context)
 321         mrs             x0, TPIDR_EL1                                           // Get thread pointer
 322         ldr             x1, [x0, TH_KSTACKPTR]                          // Get the top of the kernel stack
 323         load_general_registers  x1, x2
 324 #ifdef HAS_APPLE_PAC
 325         set_process_dependent_keys      x0, x1, x2
 326 #endif
 327         ret
 328
 329         .align  2
 330         .globl  EXT(machine_set_current_thread)
 331 LEXT(machine_set_current_thread)
 332         set_thread_registers x0, x1, x2
 333         ret
 334
 335