osfmk/arm64/cswitch.s

   1 /*
   2  * Copyright (c) 2007 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 #include <machine/asm.h>
  29 #include <arm64/machine_machdep.h>
  30 #include <arm64/machine_routines_asm.h>
  31 #include <arm64/pac_asm.h>
  32 #include <arm64/proc_reg.h>
  33 #include "assym.s"
  34
  35 /*
  36  * save_general_registers
  37  *
  38  * Saves variable registers to kernel PCB.
  39  *   arg0 - thread_kernel_state pointer
  40  *   arg1 - Scratch register
  41  */
  42
  43 .macro  save_general_registers
  44 /* AAPCS-64 Page 14
  45  *
  46  * A subroutine invocation must preserve the contents of the registers r19-r29
  47  * and SP. We also save IP0 and IP1, as machine_idle uses IP0 for saving the LR.
  48  */
  49         stp             x16, x17, [$0, SS64_KERNEL_X16]
  50         stp             x19, x20, [$0, SS64_KERNEL_X19]
  51         stp             x21, x22, [$0, SS64_KERNEL_X21]
  52         stp             x23, x24, [$0, SS64_KERNEL_X23]
  53         stp             x25, x26, [$0, SS64_KERNEL_X25]
  54         stp             x27, x28, [$0, SS64_KERNEL_X27]
  55         stp             fp, lr, [$0, SS64_KERNEL_FP]
  56         str             xzr, [$0, SS64_KERNEL_PC]
  57         MOV32   w$1, PSR64_KERNEL_POISON
  58         str             w$1, [$0, SS64_KERNEL_CPSR]
  59 #ifdef HAS_APPLE_PAC
  60         stp             x0, x1, [sp, #-16]!
  61         stp             x2, x3, [sp, #-16]!
  62         stp             x4, x5, [sp, #-16]!
  63
  64         /*
  65          * Arg0: The ARM context pointer
  66          * Arg1: PC value to sign
  67          * Arg2: CPSR value to sign
  68          * Arg3: LR to sign
  69          */
  70         mov             x0, $0
  71         mov             x1, #0
  72         mov             w2, w$1
  73         mov             x3, lr
  74         mov             x4, x16
  75         mov             x5, x17
  76         bl              EXT(ml_sign_kernel_thread_state)
  77
  78         ldp             x4, x5, [sp], #16
  79         ldp             x2, x3, [sp], #16
  80         ldp             x0, x1, [sp], #16
  81         ldp             fp, lr, [$0, SS64_KERNEL_FP]
  82 #endif /* defined(HAS_APPLE_PAC) */
  83         mov             x$1, sp
  84         str             x$1, [$0, SS64_KERNEL_SP]
  85
  86 /* AAPCS-64 Page 14
  87  *
  88  * Registers d8-d15 (s8-s15) must be preserved by a callee across subroutine
  89  * calls; the remaining registers (v0-v7, v16-v31) do not need to be preserved
  90  * (or should be preserved by the caller).
  91  */
  92         str             d8,     [$0, NS64_KERNEL_D8]
  93         str             d9,     [$0, NS64_KERNEL_D9]
  94         str             d10,[$0, NS64_KERNEL_D10]
  95         str             d11,[$0, NS64_KERNEL_D11]
  96         str             d12,[$0, NS64_KERNEL_D12]
  97         str             d13,[$0, NS64_KERNEL_D13]
  98         str             d14,[$0, NS64_KERNEL_D14]
  99         str             d15,[$0, NS64_KERNEL_D15]
 100
 101         mrs             x$1, FPCR
 102         str             w$1, [$0, NS64_KERNEL_FPCR]
 103 .endmacro
 104
 105 /*
 106  * load_general_registers
 107  *
 108  * Loads variable registers from kernel PCB.
 109  *   arg0 - thread_kernel_state pointer
 110  *   arg1 - Scratch register
 111  */
 112 .macro  load_general_registers
 113         mov             x20, x0
 114         mov             x21, x1
 115         mov             x22, x2
 116
 117         mov             x0, $0
 118         AUTH_KERNEL_THREAD_STATE_IN_X0  x23, x24, x25, x26, x27
 119
 120         mov             x0, x20
 121         mov             x1, x21
 122         mov             x2, x22
 123
 124         ldr             w$1, [$0, NS64_KERNEL_FPCR]
 125         mrs             x19, FPCR
 126         CMSR FPCR, x19, x$1, 1
 127 1:
 128
 129         // Skip x16, x17 - already loaded + authed by AUTH_THREAD_STATE_IN_X0
 130         ldp             x19, x20, [$0, SS64_KERNEL_X19]
 131         ldp             x21, x22, [$0, SS64_KERNEL_X21]
 132         ldp             x23, x24, [$0, SS64_KERNEL_X23]
 133         ldp             x25, x26, [$0, SS64_KERNEL_X25]
 134         ldp             x27, x28, [$0, SS64_KERNEL_X27]
 135         ldr             fp, [$0, SS64_KERNEL_FP]
 136         // Skip lr - already loaded + authed by AUTH_THREAD_STATE_IN_X0
 137         ldr             x$1, [$0, SS64_KERNEL_SP]
 138         mov             sp, x$1
 139
 140         ldr             d8,     [$0, NS64_KERNEL_D8]
 141         ldr             d9,     [$0, NS64_KERNEL_D9]
 142         ldr             d10,[$0, NS64_KERNEL_D10]
 143         ldr             d11,[$0, NS64_KERNEL_D11]
 144         ldr             d12,[$0, NS64_KERNEL_D12]
 145         ldr             d13,[$0, NS64_KERNEL_D13]
 146         ldr             d14,[$0, NS64_KERNEL_D14]
 147         ldr             d15,[$0, NS64_KERNEL_D15]
 148 .endmacro
 149
 150
 151 /*
 152  * set_thread_registers
 153  *
 154  * Updates thread registers during context switch
 155  *  arg0 - New thread pointer
 156  *  arg1 - Scratch register
 157  *  arg2 - Scratch register
 158  */
 159 .macro  set_thread_registers
 160         msr             TPIDR_EL1, $0                                           // Write new thread pointer to TPIDR_EL1
 161         ldr             $1, [$0, ACT_CPUDATAP]
 162         str             $0, [$1, CPU_ACTIVE_THREAD]
 163         ldr             $1, [$0, TH_CTH_SELF]                           // Get cthread pointer
 164         mrs             $2, TPIDRRO_EL0                                         // Extract cpu number from TPIDRRO_EL0
 165         and             $2, $2, #(MACHDEP_CPUNUM_MASK)
 166         orr             $2, $1, $2                                                      // Save new cthread/cpu to TPIDRRO_EL0
 167         msr             TPIDRRO_EL0, $2
 168         msr             TPIDR_EL0, xzr
 169 #if DEBUG || DEVELOPMENT
 170         ldr             $1, [$0, TH_THREAD_ID]                          // Save the bottom 32-bits of the thread ID into
 171         msr             CONTEXTIDR_EL1, $1                                      // CONTEXTIDR_EL1 (top 32-bits are RES0).
 172 #endif /* DEBUG || DEVELOPMENT */
 173 .endmacro
 174
 175 /*
 176  * set_process_dependent_keys_and_sync_context
 177  *
 178  * Updates process dependent keys and issues explicit context sync during context switch if necessary
 179  *  Per CPU Data rop_key is initialized in arm_init() for bootstrap processor
 180  *  and in cpu_data_init for slave processors
 181  *
 182  *  thread - New thread pointer
 183  *  new_key - Scratch register: New Thread Key
 184  *  tmp_key - Scratch register: Current CPU Key
 185  *  cpudatap - Scratch register: Current CPU Data pointer
 186  *  wsync - Half-width scratch register: CPU sync required flag
 187  *
 188  *  to save on ISBs, for ARMv8.5 we use the CPU_SYNC_ON_CSWITCH field, cached in wsync, for pre-ARMv8.5,
 189  *  we just use wsync to keep track of needing an ISB
 190  */
 191 .macro set_process_dependent_keys_and_sync_context      thread, new_key, tmp_key, cpudatap, wsync
 192
 193
 194 #if defined(__ARM_ARCH_8_5__) || defined(HAS_APPLE_PAC)
 195         ldr             \cpudatap, [\thread, ACT_CPUDATAP]
 196 #endif /* defined(__ARM_ARCH_8_5__) || defined(HAS_APPLE_PAC) */
 197
 198 #if defined(__ARM_ARCH_8_5__)
 199         ldrb    \wsync, [\cpudatap, CPU_SYNC_ON_CSWITCH]
 200 #else /* defined(__ARM_ARCH_8_5__) */
 201         mov             \wsync, #0
 202 #endif
 203
 204
 205 #if defined(HAS_APPLE_PAC)
 206         ldr             \new_key, [\thread, TH_ROP_PID]
 207         ldr             \tmp_key, [\cpudatap, CPU_ROP_KEY]
 208         cmp             \new_key, \tmp_key
 209         b.eq    1f
 210         str             \new_key, [\cpudatap, CPU_ROP_KEY]
 211         msr             APIBKeyLo_EL1, \new_key
 212         add             \new_key, \new_key, #1
 213         msr             APIBKeyHi_EL1, \new_key
 214         add             \new_key, \new_key, #1
 215         msr             APDBKeyLo_EL1, \new_key
 216         add             \new_key, \new_key, #1
 217         msr             APDBKeyHi_EL1, \new_key
 218         mov             \wsync, #1
 219 1:
 220
 221 #if HAS_PAC_FAST_A_KEY_SWITCHING
 222         IF_PAC_SLOW_A_KEY_SWITCHING     Lskip_jop_keys_\@, \new_key
 223         ldr             \new_key, [\thread, TH_JOP_PID]
 224         REPROGRAM_JOP_KEYS      Lskip_jop_keys_\@, \new_key, \cpudatap, \tmp_key
 225         mov             \wsync, #1
 226 Lskip_jop_keys_\@:
 227 #endif /* HAS_PAC_FAST_A_KEY_SWITCHING */
 228
 229 #endif /* defined(HAS_APPLE_PAC) */
 230
 231         cbz             \wsync, 1f
 232         isb     sy
 233
 234 #if defined(__ARM_ARCH_8_5__)
 235         strb    wzr, [\cpudatap, CPU_SYNC_ON_CSWITCH]
 236 #endif
 237 1:
 238 .endmacro
 239
 240 /*
 241  * void     machine_load_context(thread_t        thread)
 242  *
 243  * Load the context for the first thread to run on a
 244  * cpu, and go.
 245  */
 246         .text
 247         .align 2
 248         .globl  EXT(machine_load_context)
 249
 250 LEXT(machine_load_context)
 251         set_thread_registers    x0, x1, x2
 252         ldr             x1, [x0, TH_KSTACKPTR]                          // Get top of kernel stack
 253         load_general_registers  x1, 2
 254         set_process_dependent_keys_and_sync_context     x0, x1, x2, x3, w4
 255         mov             x0, #0                                                          // Clear argument to thread_continue
 256         ret
 257
 258 /*
 259  *  typedef void (*thread_continue_t)(void *param, wait_result_t)
 260  *
 261  *      void Call_continuation( thread_continue_t continuation,
 262  *                                      void *param,
 263  *                                          wait_result_t wresult,
 264  *                          bool enable interrupts)
 265  */
 266         .text
 267         .align  5
 268         .globl  EXT(Call_continuation)
 269
 270 LEXT(Call_continuation)
 271         mrs             x4, TPIDR_EL1                                           // Get the current thread pointer
 272
 273         /* ARM64_TODO arm loads the kstack top instead of arg4. What should we use? */
 274         ldr             x5, [x4, TH_KSTACKPTR]                          // Get the top of the kernel stack
 275         mov             sp, x5                                                          // Set stack pointer
 276         mov             fp, #0                                                          // Clear the frame pointer
 277
 278         set_process_dependent_keys_and_sync_context     x4, x5, x6, x7, w20
 279
 280         mov x20, x0  //continuation
 281         mov x21, x1  //continuation parameter
 282         mov x22, x2  //wait result
 283
 284         cbz x3, 1f
 285         mov x0, #1
 286         bl EXT(ml_set_interrupts_enabled)
 287 1:
 288
 289         mov             x0, x21                                                         // Set the first parameter
 290         mov             x1, x22                                                         // Set the wait result arg
 291 #ifdef HAS_APPLE_PAC
 292         mov             x21, THREAD_CONTINUE_T_DISC
 293         blraa   x20, x21                                                        // Branch to the continuation
 294 #else
 295         blr             x20                                                                     // Branch to the continuation
 296 #endif
 297         mrs             x0, TPIDR_EL1                                           // Get the current thread pointer
 298         b               EXT(thread_terminate)                           // Kill the thread
 299
 300
 301 /*
 302  *      thread_t Switch_context(thread_t        old,
 303  *                              void            (*cont)(void),
 304  *                              thread_t        new)
 305  */
 306         .text
 307         .align 5
 308         .globl  EXT(Switch_context)
 309
 310 LEXT(Switch_context)
 311         cbnz    x1, Lswitch_threads                                     // Skip saving old state if blocking on continuation
 312         ldr             x3, [x0, TH_KSTACKPTR]                          // Get the old kernel stack top
 313         save_general_registers  x3, 4
 314 Lswitch_threads:
 315         set_thread_registers    x2, x3, x4
 316         ldr             x3, [x2, TH_KSTACKPTR]
 317         load_general_registers  x3, 4
 318         set_process_dependent_keys_and_sync_context     x2, x3, x4, x5, w6
 319         ret
 320
 321 /*
 322  *      thread_t Shutdown_context(void (*doshutdown)(processor_t), processor_t processor)
 323  *
 324  */
 325         .text
 326         .align 5
 327         .globl  EXT(Shutdown_context)
 328
 329 LEXT(Shutdown_context)
 330         mrs             x10, TPIDR_EL1                                                  // Get thread pointer
 331         ldr             x11, [x10, TH_KSTACKPTR]                                // Get the top of the kernel stack
 332         save_general_registers  x11, 12
 333         msr             DAIFSet, #(DAIFSC_FIQF | DAIFSC_IRQF)   // Disable interrupts
 334         ldr             x11, [x10, ACT_CPUDATAP]                                // Get current cpu
 335         ldr             x12, [x11, CPU_ISTACKPTR]                               // Switch to interrupt stack
 336         mov             sp, x12
 337         b               EXT(cpu_doshutdown)
 338
 339 /*
 340  *      thread_t Idle_context(void)
 341  *
 342  */
 343         .text
 344         .align 5
 345         .globl  EXT(Idle_context)
 346
 347 LEXT(Idle_context)
 348         mrs             x0, TPIDR_EL1                                           // Get thread pointer
 349         ldr             x1, [x0, TH_KSTACKPTR]                          // Get the top of the kernel stack
 350         save_general_registers  x1, 2
 351         ldr             x1, [x0, ACT_CPUDATAP]                          // Get current cpu
 352         ldr             x2, [x1, CPU_ISTACKPTR]                         // Switch to interrupt stack
 353         mov             sp, x2
 354         b               EXT(cpu_idle)
 355
 356 /*
 357  *      thread_t Idle_context(void)
 358  *
 359  */
 360         .text
 361         .align 5
 362         .globl  EXT(Idle_load_context)
 363
 364 LEXT(Idle_load_context)
 365         mrs             x0, TPIDR_EL1                                           // Get thread pointer
 366         ldr             x1, [x0, TH_KSTACKPTR]                          // Get the top of the kernel stack
 367         load_general_registers  x1, 2
 368         set_process_dependent_keys_and_sync_context     x0, x1, x2, x3, w4
 369         ret
 370
 371         .align  2
 372         .globl  EXT(machine_set_current_thread)
 373 LEXT(machine_set_current_thread)
 374         set_thread_registers x0, x1, x2
 375         ret
 376
 377
 378 /* vim: set ts=4: */