[apple/xnu.git] / osfmk / arm64 / cswitch.s

/*
 * Copyright (c) 2007 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
#include <machine/asm.h>
#include <arm64/machine_machdep.h>
#include <arm64/machine_routines_asm.h>
#include <arm64/pac_asm.h>
#include <arm64/proc_reg.h>
#include "assym.s"

/*
 * save_general_registers
 *
 * Saves variable registers to kernel PCB.
 *   arg0 - thread_kernel_state pointer
 *   arg1 - Scratch register
 */

.macro	save_general_registers
/* AAPCS-64 Page 14
 *
 * A subroutine invocation must preserve the contents of the registers r19-r29
 * and SP. We also save IP0 and IP1, as machine_idle uses IP0 for saving the LR.
 */
	stp		x16, x17, [$0, SS64_KERNEL_X16]
	stp		x19, x20, [$0, SS64_KERNEL_X19]
	stp		x21, x22, [$0, SS64_KERNEL_X21]
	stp		x23, x24, [$0, SS64_KERNEL_X23]
	stp		x25, x26, [$0, SS64_KERNEL_X25]
	stp		x27, x28, [$0, SS64_KERNEL_X27]
	stp		fp, lr, [$0, SS64_KERNEL_FP]
	str		xzr, [$0, SS64_KERNEL_PC]
	MOV32	w$1, PSR64_KERNEL_POISON
	str		w$1, [$0, SS64_KERNEL_CPSR]	
#ifdef HAS_APPLE_PAC
	stp		x0, x1, [sp, #-16]!
	stp		x2, x3, [sp, #-16]!
	stp		x4, x5, [sp, #-16]!

	/*
	 * Arg0: The ARM context pointer
	 * Arg1: PC value to sign
	 * Arg2: CPSR value to sign
	 * Arg3: LR to sign
	 */
	mov		x0, $0
	mov		x1, #0
	mov		w2, w$1
	mov		x3, lr
	mov		x4, x16
	mov		x5, x17
	bl		EXT(ml_sign_kernel_thread_state)

	ldp		x4, x5, [sp], #16
	ldp		x2, x3, [sp], #16
	ldp		x0, x1, [sp], #16
	ldp		fp, lr, [$0, SS64_KERNEL_FP]
#endif /* defined(HAS_APPLE_PAC) */
	mov		x$1, sp
	str		x$1, [$0, SS64_KERNEL_SP]

/* AAPCS-64 Page 14
 *
 * Registers d8-d15 (s8-s15) must be preserved by a callee across subroutine
 * calls; the remaining registers (v0-v7, v16-v31) do not need to be preserved
 * (or should be preserved by the caller).
 */
	str		d8,	[$0, NS64_KERNEL_D8]
	str		d9,	[$0, NS64_KERNEL_D9]
	str		d10,[$0, NS64_KERNEL_D10]
	str		d11,[$0, NS64_KERNEL_D11]
	str		d12,[$0, NS64_KERNEL_D12]
	str		d13,[$0, NS64_KERNEL_D13]
	str		d14,[$0, NS64_KERNEL_D14]
	str		d15,[$0, NS64_KERNEL_D15]

	mrs		x$1, FPCR
	str		w$1, [$0, NS64_KERNEL_FPCR]
.endmacro

/*
 * load_general_registers
 *
 * Loads variable registers from kernel PCB.
 *   arg0 - thread_kernel_state pointer
 *   arg1 - Scratch register
 */
.macro	load_general_registers
	mov		x20, x0
	mov		x21, x1
	mov		x22, x2

	mov		x0, $0
	AUTH_KERNEL_THREAD_STATE_IN_X0	x23, x24, x25, x26, x27

	mov		x0, x20
	mov		x1, x21
	mov		x2, x22

	ldr		w$1, [$0, NS64_KERNEL_FPCR]
	mrs		x19, FPCR
	CMSR FPCR, x19, x$1, 1
1:

	// Skip x16, x17 - already loaded + authed by AUTH_THREAD_STATE_IN_X0
	ldp		x19, x20, [$0, SS64_KERNEL_X19]
	ldp		x21, x22, [$0, SS64_KERNEL_X21]
	ldp		x23, x24, [$0, SS64_KERNEL_X23]
	ldp		x25, x26, [$0, SS64_KERNEL_X25]
	ldp		x27, x28, [$0, SS64_KERNEL_X27]
	ldr		fp, [$0, SS64_KERNEL_FP]
	// Skip lr - already loaded + authed by AUTH_THREAD_STATE_IN_X0
	ldr		x$1, [$0, SS64_KERNEL_SP]
	mov		sp, x$1

	ldr		d8,	[$0, NS64_KERNEL_D8]
	ldr		d9,	[$0, NS64_KERNEL_D9]
	ldr		d10,[$0, NS64_KERNEL_D10]
	ldr		d11,[$0, NS64_KERNEL_D11]
	ldr		d12,[$0, NS64_KERNEL_D12]
	ldr		d13,[$0, NS64_KERNEL_D13]
	ldr		d14,[$0, NS64_KERNEL_D14]
	ldr		d15,[$0, NS64_KERNEL_D15]
.endmacro


/*
 * set_thread_registers
 *
 * Updates thread registers during context switch
 *  arg0 - New thread pointer
 *  arg1 - Scratch register
 *  arg2 - Scratch register
 */
.macro	set_thread_registers
	msr		TPIDR_EL1, $0						// Write new thread pointer to TPIDR_EL1
	ldr		$1, [$0, ACT_CPUDATAP]
	str		$0, [$1, CPU_ACTIVE_THREAD]
	ldr		$1, [$0, TH_CTH_SELF]				// Get cthread pointer
	mrs		$2, TPIDRRO_EL0						// Extract cpu number from TPIDRRO_EL0
	and		$2, $2, #(MACHDEP_CPUNUM_MASK)
	orr		$2, $1, $2							// Save new cthread/cpu to TPIDRRO_EL0
	msr		TPIDRRO_EL0, $2
	msr		TPIDR_EL0, xzr
#if DEBUG || DEVELOPMENT
	ldr		$1, [$0, TH_THREAD_ID]				// Save the bottom 32-bits of the thread ID into
	msr		CONTEXTIDR_EL1, $1					// CONTEXTIDR_EL1 (top 32-bits are RES0).
#endif /* DEBUG || DEVELOPMENT */
.endmacro

/*
 * set_process_dependent_keys_and_sync_context
 *
 * Updates process dependent keys and issues explicit context sync during context switch if necessary
 *  Per CPU Data rop_key is initialized in arm_init() for bootstrap processor
 *  and in cpu_data_init for slave processors
 *
 *  thread - New thread pointer
 *  new_key - Scratch register: New Thread Key
 *  tmp_key - Scratch register: Current CPU Key
 *  cpudatap - Scratch register: Current CPU Data pointer
 *  wsync - Half-width scratch register: CPU sync required flag
 *
 *  to save on ISBs, for ARMv8.5 we use the CPU_SYNC_ON_CSWITCH field, cached in wsync, for pre-ARMv8.5,
 *  we just use wsync to keep track of needing an ISB
 */
.macro set_process_dependent_keys_and_sync_context	thread, new_key, tmp_key, cpudatap, wsync


#if defined(__ARM_ARCH_8_5__) || defined(HAS_APPLE_PAC)
	ldr		\cpudatap, [\thread, ACT_CPUDATAP]
#endif /* defined(__ARM_ARCH_8_5__) || defined(HAS_APPLE_PAC) */

#if defined(__ARM_ARCH_8_5__)
	ldrb	\wsync, [\cpudatap, CPU_SYNC_ON_CSWITCH]
#else /* defined(__ARM_ARCH_8_5__) */
	mov		\wsync, #0
#endif


#if defined(HAS_APPLE_PAC)
	ldr		\new_key, [\thread, TH_ROP_PID]
	ldr		\tmp_key, [\cpudatap, CPU_ROP_KEY]
	cmp		\new_key, \tmp_key
	b.eq	1f
	str		\new_key, [\cpudatap, CPU_ROP_KEY]
	msr		APIBKeyLo_EL1, \new_key
	add		\new_key, \new_key, #1
	msr		APIBKeyHi_EL1, \new_key
	add		\new_key, \new_key, #1
	msr		APDBKeyLo_EL1, \new_key
	add		\new_key, \new_key, #1
	msr		APDBKeyHi_EL1, \new_key
	mov		\wsync, #1
1:

#if HAS_PAC_FAST_A_KEY_SWITCHING
	IF_PAC_SLOW_A_KEY_SWITCHING	Lskip_jop_keys_\@, \new_key
	ldr		\new_key, [\thread, TH_JOP_PID]
	REPROGRAM_JOP_KEYS	Lskip_jop_keys_\@, \new_key, \cpudatap, \tmp_key
	mov		\wsync, #1
Lskip_jop_keys_\@:
#endif /* HAS_PAC_FAST_A_KEY_SWITCHING */

#endif /* defined(HAS_APPLE_PAC) */

	cbz		\wsync, 1f
	isb 	sy

#if defined(__ARM_ARCH_8_5__)
	strb	wzr, [\cpudatap, CPU_SYNC_ON_CSWITCH]
#endif
1:
.endmacro

/*
 * void     machine_load_context(thread_t        thread)
 *
 * Load the context for the first thread to run on a
 * cpu, and go.
 */
	.text
	.align 2
	.globl	EXT(machine_load_context)

LEXT(machine_load_context)
	set_thread_registers 	x0, x1, x2
	ldr		x1, [x0, TH_KSTACKPTR]				// Get top of kernel stack
	load_general_registers 	x1, 2
	set_process_dependent_keys_and_sync_context	x0, x1, x2, x3, w4
	mov		x0, #0								// Clear argument to thread_continue
	ret

/*
 *  typedef void (*thread_continue_t)(void *param, wait_result_t)
 *
 *	void Call_continuation( thread_continue_t continuation,
 *	            			void *param,
 *				            wait_result_t wresult,
 *                          bool enable interrupts)
 */
	.text
	.align	5
	.globl	EXT(Call_continuation)

LEXT(Call_continuation)
	mrs		x4, TPIDR_EL1						// Get the current thread pointer

	/* ARM64_TODO arm loads the kstack top instead of arg4. What should we use? */
	ldr		x5, [x4, TH_KSTACKPTR]				// Get the top of the kernel stack
	mov		sp, x5								// Set stack pointer
	mov		fp, #0								// Clear the frame pointer

	set_process_dependent_keys_and_sync_context	x4, x5, x6, x7, w20

	mov x20, x0  //continuation
	mov x21, x1  //continuation parameter
	mov x22, x2  //wait result

	cbz x3, 1f
	mov x0, #1
	bl EXT(ml_set_interrupts_enabled)
1:

	mov		x0, x21								// Set the first parameter
	mov		x1, x22								// Set the wait result arg
#ifdef HAS_APPLE_PAC
	mov		x21, THREAD_CONTINUE_T_DISC
	blraa	x20, x21							// Branch to the continuation
#else
	blr		x20									// Branch to the continuation
#endif
	mrs		x0, TPIDR_EL1						// Get the current thread pointer
	b		EXT(thread_terminate)				// Kill the thread


/*
 *	thread_t Switch_context(thread_t	old,
 * 				void		(*cont)(void),
 *				thread_t	new)
 */
	.text
	.align 5
	.globl	EXT(Switch_context)

LEXT(Switch_context)
	cbnz	x1, Lswitch_threads					// Skip saving old state if blocking on continuation
	ldr		x3, [x0, TH_KSTACKPTR]				// Get the old kernel stack top
	save_general_registers	x3, 4
Lswitch_threads:
	set_thread_registers	x2, x3, x4
	ldr		x3, [x2, TH_KSTACKPTR]
	load_general_registers	x3, 4
	set_process_dependent_keys_and_sync_context	x2, x3, x4, x5, w6
	ret

/*
 *	thread_t Shutdown_context(void (*doshutdown)(processor_t), processor_t processor)
 *
 */
	.text
	.align 5
	.globl	EXT(Shutdown_context)

LEXT(Shutdown_context)
	mrs		x10, TPIDR_EL1							// Get thread pointer
	ldr		x11, [x10, TH_KSTACKPTR]				// Get the top of the kernel stack
	save_general_registers	x11, 12
	msr		DAIFSet, #(DAIFSC_FIQF | DAIFSC_IRQF)	// Disable interrupts
	ldr		x11, [x10, ACT_CPUDATAP]				// Get current cpu
	ldr		x12, [x11, CPU_ISTACKPTR]				// Switch to interrupt stack
	mov		sp, x12
	b		EXT(cpu_doshutdown)

/*
 *	thread_t Idle_context(void)
 *
 */
	.text
	.align 5
	.globl	EXT(Idle_context)

LEXT(Idle_context)
	mrs		x0, TPIDR_EL1						// Get thread pointer
	ldr		x1, [x0, TH_KSTACKPTR]				// Get the top of the kernel stack
	save_general_registers	x1, 2
	ldr		x1, [x0, ACT_CPUDATAP]				// Get current cpu
	ldr		x2, [x1, CPU_ISTACKPTR]				// Switch to interrupt stack
	mov		sp, x2
	b		EXT(cpu_idle)

/*
 *	thread_t Idle_context(void)
 *
 */
	.text
	.align 5
	.globl	EXT(Idle_load_context)

LEXT(Idle_load_context)
	mrs		x0, TPIDR_EL1						// Get thread pointer
	ldr		x1, [x0, TH_KSTACKPTR]				// Get the top of the kernel stack
	load_general_registers	x1, 2
	set_process_dependent_keys_and_sync_context	x0, x1, x2, x3, w4
	ret

	.align	2
	.globl	EXT(machine_set_current_thread)
LEXT(machine_set_current_thread)
	set_thread_registers x0, x1, x2
	ret


/* vim: set ts=4: */
Commit	Line	Data
5ba3f43e A	1	/*
	2	* Copyright (c) 2007 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28	#include <machine/asm.h>
	29	#include <arm64/machine_machdep.h>
cb323159	30	#include <arm64/machine_routines_asm.h>
f427ee49	31	#include <arm64/pac_asm.h>
5ba3f43e A	32	#include <arm64/proc_reg.h>
	33	#include "assym.s"
	34
	35	/*
	36	* save_general_registers
	37	*
	38	* Saves variable registers to kernel PCB.
	39	* arg0 - thread_kernel_state pointer
	40	* arg1 - Scratch register
	41	*/
	42
	43	.macro save_general_registers
	44	/* AAPCS-64 Page 14
	45	*
	46	* A subroutine invocation must preserve the contents of the registers r19-r29
	47	* and SP. We also save IP0 and IP1, as machine_idle uses IP0 for saving the LR.
	48	*/
f427ee49 A	49	stp x16, x17, [$0, SS64_KERNEL_X16]
	50	stp x19, x20, [$0, SS64_KERNEL_X19]
	51	stp x21, x22, [$0, SS64_KERNEL_X21]
	52	stp x23, x24, [$0, SS64_KERNEL_X23]
	53	stp x25, x26, [$0, SS64_KERNEL_X25]
	54	stp x27, x28, [$0, SS64_KERNEL_X27]
	55	stp fp, lr, [$0, SS64_KERNEL_FP]
	56	str xzr, [$0, SS64_KERNEL_PC]
eb6b6ca3	57	MOV32 w$1, PSR64_KERNEL_POISON
f427ee49	58	str w$1, [$0, SS64_KERNEL_CPSR]
cb323159 A	59	#ifdef HAS_APPLE_PAC
	60	stp x0, x1, [sp, #-16]!
	61	stp x2, x3, [sp, #-16]!
	62	stp x4, x5, [sp, #-16]!
	63
	64	/*
	65	* Arg0: The ARM context pointer
	66	* Arg1: PC value to sign
	67	* Arg2: CPSR value to sign
	68	* Arg3: LR to sign
	69	*/
	70	mov x0, $0
eb6b6ca3 A	71	mov x1, #0
eb6b6ca3 A	72	mov w2, w$1
cb323159 A	73	mov x3, lr
	74	mov x4, x16
	75	mov x5, x17
f427ee49	76	bl EXT(ml_sign_kernel_thread_state)
cb323159 A	77
	78	ldp x4, x5, [sp], #16
	79	ldp x2, x3, [sp], #16
	80	ldp x0, x1, [sp], #16
f427ee49	81	ldp fp, lr, [$0, SS64_KERNEL_FP]
cb323159	82	#endif /* defined(HAS_APPLE_PAC) */
eb6b6ca3	83	mov x$1, sp
f427ee49	84	str x$1, [$0, SS64_KERNEL_SP]
5ba3f43e A	85
	86	/* AAPCS-64 Page 14
	87	*
	88	* Registers d8-d15 (s8-s15) must be preserved by a callee across subroutine
	89	* calls; the remaining registers (v0-v7, v16-v31) do not need to be preserved
	90	* (or should be preserved by the caller).
	91	*/
f427ee49 A	92	str d8, [$0, NS64_KERNEL_D8]
	93	str d9, [$0, NS64_KERNEL_D9]
	94	str d10,[$0, NS64_KERNEL_D10]
	95	str d11,[$0, NS64_KERNEL_D11]
	96	str d12,[$0, NS64_KERNEL_D12]
	97	str d13,[$0, NS64_KERNEL_D13]
	98	str d14,[$0, NS64_KERNEL_D14]
	99	str d15,[$0, NS64_KERNEL_D15]
	100
	101	mrs x$1, FPCR
	102	str w$1, [$0, NS64_KERNEL_FPCR]
5ba3f43e A	103	.endmacro
	104
	105	/*
	106	* load_general_registers
	107	*
	108	* Loads variable registers from kernel PCB.
	109	* arg0 - thread_kernel_state pointer
	110	* arg1 - Scratch register
	111	*/
	112	.macro load_general_registers
cb323159 A	113	mov x20, x0
	114	mov x21, x1
	115	mov x22, x2
	116
	117	mov x0, $0
f427ee49	118	AUTH_KERNEL_THREAD_STATE_IN_X0 x23, x24, x25, x26, x27
d9a64523	119
cb323159 A	120	mov x0, x20
	121	mov x1, x21
	122	mov x2, x22
	123
f427ee49 A	124	ldr w$1, [$0, NS64_KERNEL_FPCR]
	125	mrs x19, FPCR
	126	CMSR FPCR, x19, x$1, 1
	127	1:
	128
cb323159	129	// Skip x16, x17 - already loaded + authed by AUTH_THREAD_STATE_IN_X0
f427ee49 A	130	ldp x19, x20, [$0, SS64_KERNEL_X19]
	131	ldp x21, x22, [$0, SS64_KERNEL_X21]
	132	ldp x23, x24, [$0, SS64_KERNEL_X23]
	133	ldp x25, x26, [$0, SS64_KERNEL_X25]
	134	ldp x27, x28, [$0, SS64_KERNEL_X27]
	135	ldr fp, [$0, SS64_KERNEL_FP]
cb323159	136	// Skip lr - already loaded + authed by AUTH_THREAD_STATE_IN_X0
f427ee49 A	137	ldr x$1, [$0, SS64_KERNEL_SP]
	138	mov sp, x$1
	139
	140	ldr d8, [$0, NS64_KERNEL_D8]
	141	ldr d9, [$0, NS64_KERNEL_D9]
	142	ldr d10,[$0, NS64_KERNEL_D10]
	143	ldr d11,[$0, NS64_KERNEL_D11]
	144	ldr d12,[$0, NS64_KERNEL_D12]
	145	ldr d13,[$0, NS64_KERNEL_D13]
	146	ldr d14,[$0, NS64_KERNEL_D14]
	147	ldr d15,[$0, NS64_KERNEL_D15]
5ba3f43e A	148	.endmacro
5ba3f43e A	149
cb323159	150
5ba3f43e A	151	/*
	152	* set_thread_registers
	153	*
	154	* Updates thread registers during context switch
	155	* arg0 - New thread pointer
	156	* arg1 - Scratch register
	157	* arg2 - Scratch register
	158	*/
	159	.macro set_thread_registers
	160	msr TPIDR_EL1, $0 // Write new thread pointer to TPIDR_EL1
ea3f0419 A	161	ldr $1, [$0, ACT_CPUDATAP]
ea3f0419 A	162	str $0, [$1, CPU_ACTIVE_THREAD]
5ba3f43e A	163	ldr $1, [$0, TH_CTH_SELF] // Get cthread pointer
	164	mrs $2, TPIDRRO_EL0 // Extract cpu number from TPIDRRO_EL0
	165	and $2, $2, #(MACHDEP_CPUNUM_MASK)
	166	orr $2, $1, $2 // Save new cthread/cpu to TPIDRRO_EL0
	167	msr TPIDRRO_EL0, $2
94ff46dc	168	msr TPIDR_EL0, xzr
f427ee49 A	169	#if DEBUG \|\| DEVELOPMENT
	170	ldr $1, [$0, TH_THREAD_ID] // Save the bottom 32-bits of the thread ID into
	171	msr CONTEXTIDR_EL1, $1 // CONTEXTIDR_EL1 (top 32-bits are RES0).
	172	#endif /* DEBUG \|\| DEVELOPMENT */
5ba3f43e A	173	.endmacro
5ba3f43e A	174
cb323159	175	/*
f427ee49	176	* set_process_dependent_keys_and_sync_context
cb323159	177	*
f427ee49	178	* Updates process dependent keys and issues explicit context sync during context switch if necessary
cb323159 A	179	* Per CPU Data rop_key is initialized in arm_init() for bootstrap processor
	180	* and in cpu_data_init for slave processors
	181	*
f427ee49 A	182	* thread - New thread pointer
	183	* new_key - Scratch register: New Thread Key
	184	* tmp_key - Scratch register: Current CPU Key
	185	* cpudatap - Scratch register: Current CPU Data pointer
	186	* wsync - Half-width scratch register: CPU sync required flag
	187	*
	188	* to save on ISBs, for ARMv8.5 we use the CPU_SYNC_ON_CSWITCH field, cached in wsync, for pre-ARMv8.5,
	189	* we just use wsync to keep track of needing an ISB
cb323159	190	*/
f427ee49 A	191	.macro set_process_dependent_keys_and_sync_context thread, new_key, tmp_key, cpudatap, wsync
	192
	193
	194	#if defined(__ARM_ARCH_8_5__) \|\| defined(HAS_APPLE_PAC)
	195	ldr \cpudatap, [\thread, ACT_CPUDATAP]
	196	#endif /* defined(__ARM_ARCH_8_5__) \|\| defined(HAS_APPLE_PAC) */
	197
2a1bd2d3 A	198	#if defined(__ARM_ARCH_8_5__)
	199	ldrb \wsync, [\cpudatap, CPU_SYNC_ON_CSWITCH]
	200	#else /* defined(__ARM_ARCH_8_5__) */
f427ee49	201	mov \wsync, #0
2a1bd2d3	202	#endif
f427ee49 A	203
	204
	205	#if defined(HAS_APPLE_PAC)
	206	ldr \new_key, [\thread, TH_ROP_PID]
	207	ldr \tmp_key, [\cpudatap, CPU_ROP_KEY]
	208	cmp \new_key, \tmp_key
cb323159	209	b.eq 1f
f427ee49 A	210	str \new_key, [\cpudatap, CPU_ROP_KEY]
	211	msr APIBKeyLo_EL1, \new_key
	212	add \new_key, \new_key, #1
	213	msr APIBKeyHi_EL1, \new_key
	214	add \new_key, \new_key, #1
	215	msr APDBKeyLo_EL1, \new_key
	216	add \new_key, \new_key, #1
	217	msr APDBKeyHi_EL1, \new_key
	218	mov \wsync, #1
	219	1:
	220
	221	#if HAS_PAC_FAST_A_KEY_SWITCHING
	222	IF_PAC_SLOW_A_KEY_SWITCHING Lskip_jop_keys_\@, \new_key
	223	ldr \new_key, [\thread, TH_JOP_PID]
	224	REPROGRAM_JOP_KEYS Lskip_jop_keys_\@, \new_key, \cpudatap, \tmp_key
	225	mov \wsync, #1
	226	Lskip_jop_keys_\@:
	227	#endif /* HAS_PAC_FAST_A_KEY_SWITCHING */
	228
	229	#endif /* defined(HAS_APPLE_PAC) */
	230
	231	cbz \wsync, 1f
cb323159	232	isb sy
f427ee49	233
2a1bd2d3 A	234	#if defined(__ARM_ARCH_8_5__)
	235	strb wzr, [\cpudatap, CPU_SYNC_ON_CSWITCH]
	236	#endif
cb323159 A	237	1:
cb323159 A	238	.endmacro
5ba3f43e A	239
	240	/*
	241	* void machine_load_context(thread_t thread)
	242	*
	243	* Load the context for the first thread to run on a
	244	* cpu, and go.
	245	*/
	246	.text
	247	.align 2
	248	.globl EXT(machine_load_context)
	249
	250	LEXT(machine_load_context)
	251	set_thread_registers x0, x1, x2
	252	ldr x1, [x0, TH_KSTACKPTR] // Get top of kernel stack
f427ee49 A	253	load_general_registers x1, 2
f427ee49 A	254	set_process_dependent_keys_and_sync_context x0, x1, x2, x3, w4
d9a64523	255	mov x0, #0 // Clear argument to thread_continue
5ba3f43e A	256	ret
	257
	258	/*
d9a64523 A	259	* typedef void (thread_continue_t)(void param, wait_result_t)
	260	*
	261	* void Call_continuation( thread_continue_t continuation,
	262	* void *param,
	263	* wait_result_t wresult,
	264	* bool enable interrupts)
5ba3f43e A	265	*/
	266	.text
	267	.align 5
	268	.globl EXT(Call_continuation)
	269
	270	LEXT(Call_continuation)
	271	mrs x4, TPIDR_EL1 // Get the current thread pointer
	272
	273	/* ARM64_TODO arm loads the kstack top instead of arg4. What should we use? */
	274	ldr x5, [x4, TH_KSTACKPTR] // Get the top of the kernel stack
	275	mov sp, x5 // Set stack pointer
d9a64523 A	276	mov fp, #0 // Clear the frame pointer
d9a64523 A	277
f427ee49	278	set_process_dependent_keys_and_sync_context x4, x5, x6, x7, w20
d9a64523	279
f427ee49 A	280	mov x20, x0 //continuation
	281	mov x21, x1 //continuation parameter
	282	mov x22, x2 //wait result
d9a64523	283
f427ee49 A	284	cbz x3, 1f
	285	mov x0, #1
	286	bl EXT(ml_set_interrupts_enabled)
d9a64523	287	1:
5ba3f43e	288
d9a64523 A	289	mov x0, x21 // Set the first parameter
d9a64523 A	290	mov x1, x22 // Set the wait result arg
cb323159	291	#ifdef HAS_APPLE_PAC
f427ee49 A	292	mov x21, THREAD_CONTINUE_T_DISC
f427ee49 A	293	blraa x20, x21 // Branch to the continuation
cb323159	294	#else
d9a64523	295	blr x20 // Branch to the continuation
cb323159	296	#endif
5ba3f43e A	297	mrs x0, TPIDR_EL1 // Get the current thread pointer
	298	b EXT(thread_terminate) // Kill the thread
	299
	300
	301	/*
	302	* thread_t Switch_context(thread_t old,
	303	* void (*cont)(void),
	304	* thread_t new)
	305	*/
	306	.text
	307	.align 5
	308	.globl EXT(Switch_context)
	309
	310	LEXT(Switch_context)
	311	cbnz x1, Lswitch_threads // Skip saving old state if blocking on continuation
	312	ldr x3, [x0, TH_KSTACKPTR] // Get the old kernel stack top
eb6b6ca3	313	save_general_registers x3, 4
5ba3f43e A	314	Lswitch_threads:
	315	set_thread_registers x2, x3, x4
	316	ldr x3, [x2, TH_KSTACKPTR]
f427ee49 A	317	load_general_registers x3, 4
f427ee49 A	318	set_process_dependent_keys_and_sync_context x2, x3, x4, x5, w6
5ba3f43e A	319	ret
	320
	321	/*
	322	* thread_t Shutdown_context(void (*doshutdown)(processor_t), processor_t processor)
	323	*
	324	*/
	325	.text
	326	.align 5
	327	.globl EXT(Shutdown_context)
	328
	329	LEXT(Shutdown_context)
	330	mrs x10, TPIDR_EL1 // Get thread pointer
	331	ldr x11, [x10, TH_KSTACKPTR] // Get the top of the kernel stack
eb6b6ca3	332	save_general_registers x11, 12
5ba3f43e A	333	msr DAIFSet, #(DAIFSC_FIQF \| DAIFSC_IRQF) // Disable interrupts
	334	ldr x11, [x10, ACT_CPUDATAP] // Get current cpu
	335	ldr x12, [x11, CPU_ISTACKPTR] // Switch to interrupt stack
	336	mov sp, x12
	337	b EXT(cpu_doshutdown)
	338
5ba3f43e A	339	/*
	340	* thread_t Idle_context(void)
	341	*
	342	*/
	343	.text
	344	.align 5
	345	.globl EXT(Idle_context)
	346
	347	LEXT(Idle_context)
	348	mrs x0, TPIDR_EL1 // Get thread pointer
	349	ldr x1, [x0, TH_KSTACKPTR] // Get the top of the kernel stack
eb6b6ca3	350	save_general_registers x1, 2
5ba3f43e A	351	ldr x1, [x0, ACT_CPUDATAP] // Get current cpu
	352	ldr x2, [x1, CPU_ISTACKPTR] // Switch to interrupt stack
	353	mov sp, x2
	354	b EXT(cpu_idle)
	355
	356	/*
	357	* thread_t Idle_context(void)
	358	*
	359	*/
	360	.text
	361	.align 5
	362	.globl EXT(Idle_load_context)
	363
	364	LEXT(Idle_load_context)
	365	mrs x0, TPIDR_EL1 // Get thread pointer
	366	ldr x1, [x0, TH_KSTACKPTR] // Get the top of the kernel stack
f427ee49 A	367	load_general_registers x1, 2
f427ee49 A	368	set_process_dependent_keys_and_sync_context x0, x1, x2, x3, w4
5ba3f43e A	369	ret
	370
	371	.align 2
	372	.globl EXT(machine_set_current_thread)
	373	LEXT(machine_set_current_thread)
	374	set_thread_registers x0, x1, x2
	375	ret
cb323159 A	376
cb323159 A	377
f427ee49	378	/* vim: set ts=4: */