[apple/xnu.git] / osfmk / arm64 / cswitch.s

/*
 * Copyright (c) 2007 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
#include <machine/asm.h>
#include <arm64/machine_machdep.h>
#include <arm64/proc_reg.h>
#include "assym.s"

/*
 * save_general_registers
 *
 * Saves variable registers to kernel PCB.
 *   arg0 - thread_kernel_state pointer
 *   arg1 - Scratch register
 */

.macro	save_general_registers
/* AAPCS-64 Page 14
 *
 * A subroutine invocation must preserve the contents of the registers r19-r29
 * and SP. We also save IP0 and IP1, as machine_idle uses IP0 for saving the LR.
 */
	stp		x16, x17, [$0, SS64_X16]
	stp		x19, x20, [$0, SS64_X19]
	stp		x21, x22, [$0, SS64_X21]
	stp		x23, x24, [$0, SS64_X23]
	stp		x25, x26, [$0, SS64_X25]
	stp		x27, x28, [$0, SS64_X27]
	stp		fp, lr, [$0, SS64_FP]
	mov		$1, sp
	str		$1, [$0, SS64_SP]

/* AAPCS-64 Page 14
 *
 * Registers d8-d15 (s8-s15) must be preserved by a callee across subroutine
 * calls; the remaining registers (v0-v7, v16-v31) do not need to be preserved
 * (or should be preserved by the caller).
 */
	str		d8,	[$0, NS64_D8]
	str		d9,	[$0, NS64_D9]
	str		d10,[$0, NS64_D10]
	str		d11,[$0, NS64_D11]
	str		d12,[$0, NS64_D12]
	str		d13,[$0, NS64_D13]
	str		d14,[$0, NS64_D14]
	str		d15,[$0, NS64_D15]
.endmacro

/*
 * load_general_registers
 *
 * Loads variable registers from kernel PCB.
 *   arg0 - thread_kernel_state pointer
 *   arg1 - Scratch register
 */
.macro	load_general_registers

	ldp		x16, x17, [$0, SS64_X16]
	ldp		x19, x20, [$0, SS64_X19]
	ldp		x21, x22, [$0, SS64_X21]
	ldp		x23, x24, [$0, SS64_X23]
	ldp		x25, x26, [$0, SS64_X25]
	ldp		x27, x28, [$0, SS64_X27]
	ldp		fp, lr, [$0, SS64_FP]
	ldr		$1, [$0, SS64_SP]
	mov		sp, $1

	ldr		d8,	[$0, NS64_D8]
	ldr		d9,	[$0, NS64_D9]
	ldr		d10,[$0, NS64_D10]
	ldr		d11,[$0, NS64_D11]
	ldr		d12,[$0, NS64_D12]
	ldr		d13,[$0, NS64_D13]
	ldr		d14,[$0, NS64_D14]
	ldr		d15,[$0, NS64_D15]
.endmacro

/*
 * set_thread_registers
 *
 * Updates thread registers during context switch
 *  arg0 - New thread pointer
 *  arg1 - Scratch register
 *  arg2 - Scratch register
 */
.macro	set_thread_registers
	msr		TPIDR_EL1, $0						// Write new thread pointer to TPIDR_EL1
	ldr		$1, [$0, TH_CTH_SELF]				// Get cthread pointer
	mrs		$2, TPIDRRO_EL0						// Extract cpu number from TPIDRRO_EL0
	and		$2, $2, #(MACHDEP_CPUNUM_MASK)
	orr		$2, $1, $2							// Save new cthread/cpu to TPIDRRO_EL0
	msr		TPIDRRO_EL0, $2
	ldr		$1, [$0, TH_CTH_DATA]				// Get new cthread data pointer
	msr		TPIDR_EL0, $1						// Save data pointer to TPIDRRW_EL0
	/* ARM64_TODO Reserve x18 until we decide what to do with it */
	mov		x18, $1								// ... and trash reserved x18
.endmacro


/*
 * void     machine_load_context(thread_t        thread)
 *
 * Load the context for the first thread to run on a
 * cpu, and go.
 */
	.text
	.align 2
	.globl	EXT(machine_load_context)

LEXT(machine_load_context)
	set_thread_registers 	x0, x1, x2
	ldr		x1, [x0, TH_KSTACKPTR]				// Get top of kernel stack
	load_general_registers 	x1, x2
	mov		x0, #0								// Clear argument to thread_continue
	ret

/*
 *  typedef void (*thread_continue_t)(void *param, wait_result_t)
 *
 *	void Call_continuation( thread_continue_t continuation,
 *	            			void *param,
 *				            wait_result_t wresult,
 *                          bool enable interrupts)
 */
	.text
	.align	5
	.globl	EXT(Call_continuation)

LEXT(Call_continuation)
	mrs		x4, TPIDR_EL1						// Get the current thread pointer

	/* ARM64_TODO arm loads the kstack top instead of arg4. What should we use? */
	ldr		x5, [x4, TH_KSTACKPTR]				// Get the top of the kernel stack
	mov		sp, x5								// Set stack pointer
	mov		fp, #0								// Clear the frame pointer


    mov x20, x0  //continuation
    mov x21, x1  //continuation parameter
    mov x22, x2  //wait result

    cbz x3, 1f
    mov x0, #1
    bl _ml_set_interrupts_enabled
1:

	mov		x0, x21								// Set the first parameter
	mov		x1, x22								// Set the wait result arg
	blr		x20									// Branch to the continuation
	mrs		x0, TPIDR_EL1						// Get the current thread pointer
	b		EXT(thread_terminate)				// Kill the thread


/*
 *	thread_t Switch_context(thread_t	old,
 * 				void		(*cont)(void),
 *				thread_t	new)
 */
	.text
	.align 5
	.globl	EXT(Switch_context)

LEXT(Switch_context)
	cbnz	x1, Lswitch_threads					// Skip saving old state if blocking on continuation
	ldr		x3, [x0, TH_KSTACKPTR]				// Get the old kernel stack top
	save_general_registers	x3, x4
Lswitch_threads:
	set_thread_registers	x2, x3, x4
	ldr		x3, [x2, TH_KSTACKPTR]
	load_general_registers	x3, x4
	ret

/*
 *	thread_t Shutdown_context(void (*doshutdown)(processor_t), processor_t processor)
 *
 */
	.text
	.align 5
	.globl	EXT(Shutdown_context)

LEXT(Shutdown_context)
	mrs		x10, TPIDR_EL1							// Get thread pointer
	ldr		x11, [x10, TH_KSTACKPTR]				// Get the top of the kernel stack
	save_general_registers	x11, x12
	msr		DAIFSet, #(DAIFSC_FIQF | DAIFSC_IRQF)	// Disable interrupts
	ldr		x11, [x10, ACT_CPUDATAP]				// Get current cpu
	ldr		x12, [x11, CPU_ISTACKPTR]				// Switch to interrupt stack
	mov		sp, x12
	b		EXT(cpu_doshutdown)


/*
 *	thread_t Idle_context(void)
 *
 */
	.text
	.align 5
	.globl	EXT(Idle_context)

LEXT(Idle_context)
	mrs		x0, TPIDR_EL1						// Get thread pointer
	ldr		x1, [x0, TH_KSTACKPTR]				// Get the top of the kernel stack
	save_general_registers	x1, x2
	ldr		x1, [x0, ACT_CPUDATAP]				// Get current cpu
	ldr		x2, [x1, CPU_ISTACKPTR]				// Switch to interrupt stack
	mov		sp, x2
	b		EXT(cpu_idle)

/*
 *	thread_t Idle_context(void)
 *
 */
	.text
	.align 5
	.globl	EXT(Idle_load_context)

LEXT(Idle_load_context)
	mrs		x0, TPIDR_EL1						// Get thread pointer
	ldr		x1, [x0, TH_KSTACKPTR]				// Get the top of the kernel stack
	load_general_registers	x1, x2
	ret

	.align	2
	.globl	EXT(machine_set_current_thread)
LEXT(machine_set_current_thread)
	set_thread_registers x0, x1, x2
	ret
Commit	Line	Data
5ba3f43e A	1	/*
	2	* Copyright (c) 2007 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28	#include <machine/asm.h>
	29	#include <arm64/machine_machdep.h>
	30	#include <arm64/proc_reg.h>
	31	#include "assym.s"
	32
	33	/*
	34	* save_general_registers
	35	*
	36	* Saves variable registers to kernel PCB.
	37	* arg0 - thread_kernel_state pointer
	38	* arg1 - Scratch register
	39	*/
	40
	41	.macro save_general_registers
	42	/* AAPCS-64 Page 14
	43	*
	44	* A subroutine invocation must preserve the contents of the registers r19-r29
	45	* and SP. We also save IP0 and IP1, as machine_idle uses IP0 for saving the LR.
	46	*/
	47	stp x16, x17, [$0, SS64_X16]
	48	stp x19, x20, [$0, SS64_X19]
	49	stp x21, x22, [$0, SS64_X21]
	50	stp x23, x24, [$0, SS64_X23]
	51	stp x25, x26, [$0, SS64_X25]
	52	stp x27, x28, [$0, SS64_X27]
	53	stp fp, lr, [$0, SS64_FP]
	54	mov $1, sp
	55	str $1, [$0, SS64_SP]
	56
	57	/* AAPCS-64 Page 14
	58	*
	59	* Registers d8-d15 (s8-s15) must be preserved by a callee across subroutine
	60	* calls; the remaining registers (v0-v7, v16-v31) do not need to be preserved
	61	* (or should be preserved by the caller).
	62	*/
	63	str d8, [$0, NS64_D8]
	64	str d9, [$0, NS64_D9]
65	str d10,[$0, NS64_D10]
66	str d11,[$0, NS64_D11]
67	str d12,[$0, NS64_D12]
68	str d13,[$0, NS64_D13]
69	str d14,[$0, NS64_D14]
70	str d15,[$0, NS64_D15]
71	.endmacro
72
73	/*
74	* load_general_registers
75	*
76	* Loads variable registers from kernel PCB.
77	* arg0 - thread_kernel_state pointer
78	* arg1 - Scratch register
79	*/
80	.macro load_general_registers
d9a64523	81
5ba3f43e A	82	ldp x16, x17, [$0, SS64_X16]
	83	ldp x19, x20, [$0, SS64_X19]
	84	ldp x21, x22, [$0, SS64_X21]
	85	ldp x23, x24, [$0, SS64_X23]
	86	ldp x25, x26, [$0, SS64_X25]
	87	ldp x27, x28, [$0, SS64_X27]
	88	ldp fp, lr, [$0, SS64_FP]
	89	ldr $1, [$0, SS64_SP]
	90	mov sp, $1
	91
	92	ldr d8, [$0, NS64_D8]
	93	ldr d9, [$0, NS64_D9]
	94	ldr d10,[$0, NS64_D10]
	95	ldr d11,[$0, NS64_D11]
	96	ldr d12,[$0, NS64_D12]
	97	ldr d13,[$0, NS64_D13]
	98	ldr d14,[$0, NS64_D14]
	99	ldr d15,[$0, NS64_D15]
	100	.endmacro
	101
	102	/*
	103	* set_thread_registers
	104	*
	105	* Updates thread registers during context switch
	106	* arg0 - New thread pointer
	107	* arg1 - Scratch register
	108	* arg2 - Scratch register
	109	*/
	110	.macro set_thread_registers
	111	msr TPIDR_EL1, $0 // Write new thread pointer to TPIDR_EL1
	112	ldr $1, [$0, TH_CTH_SELF] // Get cthread pointer
	113	mrs $2, TPIDRRO_EL0 // Extract cpu number from TPIDRRO_EL0
	114	and $2, $2, #(MACHDEP_CPUNUM_MASK)
	115	orr $2, $1, $2 // Save new cthread/cpu to TPIDRRO_EL0
	116	msr TPIDRRO_EL0, $2
	117	ldr $1, [$0, TH_CTH_DATA] // Get new cthread data pointer
	118	msr TPIDR_EL0, $1 // Save data pointer to TPIDRRW_EL0
	119	/* ARM64_TODO Reserve x18 until we decide what to do with it */
	120	mov x18, $1 // ... and trash reserved x18
	121	.endmacro
	122
	123
	124	/*
	125	* void machine_load_context(thread_t thread)
	126	*
	127	* Load the context for the first thread to run on a
	128	* cpu, and go.
	129	*/
	130	.text
	131	.align 2
	132	.globl EXT(machine_load_context)
	133
	134	LEXT(machine_load_context)
	135	set_thread_registers x0, x1, x2
	136	ldr x1, [x0, TH_KSTACKPTR] // Get top of kernel stack
	137	load_general_registers x1, x2
d9a64523	138	mov x0, #0 // Clear argument to thread_continue
5ba3f43e A	139	ret
	140
	141	/*
d9a64523 A	142	* typedef void (thread_continue_t)(void param, wait_result_t)
	143	*
	144	* void Call_continuation( thread_continue_t continuation,
	145	* void *param,
	146	* wait_result_t wresult,
	147	* bool enable interrupts)
5ba3f43e A	148	*/
	149	.text
	150	.align 5
	151	.globl EXT(Call_continuation)
	152
	153	LEXT(Call_continuation)
	154	mrs x4, TPIDR_EL1 // Get the current thread pointer
	155
	156	/* ARM64_TODO arm loads the kstack top instead of arg4. What should we use? */
	157	ldr x5, [x4, TH_KSTACKPTR] // Get the top of the kernel stack
	158	mov sp, x5 // Set stack pointer
d9a64523 A	159	mov fp, #0 // Clear the frame pointer
	160
	161
	162	mov x20, x0 //continuation
	163	mov x21, x1 //continuation parameter
	164	mov x22, x2 //wait result
	165
	166	cbz x3, 1f
	167	mov x0, #1
	168	bl _ml_set_interrupts_enabled
	169	1:
5ba3f43e	170
d9a64523 A	171	mov x0, x21 // Set the first parameter
	172	mov x1, x22 // Set the wait result arg
	173	blr x20 // Branch to the continuation
5ba3f43e A	174	mrs x0, TPIDR_EL1 // Get the current thread pointer
	175	b EXT(thread_terminate) // Kill the thread
	176
	177
	178	/*
	179	* thread_t Switch_context(thread_t old,
	180	* void (*cont)(void),
	181	* thread_t new)
	182	*/
	183	.text
	184	.align 5
	185	.globl EXT(Switch_context)
	186
	187	LEXT(Switch_context)
	188	cbnz x1, Lswitch_threads // Skip saving old state if blocking on continuation
	189	ldr x3, [x0, TH_KSTACKPTR] // Get the old kernel stack top
	190	save_general_registers x3, x4
	191	Lswitch_threads:
	192	set_thread_registers x2, x3, x4
	193	ldr x3, [x2, TH_KSTACKPTR]
	194	load_general_registers x3, x4
	195	ret
	196
	197	/*
	198	* thread_t Shutdown_context(void (*doshutdown)(processor_t), processor_t processor)
	199	*
	200	*/
	201	.text
	202	.align 5
	203	.globl EXT(Shutdown_context)
	204
	205	LEXT(Shutdown_context)
	206	mrs x10, TPIDR_EL1 // Get thread pointer
	207	ldr x11, [x10, TH_KSTACKPTR] // Get the top of the kernel stack
	208	save_general_registers x11, x12
	209	msr DAIFSet, #(DAIFSC_FIQF \| DAIFSC_IRQF) // Disable interrupts
	210	ldr x11, [x10, ACT_CPUDATAP] // Get current cpu
	211	ldr x12, [x11, CPU_ISTACKPTR] // Switch to interrupt stack
	212	mov sp, x12
	213	b EXT(cpu_doshutdown)
	214
	215
	216	/*
	217	* thread_t Idle_context(void)
	218	*
	219	*/
	220	.text
	221	.align 5
	222	.globl EXT(Idle_context)
	223
	224	LEXT(Idle_context)
	225	mrs x0, TPIDR_EL1 // Get thread pointer
	226	ldr x1, [x0, TH_KSTACKPTR] // Get the top of the kernel stack
	227	save_general_registers x1, x2
	228	ldr x1, [x0, ACT_CPUDATAP] // Get current cpu
	229	ldr x2, [x1, CPU_ISTACKPTR] // Switch to interrupt stack
	230	mov sp, x2
	231	b EXT(cpu_idle)
	232
	233	/*
	234	* thread_t Idle_context(void)
	235	*
	236	*/
	237	.text
238	.align 5
239	.globl EXT(Idle_load_context)
240
241	LEXT(Idle_load_context)
242	mrs x0, TPIDR_EL1 // Get thread pointer
243	ldr x1, [x0, TH_KSTACKPTR] // Get the top of the kernel stack
244	load_general_registers x1, x2
245	ret
246
247	.align 2
248	.globl EXT(machine_set_current_thread)
249	LEXT(machine_set_current_thread)
250	set_thread_registers x0, x1, x2
251	ret