[apple/xnu.git] / osfmk / arm / pcb.c

/*
 * Copyright (c) 2007-2019 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
#include <debug.h>

#include <types.h>

#include <mach/mach_types.h>
#include <mach/thread_status.h>
#include <mach/vm_types.h>

#include <kern/kern_types.h>
#include <kern/task.h>
#include <kern/thread.h>
#include <kern/misc_protos.h>
#include <kern/mach_param.h>
#include <kern/spl.h>
#include <kern/machine.h>
#include <kern/kalloc.h>
#include <kern/kpc.h>

#include <arm/proc_reg.h>
#include <arm/cpu_data_internal.h>
#include <arm/misc_protos.h>
#include <arm/cpuid.h>

#include <vm/vm_map.h>
#include <vm/vm_protos.h>

#include <sys/kdebug.h>

extern int      debug_task;

zone_t		ads_zone;		/* zone for debug_state area */

/*
 * Routine:	consider_machine_collect
 *
 */
void
consider_machine_collect(void)
{
	pmap_gc();
}

/*
 * Routine:	consider_machine_adjust
 *
 */
void
consider_machine_adjust(void)
{
}

/*
 * Routine:	machine_switch_context
 *
 */
thread_t
machine_switch_context(
		       thread_t old,
		       thread_continue_t continuation,
		       thread_t new)
{
	thread_t retval;
	cpu_data_t	*cpu_data_ptr;

#define machine_switch_context_kprintf(x...)	/* kprintf("machine_switch_con
						 * text: " x) */

	cpu_data_ptr = getCpuDatap();
	if (old == new)
		panic("machine_switch_context");

	kpc_off_cpu(old);

	pmap_set_pmap(new->map->pmap, new);

	new->machine.CpuDatap = cpu_data_ptr;

#if __SMP__
	/* TODO: Should this be ordered? */
	old->machine.machine_thread_flags &= ~MACHINE_THREAD_FLAGS_ON_CPU;
	new->machine.machine_thread_flags |= MACHINE_THREAD_FLAGS_ON_CPU;
#endif /* __SMP__ */

	machine_switch_context_kprintf("old= %x contination = %x new = %x\n", old, continuation, new);
	retval = Switch_context(old, continuation, new);
	assert(retval != NULL);

	return retval;
}

boolean_t
machine_thread_on_core(thread_t thread)
{
	return thread->machine.machine_thread_flags & MACHINE_THREAD_FLAGS_ON_CPU;
}

/*
 * Routine:	machine_thread_create
 *
 */
kern_return_t
machine_thread_create(
		      thread_t thread,
#if	!__ARM_USER_PROTECT__
		      __unused
#endif
		      task_t task)
{

#define machine_thread_create_kprintf(x...)	/* kprintf("machine_thread_create: " x) */

	machine_thread_create_kprintf("thread = %x\n", thread);

	if (current_thread() != thread) {
		thread->machine.CpuDatap = (cpu_data_t *)0;
	}
	thread->machine.preemption_count = 0;
	thread->machine.cthread_self = 0;
#if	__ARM_USER_PROTECT__
	{
	struct pmap *new_pmap = vm_map_pmap(task->map);

	thread->machine.kptw_ttb = ((unsigned int) kernel_pmap->ttep) | TTBR_SETUP;
	thread->machine.asid = new_pmap->hw_asid;
	if (new_pmap->tte_index_max == NTTES) {
		thread->machine.uptw_ttc = 2;
		thread->machine.uptw_ttb = ((unsigned int) new_pmap->ttep) | TTBR_SETUP;
	} else {
		thread->machine.uptw_ttc = 1;
		thread->machine.uptw_ttb = ((unsigned int) new_pmap->ttep ) | TTBR_SETUP;
	}
	}
#endif
	machine_thread_state_initialize(thread);

	return (KERN_SUCCESS);
}

/*
 * Routine:	machine_thread_destroy
 *
 */
void
machine_thread_destroy(
		       thread_t thread)
{

        if (thread->machine.DebugData != NULL) {
		if (thread->machine.DebugData == getCpuDatap()->cpu_user_debug)
			arm_debug_set(NULL);
		zfree(ads_zone, thread->machine.DebugData);
	}
}


/*
 * Routine:	machine_thread_init
 *
 */
void
machine_thread_init(void)
{
	ads_zone = zinit(sizeof(arm_debug_state_t),
					 THREAD_CHUNK * (sizeof(arm_debug_state_t)),
					 THREAD_CHUNK * (sizeof(arm_debug_state_t)),
					 "arm debug state");
}

/*
 * Routine:	machine_thread_template_init
 *
 */
void
machine_thread_template_init(thread_t __unused thr_template)
{
	/* Nothing to do on this platform. */
}

/*
 * Routine:	get_useraddr
 *
 */
user_addr_t
get_useraddr()
{
	return (current_thread()->machine.PcbData.pc);
}

/*
 * Routine:	machine_stack_detach
 *
 */
vm_offset_t
machine_stack_detach(
		     thread_t thread)
{
	vm_offset_t     stack;

	KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_DETACH),
		     (uintptr_t)thread_tid(thread), thread->priority, thread->sched_pri, 0, 0);

	stack = thread->kernel_stack;
	thread->kernel_stack = 0;
	thread->machine.kstackptr = 0;

	return (stack);
}


/*
 * Routine:	machine_stack_attach
 *
 */
void
machine_stack_attach(
		     thread_t thread,
		     vm_offset_t stack)
{
	struct arm_saved_state *savestate;

#define machine_stack_attach_kprintf(x...)	/* kprintf("machine_stack_attach: " x) */

	KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_ATTACH),
		     (uintptr_t)thread_tid(thread), thread->priority, thread->sched_pri, 0, 0);

	thread->kernel_stack = stack;
	thread->machine.kstackptr = stack + kernel_stack_size - sizeof(struct thread_kernel_state);
	thread_initialize_kernel_state(thread);
	savestate = (struct arm_saved_state *) thread->machine.kstackptr;

	savestate->lr = (uint32_t) thread_continue;
	savestate->sp = thread->machine.kstackptr;
	savestate->r[7] = 0x0UL;
	savestate->r[9] = (uint32_t) NULL;
	savestate->cpsr = PSR_SVC_MODE | PSR_INTMASK;
	vfp_state_initialize(&savestate->VFPdata);
	machine_stack_attach_kprintf("thread = %x pc = %x, sp = %x\n", thread, savestate->lr, savestate->sp);
}


/*
 * Routine:	machine_stack_handoff
 *
 */
void
machine_stack_handoff(
		      thread_t old,
		      thread_t new)
{
	vm_offset_t     stack;
	cpu_data_t	*cpu_data_ptr;

	kpc_off_cpu(old);

	stack = machine_stack_detach(old);
	cpu_data_ptr = getCpuDatap();
	new->kernel_stack = stack;
	new->machine.kstackptr = stack + kernel_stack_size - sizeof(struct thread_kernel_state);
	if (stack == old->reserved_stack) {
		assert(new->reserved_stack);
		old->reserved_stack = new->reserved_stack;
		new->reserved_stack = stack;
	}

	pmap_set_pmap(new->map->pmap, new);
	new->machine.CpuDatap = cpu_data_ptr;

#if __SMP__
	/* TODO: Should this be ordered? */
	old->machine.machine_thread_flags &= ~MACHINE_THREAD_FLAGS_ON_CPU;
	new->machine.machine_thread_flags |= MACHINE_THREAD_FLAGS_ON_CPU;
#endif /* __SMP__ */

	machine_set_current_thread(new);
	thread_initialize_kernel_state(new);

	return;
}


/*
 * Routine:	call_continuation
 *
 */
void
call_continuation(
		  thread_continue_t continuation,
		  void *parameter,
		  wait_result_t wresult, 
		  boolean_t enable_interrupts)
{
#define call_continuation_kprintf(x...)	/* kprintf("call_continuation_kprintf:
					 *  " x) */

	call_continuation_kprintf("thread = %x continuation = %x, stack = %x\n", current_thread(), continuation, current_thread()->machine.kstackptr);
	Call_continuation(continuation, parameter, wresult, enable_interrupts);
}

void arm_debug_set(arm_debug_state_t *debug_state)
{
	/* If this CPU supports the memory-mapped debug interface, use it, otherwise
	 * attempt the Extended CP14 interface.  The two routines need to be kept in sync,
	 * functionality-wise.
	 */
	struct cpu_data *cpu_data_ptr;
	arm_debug_info_t *debug_info = arm_debug_info();
	boolean_t       intr;

	intr = ml_set_interrupts_enabled(FALSE);
	cpu_data_ptr = getCpuDatap();

	// Set current user debug
	cpu_data_ptr->cpu_user_debug = debug_state;

	if (debug_info->memory_mapped_core_debug) {
		int i;
		uintptr_t debug_map = cpu_data_ptr->cpu_debug_interface_map;

		// unlock debug registers
		*(volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGLAR) = ARM_DBG_LOCK_ACCESS_KEY;

		// read DBGPRSR to clear the sticky power-down bit (necessary to access debug registers)
		*(volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGPRSR);

		// enable monitor mode (needed to set and use debug registers)
		*(volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGDSCR) |= ARM_DBGDSCR_MDBGEN;

		// first turn off all breakpoints/watchpoints
		for (i = 0; i < 16; i++) {
			((volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGBCR))[i] = 0;
			((volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGWCR))[i] = 0;
		}

		// if (debug_state == NULL) disable monitor mode
		if (debug_state == NULL) {
			*(volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGDSCR) &= ~ARM_DBGDSCR_MDBGEN;
		} else {
			for (i = 0; i < 16; i++) {
				((volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGBVR))[i] = debug_state->bvr[i];
				((volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGBCR))[i] = debug_state->bcr[i];
				((volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGWVR))[i] = debug_state->wvr[i];
				((volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGWCR))[i] = debug_state->wcr[i];
			}
		}	    

		// lock debug registers
		*(volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGLAR) = 0;

    } else if (debug_info->coprocessor_core_debug) {
		arm_debug_set_cp14(debug_state);
	}

	(void) ml_set_interrupts_enabled(intr);

	return;
}

/*
 * Duplicate one arm_debug_state_t to another.  "all" parameter
 * is ignored in the case of ARM -- Is this the right assumption?
 */
void
copy_debug_state(
		arm_debug_state_t *src,
		arm_debug_state_t *target,
		__unused boolean_t all)
{
	bcopy(src, target, sizeof(arm_debug_state_t));
}

kern_return_t
machine_thread_set_tsd_base(
	thread_t			thread,
	mach_vm_offset_t	tsd_base)
{

	if (thread->task == kernel_task) {
		return KERN_INVALID_ARGUMENT;
	}

	if (tsd_base & 0x3) {
		return KERN_INVALID_ARGUMENT;
	}

	if (tsd_base > UINT32_MAX)
		tsd_base = 0ULL;

	thread->machine.cthread_self = tsd_base;

	/* For current thread, make the TSD base active immediately */
	if (thread == current_thread()) {

		mp_disable_preemption();
		__asm__ volatile(
			"mrc    p15, 0, r6, c13, c0, 3\n"
			"and	r6, r6, #3\n"
			"orr	r6, r6, %0\n"
			"mcr	p15, 0, r6, c13, c0, 3\n"
			:		/* output */
			: "r"((uint32_t)tsd_base)	/* input */
			: "r6"		/* clobbered register */
			);
		mp_enable_preemption();

	}

	return KERN_SUCCESS;
}

void
machine_tecs(__unused thread_t thr)
{
}

int
machine_csv(__unused cpuvn_e cve)
{
	return 0;
}
Commit	Line	Data
5ba3f43e	1	/*
0a7de745	2	* Copyright (c) 2007-2019 Apple Inc. All rights reserved.
5ba3f43e A	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28	#include <debug.h>
	29
	30	#include <types.h>
	31
	32	#include <mach/mach_types.h>
	33	#include <mach/thread_status.h>
	34	#include <mach/vm_types.h>
	35
	36	#include <kern/kern_types.h>
	37	#include <kern/task.h>
	38	#include <kern/thread.h>
	39	#include <kern/misc_protos.h>
	40	#include <kern/mach_param.h>
	41	#include <kern/spl.h>
	42	#include <kern/machine.h>
	43	#include <kern/kalloc.h>
	44	#include <kern/kpc.h>
	45
	46	#include <arm/proc_reg.h>
	47	#include <arm/cpu_data_internal.h>
	48	#include <arm/misc_protos.h>
	49	#include <arm/cpuid.h>
	50
	51	#include <vm/vm_map.h>
	52	#include <vm/vm_protos.h>
	53
	54	#include <sys/kdebug.h>
	55
	56	extern int debug_task;
	57
	58	zone_t ads_zone; /* zone for debug_state area */
	59
	60	/*
	61	* Routine: consider_machine_collect
	62	*
	63	*/
	64	void
	65	consider_machine_collect(void)
	66	{
67	pmap_gc();
68	}
69
70	/*
71	* Routine: consider_machine_adjust
72	*
73	*/
74	void
75	consider_machine_adjust(void)
76	{
77	}
78
79	/*
80	* Routine: machine_switch_context
81	*
82	*/
83	thread_t
84	machine_switch_context(
85	thread_t old,
86	thread_continue_t continuation,
87	thread_t new)
88	{
89	thread_t retval;
90	cpu_data_t *cpu_data_ptr;
91
92	#define machine_switch_context_kprintf(x...) /* kprintf("machine_switch_con
93	* text: " x) */
94
95	cpu_data_ptr = getCpuDatap();
96	if (old == new)
97	panic("machine_switch_context");
98
99	kpc_off_cpu(old);
100
101	pmap_set_pmap(new->map->pmap, new);
102
103	new->machine.CpuDatap = cpu_data_ptr;
104
0a7de745 A	105	#if __SMP__
	106	/* TODO: Should this be ordered? */
	107	old->machine.machine_thread_flags &= ~MACHINE_THREAD_FLAGS_ON_CPU;
	108	new->machine.machine_thread_flags \|= MACHINE_THREAD_FLAGS_ON_CPU;
	109	#endif /* __SMP__ */
	110
5ba3f43e A	111	machine_switch_context_kprintf("old= %x contination = %x new = %x\n", old, continuation, new);
	112	retval = Switch_context(old, continuation, new);
	113	assert(retval != NULL);
	114
	115	return retval;
	116	}
	117
cb323159 A	118	boolean_t
	119	machine_thread_on_core(thread_t thread)
	120	{
	121	return thread->machine.machine_thread_flags & MACHINE_THREAD_FLAGS_ON_CPU;
	122	}
	123
5ba3f43e A	124	/*
	125	* Routine: machine_thread_create
	126	*
	127	*/
	128	kern_return_t
	129	machine_thread_create(
	130	thread_t thread,
	131	#if !__ARM_USER_PROTECT__
	132	__unused
	133	#endif
	134	task_t task)
	135	{
	136
	137	#define machine_thread_create_kprintf(x...) /* kprintf("machine_thread_create: " x) */
	138
	139	machine_thread_create_kprintf("thread = %x\n", thread);
	140
	141	if (current_thread() != thread) {
	142	thread->machine.CpuDatap = (cpu_data_t *)0;
	143	}
	144	thread->machine.preemption_count = 0;
	145	thread->machine.cthread_self = 0;
5ba3f43e A	146	#if __ARM_USER_PROTECT__
	147	{
	148	struct pmap *new_pmap = vm_map_pmap(task->map);
	149
	150	thread->machine.kptw_ttb = ((unsigned int) kernel_pmap->ttep) \| TTBR_SETUP;
cb323159	151	thread->machine.asid = new_pmap->hw_asid;
5ba3f43e A	152	if (new_pmap->tte_index_max == NTTES) {
	153	thread->machine.uptw_ttc = 2;
	154	thread->machine.uptw_ttb = ((unsigned int) new_pmap->ttep) \| TTBR_SETUP;
	155	} else {
	156	thread->machine.uptw_ttc = 1;
	157	thread->machine.uptw_ttb = ((unsigned int) new_pmap->ttep ) \| TTBR_SETUP;
	158	}
	159	}
	160	#endif
	161	machine_thread_state_initialize(thread);
	162
	163	return (KERN_SUCCESS);
	164	}
	165
	166	/*
	167	* Routine: machine_thread_destroy
	168	*
	169	*/
	170	void
	171	machine_thread_destroy(
	172	thread_t thread)
	173	{
	174
	175	if (thread->machine.DebugData != NULL) {
	176	if (thread->machine.DebugData == getCpuDatap()->cpu_user_debug)
	177	arm_debug_set(NULL);
	178	zfree(ads_zone, thread->machine.DebugData);
	179	}
	180	}
	181
	182
	183	/*
	184	* Routine: machine_thread_init
	185	*
	186	*/
	187	void
	188	machine_thread_init(void)
	189	{
	190	ads_zone = zinit(sizeof(arm_debug_state_t),
	191	THREAD_CHUNK * (sizeof(arm_debug_state_t)),
	192	THREAD_CHUNK * (sizeof(arm_debug_state_t)),
	193	"arm debug state");
	194	}
	195
ea3f0419 A	196	/*
	197	* Routine: machine_thread_template_init
	198	*
	199	*/
	200	void
	201	machine_thread_template_init(thread_t __unused thr_template)
	202	{
	203	/* Nothing to do on this platform. */
	204	}
5ba3f43e A	205
	206	/*
	207	* Routine: get_useraddr
	208	*
	209	*/
	210	user_addr_t
	211	get_useraddr()
	212	{
	213	return (current_thread()->machine.PcbData.pc);
	214	}
	215
	216	/*
	217	* Routine: machine_stack_detach
	218	*
	219	*/
	220	vm_offset_t
	221	machine_stack_detach(
	222	thread_t thread)
	223	{
	224	vm_offset_t stack;
	225
	226	KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_DETACH),
	227	(uintptr_t)thread_tid(thread), thread->priority, thread->sched_pri, 0, 0);
	228
	229	stack = thread->kernel_stack;
	230	thread->kernel_stack = 0;
	231	thread->machine.kstackptr = 0;
	232
	233	return (stack);
	234	}
	235
	236
	237	/*
	238	* Routine: machine_stack_attach
	239	*
	240	*/
	241	void
	242	machine_stack_attach(
	243	thread_t thread,
	244	vm_offset_t stack)
	245	{
	246	struct arm_saved_state *savestate;
	247
	248	#define machine_stack_attach_kprintf(x...) /* kprintf("machine_stack_attach: " x) */
	249
	250	KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_ATTACH),
	251	(uintptr_t)thread_tid(thread), thread->priority, thread->sched_pri, 0, 0);
	252
	253	thread->kernel_stack = stack;
	254	thread->machine.kstackptr = stack + kernel_stack_size - sizeof(struct thread_kernel_state);
	255	thread_initialize_kernel_state(thread);
	256	savestate = (struct arm_saved_state *) thread->machine.kstackptr;
	257
	258	savestate->lr = (uint32_t) thread_continue;
	259	savestate->sp = thread->machine.kstackptr;
	260	savestate->r[7] = 0x0UL;
	261	savestate->r[9] = (uint32_t) NULL;
	262	savestate->cpsr = PSR_SVC_MODE \| PSR_INTMASK;
94ff46dc	263	vfp_state_initialize(&savestate->VFPdata);
5ba3f43e A	264	machine_stack_attach_kprintf("thread = %x pc = %x, sp = %x\n", thread, savestate->lr, savestate->sp);
	265	}
	266
	267
	268	/*
	269	* Routine: machine_stack_handoff
	270	*
	271	*/
	272	void
	273	machine_stack_handoff(
	274	thread_t old,
	275	thread_t new)
	276	{
	277	vm_offset_t stack;
	278	cpu_data_t *cpu_data_ptr;
	279
	280	kpc_off_cpu(old);
	281
	282	stack = machine_stack_detach(old);
	283	cpu_data_ptr = getCpuDatap();
	284	new->kernel_stack = stack;
	285	new->machine.kstackptr = stack + kernel_stack_size - sizeof(struct thread_kernel_state);
	286	if (stack == old->reserved_stack) {
	287	assert(new->reserved_stack);
	288	old->reserved_stack = new->reserved_stack;
	289	new->reserved_stack = stack;
	290	}
	291
	292	pmap_set_pmap(new->map->pmap, new);
	293	new->machine.CpuDatap = cpu_data_ptr;
0a7de745 A	294
	295	#if __SMP__
	296	/* TODO: Should this be ordered? */
	297	old->machine.machine_thread_flags &= ~MACHINE_THREAD_FLAGS_ON_CPU;
	298	new->machine.machine_thread_flags \|= MACHINE_THREAD_FLAGS_ON_CPU;
	299	#endif /* __SMP__ */
	300
5ba3f43e A	301	machine_set_current_thread(new);
	302	thread_initialize_kernel_state(new);
	303
	304	return;
	305	}
	306
	307
	308	/*
	309	* Routine: call_continuation
	310	*
	311	*/
	312	void
	313	call_continuation(
	314	thread_continue_t continuation,
	315	void *parameter,
d9a64523 A	316	wait_result_t wresult,
d9a64523 A	317	boolean_t enable_interrupts)
5ba3f43e A	318	{
	319	#define call_continuation_kprintf(x...) /* kprintf("call_continuation_kprintf:
	320	* " x) */
	321
	322	call_continuation_kprintf("thread = %x continuation = %x, stack = %x\n", current_thread(), continuation, current_thread()->machine.kstackptr);
d9a64523	323	Call_continuation(continuation, parameter, wresult, enable_interrupts);
5ba3f43e A	324	}
	325
	326	void arm_debug_set(arm_debug_state_t *debug_state)
	327	{
	328	/* If this CPU supports the memory-mapped debug interface, use it, otherwise
	329	* attempt the Extended CP14 interface. The two routines need to be kept in sync,
	330	* functionality-wise.
	331	*/
	332	struct cpu_data *cpu_data_ptr;
	333	arm_debug_info_t *debug_info = arm_debug_info();
	334	boolean_t intr;
	335
	336	intr = ml_set_interrupts_enabled(FALSE);
	337	cpu_data_ptr = getCpuDatap();
	338
	339	// Set current user debug
	340	cpu_data_ptr->cpu_user_debug = debug_state;
	341
	342	if (debug_info->memory_mapped_core_debug) {
	343	int i;
	344	uintptr_t debug_map = cpu_data_ptr->cpu_debug_interface_map;
	345
	346	// unlock debug registers
	347	(volatile uint32_t )(debug_map + ARM_DEBUG_OFFSET_DBGLAR) = ARM_DBG_LOCK_ACCESS_KEY;
	348
	349	// read DBGPRSR to clear the sticky power-down bit (necessary to access debug registers)
	350	(volatile uint32_t )(debug_map + ARM_DEBUG_OFFSET_DBGPRSR);
	351
	352	// enable monitor mode (needed to set and use debug registers)
	353	(volatile uint32_t )(debug_map + ARM_DEBUG_OFFSET_DBGDSCR) \|= ARM_DBGDSCR_MDBGEN;
	354
	355	// first turn off all breakpoints/watchpoints
	356	for (i = 0; i < 16; i++) {
	357	((volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGBCR))[i] = 0;
	358	((volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGWCR))[i] = 0;
	359	}
	360
	361	// if (debug_state == NULL) disable monitor mode
	362	if (debug_state == NULL) {
	363	(volatile uint32_t )(debug_map + ARM_DEBUG_OFFSET_DBGDSCR) &= ~ARM_DBGDSCR_MDBGEN;
	364	} else {
	365	for (i = 0; i < 16; i++) {
	366	((volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGBVR))[i] = debug_state->bvr[i];
	367	((volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGBCR))[i] = debug_state->bcr[i];
	368	((volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGWVR))[i] = debug_state->wvr[i];
	369	((volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGWCR))[i] = debug_state->wcr[i];
	370	}
	371	}
	372
	373	// lock debug registers
	374	(volatile uint32_t )(debug_map + ARM_DEBUG_OFFSET_DBGLAR) = 0;
	375
	376	} else if (debug_info->coprocessor_core_debug) {
	377	arm_debug_set_cp14(debug_state);
	378	}
	379
	380	(void) ml_set_interrupts_enabled(intr);
	381
	382	return;
	383	}
	384
	385	/*
	386	* Duplicate one arm_debug_state_t to another. "all" parameter
	387	* is ignored in the case of ARM -- Is this the right assumption?
388	*/
389	void
390	copy_debug_state(
391	arm_debug_state_t *src,
392	arm_debug_state_t *target,
393	__unused boolean_t all)
394	{
395	bcopy(src, target, sizeof(arm_debug_state_t));
396	}
397
398	kern_return_t
399	machine_thread_set_tsd_base(
400	thread_t thread,
401	mach_vm_offset_t tsd_base)
402	{
403
404	if (thread->task == kernel_task) {
405	return KERN_INVALID_ARGUMENT;
406	}
407
408	if (tsd_base & 0x3) {
409	return KERN_INVALID_ARGUMENT;
410	}
411
412	if (tsd_base > UINT32_MAX)
413	tsd_base = 0ULL;
414
415	thread->machine.cthread_self = tsd_base;
416
417	/* For current thread, make the TSD base active immediately */
418	if (thread == current_thread()) {
419
420	mp_disable_preemption();
421	__asm__ volatile(
422	"mrc p15, 0, r6, c13, c0, 3\n"
423	"and r6, r6, #3\n"
424	"orr r6, r6, %0\n"
425	"mcr p15, 0, r6, c13, c0, 3\n"
426	: /* output */
427	: "r"((uint32_t)tsd_base) /* input */
428	: "r6" /* clobbered register */
429	);
430	mp_enable_preemption();
431
432	}
433
434	return KERN_SUCCESS;
435	}
0a7de745 A	436
	437	void
	438	machine_tecs(__unused thread_t thr)
	439	{
	440	}
	441
	442	int
	443	machine_csv(__unused cpuvn_e cve)
	444	{
	445	return 0;
	446	}