[apple/xnu.git] / osfmk / i386 / pcb_native.c

/*
 * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
/*
 * @OSF_COPYRIGHT@
 */
/* 
 * Mach Operating System
 * Copyright (c) 1991,1990 Carnegie Mellon University
 * All Rights Reserved.
 * 
 * Permission to use, copy, modify and distribute this software and its
 * documentation is hereby granted, provided that both the copyright
 * notice and this permission notice appear in all copies of the
 * software, derivative works or modified versions, and any portions
 * thereof, and that both notices appear in supporting documentation.
 * 
 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
 * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
 * 
 * Carnegie Mellon requests users of this software to return to
 * 
 *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
 *  School of Computer Science
 *  Carnegie Mellon University
 *  Pittsburgh PA 15213-3890
 * 
 * any improvements or extensions that they make and grant Carnegie Mellon
 * the rights to redistribute these changes.
 */

#include <mach_debug.h>
#include <mach_ldebug.h>

#include <sys/kdebug.h>

#include <mach/kern_return.h>
#include <mach/thread_status.h>
#include <mach/vm_param.h>

#include <kern/counters.h>
#include <kern/kalloc.h>
#include <kern/mach_param.h>
#include <kern/processor.h>
#include <kern/cpu_data.h>
#include <kern/cpu_number.h>
#include <kern/task.h>
#include <kern/thread.h>
#include <kern/sched_prim.h>
#include <kern/misc_protos.h>
#include <kern/assert.h>
#include <kern/spl.h>
#include <kern/machine.h>
#include <ipc/ipc_port.h>
#include <vm/vm_kern.h>
#include <vm/vm_map.h>
#include <vm/pmap.h>
#include <vm/vm_protos.h>

#include <i386/commpage/commpage.h>
#include <i386/cpu_data.h>
#include <i386/cpu_number.h>
#include <i386/eflags.h>
#include <i386/proc_reg.h>
#include <i386/tss.h>
#include <i386/user_ldt.h>
#include <i386/fpu.h>
#include <i386/mp_desc.h>
#include <i386/misc_protos.h>
#include <i386/thread.h>
#include <i386/seg.h>
#include <i386/machine_routines.h>

#if HYPERVISOR
#include <kern/hv_support.h>
#endif

#define ASSERT_IS_16BYTE_MULTIPLE_SIZEOF(_type_)	\
extern char assert_is_16byte_multiple_sizeof_ ## _type_	\
		[(sizeof(_type_) % 16) == 0 ? 1 : -1]

/* Compile-time checks for vital save area sizing: */
ASSERT_IS_16BYTE_MULTIPLE_SIZEOF(x86_64_intr_stack_frame_t);
ASSERT_IS_16BYTE_MULTIPLE_SIZEOF(x86_saved_state_t);

#define DIRECTION_FLAG_DEBUG (DEBUG | DEVELOPMENT)

extern zone_t		iss_zone;		/* zone for saved_state area */
extern zone_t		ids_zone;		/* zone for debug_state area */

void
act_machine_switch_pcb(__unused thread_t old, thread_t new)
{
        pcb_t			pcb = THREAD_TO_PCB(new);
	cpu_data_t      	*cdp = current_cpu_datap();
	struct real_descriptor	*ldtp;
	mach_vm_offset_t	pcb_stack_top;

	assert(new->kernel_stack != 0);
	assert(ml_get_interrupts_enabled() == FALSE);
#ifdef	DIRECTION_FLAG_DEBUG
	if (x86_get_flags() & EFL_DF) {
		panic("Direction flag detected: 0x%lx", x86_get_flags());
	}
#endif

	/*
	 * Clear segment state
	 * unconditionally for DS/ES/FS but more carefully for GS whose
	 * cached state we track.
	 */
	set_ds(NULL_SEG);
	set_es(NULL_SEG);
	set_fs(NULL_SEG);

	if (get_gs() != NULL_SEG) {
		swapgs();		/* switch to user's GS context */
		set_gs(NULL_SEG);
		swapgs();		/* and back to kernel */

		/* record the active machine state lost */
		cdp->cpu_uber.cu_user_gs_base = 0;
	} 

	vm_offset_t			isf;

	/*
	 * Set pointer to PCB's interrupt stack frame in cpu data.
	 * Used by syscall and double-fault trap handlers.
	 */
	isf = (vm_offset_t) &pcb->iss->ss_64.isf;
	cdp->cpu_uber.cu_isf = isf;
	pcb_stack_top = (vm_offset_t) (pcb->iss + 1);
	/* require 16-byte alignment */
	assert((pcb_stack_top & 0xF) == 0);

	current_ktss64()->rsp0 = cdp->cpu_desc_index.cdi_sstku;
	/*
	 * Top of temporary sysenter stack points to pcb stack.
	 * Although this is not normally used by 64-bit users,
	 * it needs to be set in case a sysenter is attempted.
	 */
	*current_sstk64() = pcb_stack_top;

	cdp->cd_estack = cpu_shadowp(cdp->cpu_number)->cd_estack = cdp->cpu_desc_index.cdi_sstku;

	if (is_saved_state64(pcb->iss)) {

		cdp->cpu_task_map = new->map->pmap->pm_task_map; 

		/*
		 * Enable the 64-bit user code segment, USER64_CS.
		 * Disable the 32-bit user code segment, USER_CS.
		 */
		gdt_desc_p(USER64_CS)->access |= ACC_PL_U;
		gdt_desc_p(USER_CS)->access &= ~ACC_PL_U;

		/*
		 * Switch user's GS base if necessary
		 * by setting the Kernel's GS base MSR
		 * - this will become the user's on the swapgs when
		 * returning to user-space.  Avoid this for
		 * kernel threads (no user TLS support required)
		 * and verify the memory shadow of the segment base
		 * in the event it was altered in user space.
		 */
		if ((pcb->cthread_self != 0) || (new->task != kernel_task)) {
			if ((cdp->cpu_uber.cu_user_gs_base != pcb->cthread_self) ||
			    (pcb->cthread_self != rdmsr64(MSR_IA32_KERNEL_GS_BASE))) {
				cdp->cpu_uber.cu_user_gs_base = pcb->cthread_self;
				wrmsr64(MSR_IA32_KERNEL_GS_BASE, pcb->cthread_self);
			}
		}
	} else {

		cdp->cpu_task_map = TASK_MAP_32BIT;

		/*
		 * Disable USER64_CS
		 * Enable USER_CS
		 */

		/* It's possible that writing to the GDT areas
		 * is expensive, if the processor intercepts those
		 * writes to invalidate its internal segment caches
		 * TODO: perhaps only do this if switching bitness
		 */
		gdt_desc_p(USER64_CS)->access &= ~ACC_PL_U;
		gdt_desc_p(USER_CS)->access |= ACC_PL_U;

		/*
		 * Set the thread`s cthread (a.k.a pthread)
		 * For 32-bit user this involves setting the USER_CTHREAD
		 * descriptor in the LDT to point to the cthread data.
		 * The involves copying in the pre-initialized descriptor.
		 */ 
		ldtp = (struct real_descriptor *)current_ldt();
		ldtp[sel_idx(USER_CTHREAD)] = pcb->cthread_desc;
		if (pcb->uldt_selector != 0)
			ldtp[sel_idx(pcb->uldt_selector)] = pcb->uldt_desc;
		cdp->cpu_uber.cu_user_gs_base = pcb->cthread_self;

		/*
		 * Set the thread`s LDT or LDT entry.
		 */
		if (new->task == TASK_NULL || new->task->i386_ldt == 0) {
			/*
			 * Use system LDT.
			 */
		       	ml_cpu_set_ldt(KERNEL_LDT);
		} else {
			/*
			 * Task has its own LDT.
			 */
			user_ldt_set(new);
		}
	}

	/*
	 * Bump the scheduler generation count in the commpage.
	 * This can be read by user code to detect its preemption.
	 */
	commpage_sched_gen_inc();
}

kern_return_t
thread_set_wq_state32(thread_t thread, thread_state_t tstate)
{
        x86_thread_state32_t	*state;
        x86_saved_state32_t	*saved_state;
	thread_t curth = current_thread();
	spl_t			s=0;

	pal_register_cache_state(thread, DIRTY);

	saved_state = USER_REGS32(thread);

	state = (x86_thread_state32_t *)tstate;
	
	if (curth != thread) {
		s = splsched();
	        thread_lock(thread);
	}

	saved_state->ebp = 0;
	saved_state->eip = state->eip;
	saved_state->eax = state->eax;
	saved_state->ebx = state->ebx;
	saved_state->ecx = state->ecx;
	saved_state->edx = state->edx;
	saved_state->edi = state->edi;
	saved_state->esi = state->esi;
	saved_state->uesp = state->esp;
	saved_state->efl = EFL_USER_SET;

	saved_state->cs = USER_CS;
	saved_state->ss = USER_DS;
	saved_state->ds = USER_DS;
	saved_state->es = USER_DS;

	if (curth != thread) {
	        thread_unlock(thread);
		splx(s);
	}

	return KERN_SUCCESS;
}


kern_return_t
thread_set_wq_state64(thread_t thread, thread_state_t tstate)
{
        x86_thread_state64_t	*state;
        x86_saved_state64_t	*saved_state;
	thread_t curth = current_thread();
	spl_t			s=0;

	saved_state = USER_REGS64(thread);
	state = (x86_thread_state64_t *)tstate;
	
	/* Disallow setting non-canonical PC or stack */
	if (!IS_USERADDR64_CANONICAL(state->rsp) ||
	    !IS_USERADDR64_CANONICAL(state->rip)) {
		return KERN_FAILURE;
	}

	pal_register_cache_state(thread, DIRTY);

	if (curth != thread) {
		s = splsched();
	        thread_lock(thread);
	}

	saved_state->rbp = 0;
	saved_state->rdi = state->rdi;
	saved_state->rsi = state->rsi;
	saved_state->rdx = state->rdx;
	saved_state->rcx = state->rcx;
	saved_state->r8  = state->r8;
	saved_state->r9  = state->r9;

	saved_state->isf.rip = state->rip;
	saved_state->isf.rsp = state->rsp;
	saved_state->isf.cs = USER64_CS;
	saved_state->isf.rflags = EFL_USER_SET;

	if (curth != thread) {
	        thread_unlock(thread);
		splx(s);
	}

	return KERN_SUCCESS;
}

/*
 * Initialize the machine-dependent state for a new thread.
 */
kern_return_t
machine_thread_create(
	thread_t		thread,
	task_t			task)
{
        pcb_t			pcb = THREAD_TO_PCB(thread);

#if NCOPY_WINDOWS > 0
	inval_copy_windows(thread);

	thread->machine.physwindow_pte = 0;
	thread->machine.physwindow_busy = 0;
#endif

	/*
	 * Allocate save frame only if required.
	 */
	if (pcb->iss == NULL) {
		assert((get_preemption_level() == 0));
		pcb->iss = (x86_saved_state_t *) zalloc(iss_zone);
		if (pcb->iss == NULL)
			panic("iss_zone");
	}

	/*
	 * Ensure that the synthesized 32-bit state including
	 * the 64-bit interrupt state can be acommodated in the 
	 * 64-bit state we allocate for both 32-bit and 64-bit threads.
	 */
	assert(sizeof(pcb->iss->ss_32) + sizeof(pcb->iss->ss_64.isf) <=
	       sizeof(pcb->iss->ss_64));

	bzero((char *)pcb->iss, sizeof(x86_saved_state_t));

        if (task_has_64BitAddr(task)) {
		pcb->iss->flavor = x86_SAVED_STATE64;

		pcb->iss->ss_64.isf.cs = USER64_CS;
		pcb->iss->ss_64.isf.ss = USER_DS;
		pcb->iss->ss_64.fs = USER_DS;
		pcb->iss->ss_64.gs = USER_DS;
		pcb->iss->ss_64.isf.rflags = EFL_USER_SET;
	} else {
		pcb->iss->flavor = x86_SAVED_STATE32;

		pcb->iss->ss_32.cs = USER_CS;
		pcb->iss->ss_32.ss = USER_DS;
		pcb->iss->ss_32.ds = USER_DS;
		pcb->iss->ss_32.es = USER_DS;
		pcb->iss->ss_32.fs = USER_DS;
		pcb->iss->ss_32.gs = USER_DS;
		pcb->iss->ss_32.efl = EFL_USER_SET;
	}

	simple_lock_init(&pcb->lock, 0);

	pcb->cthread_self = 0;
	pcb->uldt_selector = 0;
	pcb->thread_gpu_ns = 0;
	/* Ensure that the "cthread" descriptor describes a valid
	 * segment.
	 */
	if ((pcb->cthread_desc.access & ACC_P) == 0) {
		pcb->cthread_desc = *gdt_desc_p(USER_DS);
	}

	return(KERN_SUCCESS);
}

/*
 * Machine-dependent cleanup prior to destroying a thread
 */
void
machine_thread_destroy(
	thread_t		thread)
{
	pcb_t	pcb = THREAD_TO_PCB(thread);

#if HYPERVISOR
	if (thread->hv_thread_target) {
		hv_callbacks.thread_destroy(thread->hv_thread_target);
		thread->hv_thread_target = NULL;
	}
#endif

	if (pcb->ifps != 0)
		fpu_free(thread, pcb->ifps);
	if (pcb->iss != 0) {
		zfree(iss_zone, pcb->iss);
		pcb->iss = 0;
	}
	if (pcb->ids) {
		zfree(ids_zone, pcb->ids);
		pcb->ids = NULL;
	}
}

kern_return_t
machine_thread_set_tsd_base(
	thread_t			thread,
	mach_vm_offset_t	tsd_base)
{

	if (thread->task == kernel_task) {
		return KERN_INVALID_ARGUMENT;
	}

	if (thread_is_64bit(thread)) {
		/* check for canonical address, set 0 otherwise  */
		if (!IS_USERADDR64_CANONICAL(tsd_base))
			tsd_base = 0ULL;
	} else {
		if (tsd_base > UINT32_MAX)
			tsd_base = 0ULL;
	}

	pcb_t pcb = THREAD_TO_PCB(thread);
	pcb->cthread_self = tsd_base;

	if (!thread_is_64bit(thread)) {
		/* Set up descriptor for later use */
		struct real_descriptor desc = {
			.limit_low = 1,
			.limit_high = 0,
			.base_low = tsd_base & 0xffff,
			.base_med = (tsd_base >> 16) & 0xff,
			.base_high = (tsd_base >> 24) & 0xff,
			.access = ACC_P|ACC_PL_U|ACC_DATA_W,
			.granularity = SZ_32|SZ_G,
		};

		pcb->cthread_desc = desc;
		saved_state32(pcb->iss)->gs = USER_CTHREAD;
	}

	/* For current thread, make the TSD base active immediately */
	if (thread == current_thread()) {

		if (thread_is_64bit(thread)) {
			cpu_data_t              *cdp;

			mp_disable_preemption();
			cdp = current_cpu_datap();
			if ((cdp->cpu_uber.cu_user_gs_base != pcb->cthread_self) ||
				(pcb->cthread_self != rdmsr64(MSR_IA32_KERNEL_GS_BASE)))
				wrmsr64(MSR_IA32_KERNEL_GS_BASE, tsd_base);
			cdp->cpu_uber.cu_user_gs_base = tsd_base;
			mp_enable_preemption();
		} else {

			/* assign descriptor */
			mp_disable_preemption();
			*ldt_desc_p(USER_CTHREAD) = pcb->cthread_desc;
			mp_enable_preemption();
		}
	}

	return KERN_SUCCESS;
}
Commit	Line	Data
6d2010ae	1	/*
39236c6e	2	* Copyright (c) 2000-2012 Apple Inc. All rights reserved.
6d2010ae A	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28	/*
	29	* @OSF_COPYRIGHT@
	30	*/
	31	/*
	32	* Mach Operating System
	33	* Copyright (c) 1991,1990 Carnegie Mellon University
	34	* All Rights Reserved.
	35	*
	36	* Permission to use, copy, modify and distribute this software and its
	37	* documentation is hereby granted, provided that both the copyright
	38	* notice and this permission notice appear in all copies of the
	39	* software, derivative works or modified versions, and any portions
	40	* thereof, and that both notices appear in supporting documentation.
	41	*
	42	* CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
	43	* CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
	44	* ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
	45	*
	46	* Carnegie Mellon requests users of this software to return to
	47	*
	48	* Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
	49	* School of Computer Science
	50	* Carnegie Mellon University
	51	* Pittsburgh PA 15213-3890
	52	*
	53	* any improvements or extensions that they make and grant Carnegie Mellon
	54	* the rights to redistribute these changes.
	55	*/
	56
6d2010ae A	57	#include <mach_debug.h>
	58	#include <mach_ldebug.h>
	59
	60	#include <sys/kdebug.h>
	61
	62	#include <mach/kern_return.h>
	63	#include <mach/thread_status.h>
	64	#include <mach/vm_param.h>
	65
	66	#include <kern/counters.h>
	67	#include <kern/kalloc.h>
	68	#include <kern/mach_param.h>
	69	#include <kern/processor.h>
	70	#include <kern/cpu_data.h>
	71	#include <kern/cpu_number.h>
	72	#include <kern/task.h>
	73	#include <kern/thread.h>
	74	#include <kern/sched_prim.h>
	75	#include <kern/misc_protos.h>
	76	#include <kern/assert.h>
	77	#include <kern/spl.h>
	78	#include <kern/machine.h>
	79	#include <ipc/ipc_port.h>
	80	#include <vm/vm_kern.h>
	81	#include <vm/vm_map.h>
	82	#include <vm/pmap.h>
	83	#include <vm/vm_protos.h>
	84
	85	#include <i386/commpage/commpage.h>
	86	#include <i386/cpu_data.h>
	87	#include <i386/cpu_number.h>
	88	#include <i386/eflags.h>
	89	#include <i386/proc_reg.h>
	90	#include <i386/tss.h>
	91	#include <i386/user_ldt.h>
	92	#include <i386/fpu.h>
	93	#include <i386/mp_desc.h>
	94	#include <i386/misc_protos.h>
	95	#include <i386/thread.h>
6d2010ae A	96	#include <i386/seg.h>
	97	#include <i386/machine_routines.h>
	98
fe8ab488 A	99	#if HYPERVISOR
	100	#include <kern/hv_support.h>
	101	#endif
	102
6d2010ae A	103	#define ASSERT_IS_16BYTE_MULTIPLE_SIZEOF(_type_) \
	104	extern char assert_is_16byte_multiple_sizeof_ ## _type_ \
	105	[(sizeof(_type_) % 16) == 0 ? 1 : -1]
	106
	107	/* Compile-time checks for vital save area sizing: */
	108	ASSERT_IS_16BYTE_MULTIPLE_SIZEOF(x86_64_intr_stack_frame_t);
6d2010ae A	109	ASSERT_IS_16BYTE_MULTIPLE_SIZEOF(x86_saved_state_t);
	110
	111	#define DIRECTION_FLAG_DEBUG (DEBUG \| DEVELOPMENT)
	112
	113	extern zone_t iss_zone; /* zone for saved_state area */
	114	extern zone_t ids_zone; /* zone for debug_state area */
	115
6d2010ae A	116	void
	117	act_machine_switch_pcb(__unused thread_t old, thread_t new)
	118	{
	119	pcb_t pcb = THREAD_TO_PCB(new);
	120	cpu_data_t *cdp = current_cpu_datap();
	121	struct real_descriptor *ldtp;
	122	mach_vm_offset_t pcb_stack_top;
	123
	124	assert(new->kernel_stack != 0);
	125	assert(ml_get_interrupts_enabled() == FALSE);
	126	#ifdef DIRECTION_FLAG_DEBUG
	127	if (x86_get_flags() & EFL_DF) {
	128	panic("Direction flag detected: 0x%lx", x86_get_flags());
	129	}
	130	#endif
	131
6d2010ae A	132	/*
	133	* Clear segment state
	134	* unconditionally for DS/ES/FS but more carefully for GS whose
	135	* cached state we track.
	136	*/
	137	set_ds(NULL_SEG);
	138	set_es(NULL_SEG);
	139	set_fs(NULL_SEG);
5c9f4661	140
6d2010ae A	141	if (get_gs() != NULL_SEG) {
	142	swapgs(); /* switch to user's GS context */
	143	set_gs(NULL_SEG);
	144	swapgs(); /* and back to kernel */
	145
	146	/* record the active machine state lost */
	147	cdp->cpu_uber.cu_user_gs_base = 0;
	148	}
	149
39236c6e	150	vm_offset_t isf;
6d2010ae	151
39236c6e A	152	/*
	153	* Set pointer to PCB's interrupt stack frame in cpu data.
	154	* Used by syscall and double-fault trap handlers.
	155	*/
	156	isf = (vm_offset_t) &pcb->iss->ss_64.isf;
	157	cdp->cpu_uber.cu_isf = isf;
	158	pcb_stack_top = (vm_offset_t) (pcb->iss + 1);
	159	/* require 16-byte alignment */
	160	assert((pcb_stack_top & 0xF) == 0);
6d2010ae	161
5c9f4661	162	current_ktss64()->rsp0 = cdp->cpu_desc_index.cdi_sstku;
39236c6e A	163	/*
	164	* Top of temporary sysenter stack points to pcb stack.
	165	* Although this is not normally used by 64-bit users,
	166	* it needs to be set in case a sysenter is attempted.
	167	*/
	168	*current_sstk64() = pcb_stack_top;
	169
5c9f4661 A	170	cdp->cd_estack = cpu_shadowp(cdp->cpu_number)->cd_estack = cdp->cpu_desc_index.cdi_sstku;
5c9f4661 A	171
39236c6e	172	if (is_saved_state64(pcb->iss)) {
6d2010ae A	173
	174	cdp->cpu_task_map = new->map->pmap->pm_task_map;
	175
	176	/*
	177	* Enable the 64-bit user code segment, USER64_CS.
	178	* Disable the 32-bit user code segment, USER_CS.
	179	*/
00867663 A	180	gdt_desc_p(USER64_CS)->access \|= ACC_PL_U;
00867663 A	181	gdt_desc_p(USER_CS)->access &= ~ACC_PL_U;
6d2010ae A	182
	183	/*
	184	* Switch user's GS base if necessary
	185	* by setting the Kernel's GS base MSR
	186	* - this will become the user's on the swapgs when
	187	* returning to user-space. Avoid this for
	188	* kernel threads (no user TLS support required)
	189	* and verify the memory shadow of the segment base
	190	* in the event it was altered in user space.
	191	*/
	192	if ((pcb->cthread_self != 0) \|\| (new->task != kernel_task)) {
00867663 A	193	if ((cdp->cpu_uber.cu_user_gs_base != pcb->cthread_self) \|\|
00867663 A	194	(pcb->cthread_self != rdmsr64(MSR_IA32_KERNEL_GS_BASE))) {
6d2010ae A	195	cdp->cpu_uber.cu_user_gs_base = pcb->cthread_self;
	196	wrmsr64(MSR_IA32_KERNEL_GS_BASE, pcb->cthread_self);
	197	}
	198	}
39236c6e	199	} else {
6d2010ae A	200
6d2010ae A	201	cdp->cpu_task_map = TASK_MAP_32BIT;
6d2010ae A	202
	203	/*
	204	* Disable USER64_CS
	205	* Enable USER_CS
	206	*/
00867663 A	207
	208	/* It's possible that writing to the GDT areas
	209	* is expensive, if the processor intercepts those
	210	* writes to invalidate its internal segment caches
	211	* TODO: perhaps only do this if switching bitness
	212	*/
	213	gdt_desc_p(USER64_CS)->access &= ~ACC_PL_U;
	214	gdt_desc_p(USER_CS)->access \|= ACC_PL_U;
6d2010ae A	215
	216	/*
	217	* Set the thread`s cthread (a.k.a pthread)
	218	* For 32-bit user this involves setting the USER_CTHREAD
	219	* descriptor in the LDT to point to the cthread data.
	220	* The involves copying in the pre-initialized descriptor.
	221	*/
	222	ldtp = (struct real_descriptor *)current_ldt();
	223	ldtp[sel_idx(USER_CTHREAD)] = pcb->cthread_desc;
	224	if (pcb->uldt_selector != 0)
	225	ldtp[sel_idx(pcb->uldt_selector)] = pcb->uldt_desc;
	226	cdp->cpu_uber.cu_user_gs_base = pcb->cthread_self;
	227
	228	/*
	229	* Set the thread`s LDT or LDT entry.
	230	*/
	231	if (new->task == TASK_NULL \|\| new->task->i386_ldt == 0) {
	232	/*
	233	* Use system LDT.
	234	*/
	235	ml_cpu_set_ldt(KERNEL_LDT);
	236	} else {
	237	/*
	238	* Task has its own LDT.
	239	*/
	240	user_ldt_set(new);
	241	}
	242	}
	243
6d2010ae A	244	/*
	245	* Bump the scheduler generation count in the commpage.
	246	* This can be read by user code to detect its preemption.
	247	*/
	248	commpage_sched_gen_inc();
	249	}
39236c6e A	250
39236c6e A	251	kern_return_t
6d2010ae A	252	thread_set_wq_state32(thread_t thread, thread_state_t tstate)
	253	{
	254	x86_thread_state32_t *state;
	255	x86_saved_state32_t *saved_state;
	256	thread_t curth = current_thread();
	257	spl_t s=0;
	258
	259	pal_register_cache_state(thread, DIRTY);
	260
	261	saved_state = USER_REGS32(thread);
	262
	263	state = (x86_thread_state32_t *)tstate;
	264
	265	if (curth != thread) {
	266	s = splsched();
	267	thread_lock(thread);
	268	}
	269
	270	saved_state->ebp = 0;
	271	saved_state->eip = state->eip;
	272	saved_state->eax = state->eax;
	273	saved_state->ebx = state->ebx;
	274	saved_state->ecx = state->ecx;
	275	saved_state->edx = state->edx;
	276	saved_state->edi = state->edi;
	277	saved_state->esi = state->esi;
	278	saved_state->uesp = state->esp;
	279	saved_state->efl = EFL_USER_SET;
	280
	281	saved_state->cs = USER_CS;
	282	saved_state->ss = USER_DS;
	283	saved_state->ds = USER_DS;
	284	saved_state->es = USER_DS;
	285
	286	if (curth != thread) {
	287	thread_unlock(thread);
	288	splx(s);
	289	}
39236c6e A	290
39236c6e A	291	return KERN_SUCCESS;
6d2010ae A	292	}
	293
	294
39236c6e	295	kern_return_t
6d2010ae A	296	thread_set_wq_state64(thread_t thread, thread_state_t tstate)
	297	{
	298	x86_thread_state64_t *state;
	299	x86_saved_state64_t *saved_state;
	300	thread_t curth = current_thread();
	301	spl_t s=0;
	302
6d2010ae A	303	saved_state = USER_REGS64(thread);
	304	state = (x86_thread_state64_t *)tstate;
	305
39236c6e A	306	/* Disallow setting non-canonical PC or stack */
	307	if (!IS_USERADDR64_CANONICAL(state->rsp) \|\|
	308	!IS_USERADDR64_CANONICAL(state->rip)) {
	309	return KERN_FAILURE;
	310	}
	311
	312	pal_register_cache_state(thread, DIRTY);
	313
6d2010ae A	314	if (curth != thread) {
	315	s = splsched();
	316	thread_lock(thread);
	317	}
	318
	319	saved_state->rbp = 0;
	320	saved_state->rdi = state->rdi;
	321	saved_state->rsi = state->rsi;
	322	saved_state->rdx = state->rdx;
	323	saved_state->rcx = state->rcx;
	324	saved_state->r8 = state->r8;
	325	saved_state->r9 = state->r9;
	326
	327	saved_state->isf.rip = state->rip;
	328	saved_state->isf.rsp = state->rsp;
	329	saved_state->isf.cs = USER64_CS;
	330	saved_state->isf.rflags = EFL_USER_SET;
	331
	332	if (curth != thread) {
	333	thread_unlock(thread);
	334	splx(s);
	335	}
39236c6e A	336
39236c6e A	337	return KERN_SUCCESS;
6d2010ae A	338	}
	339
	340	/*
	341	* Initialize the machine-dependent state for a new thread.
	342	*/
	343	kern_return_t
	344	machine_thread_create(
	345	thread_t thread,
	346	task_t task)
	347	{
	348	pcb_t pcb = THREAD_TO_PCB(thread);
6d2010ae A	349
	350	#if NCOPY_WINDOWS > 0
	351	inval_copy_windows(thread);
	352
	353	thread->machine.physwindow_pte = 0;
	354	thread->machine.physwindow_busy = 0;
	355	#endif
	356
	357	/*
	358	* Allocate save frame only if required.
	359	*/
39236c6e	360	if (pcb->iss == NULL) {
6d2010ae	361	assert((get_preemption_level() == 0));
39236c6e A	362	pcb->iss = (x86_saved_state_t *) zalloc(iss_zone);
39236c6e A	363	if (pcb->iss == NULL)
6d2010ae A	364	panic("iss_zone");
	365	}
	366
39236c6e	367	/*
00867663	368	* Ensure that the synthesized 32-bit state including
39236c6e A	369	* the 64-bit interrupt state can be acommodated in the
	370	* 64-bit state we allocate for both 32-bit and 64-bit threads.
	371	*/
	372	assert(sizeof(pcb->iss->ss_32) + sizeof(pcb->iss->ss_64.isf) <=
	373	sizeof(pcb->iss->ss_64));
6d2010ae	374
39236c6e	375	bzero((char *)pcb->iss, sizeof(x86_saved_state_t));
6d2010ae	376
39236c6e A	377	if (task_has_64BitAddr(task)) {
39236c6e A	378	pcb->iss->flavor = x86_SAVED_STATE64;
6d2010ae	379
39236c6e A	380	pcb->iss->ss_64.isf.cs = USER64_CS;
	381	pcb->iss->ss_64.isf.ss = USER_DS;
	382	pcb->iss->ss_64.fs = USER_DS;
	383	pcb->iss->ss_64.gs = USER_DS;
	384	pcb->iss->ss_64.isf.rflags = EFL_USER_SET;
6d2010ae	385	} else {
39236c6e A	386	pcb->iss->flavor = x86_SAVED_STATE32;
	387
	388	pcb->iss->ss_32.cs = USER_CS;
	389	pcb->iss->ss_32.ss = USER_DS;
	390	pcb->iss->ss_32.ds = USER_DS;
	391	pcb->iss->ss_32.es = USER_DS;
	392	pcb->iss->ss_32.fs = USER_DS;
	393	pcb->iss->ss_32.gs = USER_DS;
	394	pcb->iss->ss_32.efl = EFL_USER_SET;
6d2010ae	395	}
6d2010ae A	396
	397	simple_lock_init(&pcb->lock, 0);
	398
6d2010ae A	399	pcb->cthread_self = 0;
6d2010ae A	400	pcb->uldt_selector = 0;
fe8ab488	401	pcb->thread_gpu_ns = 0;
6d2010ae A	402	/* Ensure that the "cthread" descriptor describes a valid
	403	* segment.
	404	*/
	405	if ((pcb->cthread_desc.access & ACC_P) == 0) {
5c9f4661	406	pcb->cthread_desc = *gdt_desc_p(USER_DS);
6d2010ae A	407	}
	408
	409	return(KERN_SUCCESS);
	410	}
	411
	412	/*
	413	* Machine-dependent cleanup prior to destroying a thread
	414	*/
	415	void
	416	machine_thread_destroy(
	417	thread_t thread)
	418	{
39037602	419	pcb_t pcb = THREAD_TO_PCB(thread);
6d2010ae	420
fe8ab488 A	421	#if HYPERVISOR
	422	if (thread->hv_thread_target) {
	423	hv_callbacks.thread_destroy(thread->hv_thread_target);
	424	thread->hv_thread_target = NULL;
	425	}
	426	#endif
	427
6d2010ae	428	if (pcb->ifps != 0)
5ba3f43e	429	fpu_free(thread, pcb->ifps);
39236c6e A	430	if (pcb->iss != 0) {
	431	zfree(iss_zone, pcb->iss);
	432	pcb->iss = 0;
6d2010ae A	433	}
	434	if (pcb->ids) {
	435	zfree(ids_zone, pcb->ids);
	436	pcb->ids = NULL;
	437	}
	438	}
fe8ab488 A	439
	440	kern_return_t
	441	machine_thread_set_tsd_base(
	442	thread_t thread,
	443	mach_vm_offset_t tsd_base)
	444	{
	445
	446	if (thread->task == kernel_task) {
	447	return KERN_INVALID_ARGUMENT;
	448	}
	449
	450	if (thread_is_64bit(thread)) {
	451	/* check for canonical address, set 0 otherwise */
	452	if (!IS_USERADDR64_CANONICAL(tsd_base))
	453	tsd_base = 0ULL;
	454	} else {
	455	if (tsd_base > UINT32_MAX)
	456	tsd_base = 0ULL;
	457	}
	458
	459	pcb_t pcb = THREAD_TO_PCB(thread);
	460	pcb->cthread_self = tsd_base;
	461
	462	if (!thread_is_64bit(thread)) {
	463	/* Set up descriptor for later use */
	464	struct real_descriptor desc = {
	465	.limit_low = 1,
	466	.limit_high = 0,
	467	.base_low = tsd_base & 0xffff,
	468	.base_med = (tsd_base >> 16) & 0xff,
	469	.base_high = (tsd_base >> 24) & 0xff,
	470	.access = ACC_P\|ACC_PL_U\|ACC_DATA_W,
	471	.granularity = SZ_32\|SZ_G,
	472	};
	473
	474	pcb->cthread_desc = desc;
	475	saved_state32(pcb->iss)->gs = USER_CTHREAD;
	476	}
	477
	478	/* For current thread, make the TSD base active immediately */
	479	if (thread == current_thread()) {
	480
	481	if (thread_is_64bit(thread)) {
	482	cpu_data_t *cdp;
	483
	484	mp_disable_preemption();
	485	cdp = current_cpu_datap();
	486	if ((cdp->cpu_uber.cu_user_gs_base != pcb->cthread_self) \|\|
	487	(pcb->cthread_self != rdmsr64(MSR_IA32_KERNEL_GS_BASE)))
	488	wrmsr64(MSR_IA32_KERNEL_GS_BASE, tsd_base);
	489	cdp->cpu_uber.cu_user_gs_base = tsd_base;
	490	mp_enable_preemption();
	491	} else {
	492
	493	/* assign descriptor */
	494	mp_disable_preemption();
	495	*ldt_desc_p(USER_CTHREAD) = pcb->cthread_desc;
	496	mp_enable_preemption();
	497	}
	498	}
	499
	500	return KERN_SUCCESS;
	501	}