[apple/xnu.git] / osfmk / i386 / cpu_data.h

/*
 * Copyright (c) 2000-2008 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
/*
 * @OSF_COPYRIGHT@
 * 
 */

#ifndef	I386_CPU_DATA
#define I386_CPU_DATA

#include <mach_assert.h>

#if	defined(__GNUC__)

#include <kern/assert.h>
#include <kern/kern_types.h>
#include <kern/processor.h>
#include <kern/pms.h>
#include <pexpert/pexpert.h>
#include <mach/i386/thread_status.h>
#include <i386/rtclock.h>
#include <i386/pmCPU.h>
#include <i386/cpu_topology.h>

#include <i386/vmx/vmx_cpu.h>

/*
 * Data structures referenced (anonymously) from per-cpu data:
 */
struct cpu_cons_buffer;
struct cpu_desc_table;
struct mca_state;


/*
 * Data structures embedded in per-cpu data:
 */
typedef struct rtclock_timer {
	queue_head_t	queue;
	uint64_t		deadline;
	boolean_t		is_set;
	boolean_t		has_expired;
} rtclock_timer_t;


typedef struct {
	struct i386_tss         *cdi_ktss;
#if    MACH_KDB
	struct i386_tss         *cdi_dbtss;
#endif /* MACH_KDB */
	struct fake_descriptor  *cdi_gdt;
	struct fake_descriptor  *cdi_idt;
	struct fake_descriptor  *cdi_ldt;
	vm_offset_t		cdi_sstk;
} cpu_desc_index_t;

typedef enum {
	TASK_MAP_32BIT,			/* 32-bit, compatibility mode */ 
	TASK_MAP_64BIT,			/* 64-bit, separate address space */ 
	TASK_MAP_64BIT_SHARED		/* 64-bit, kernel-shared addr space */
} task_map_t;

/*
 * This structure is used on entry into the (uber-)kernel on syscall from
 * a 64-bit user. It contains the address of the machine state save area
 * for the current thread and a temporary place to save the user's rsp
 * before loading this address into rsp.
 */
typedef struct {
	addr64_t	cu_isf;		/* thread->pcb->iss.isf */
	uint64_t	cu_tmp;		/* temporary scratch */	
        addr64_t	cu_user_gs_base;
} cpu_uber_t;

/*
 * Per-cpu data.
 *
 * Each processor has a per-cpu data area which is dereferenced through the
 * current_cpu_datap() macro. For speed, the %gs segment is based here, and
 * using this, inlines provides single-instruction access to frequently used
 * members - such as get_cpu_number()/cpu_number(), and get_active_thread()/
 * current_thread(). 
 * 
 * Cpu data owned by another processor can be accessed using the
 * cpu_datap(cpu_number) macro which uses the cpu_data_ptr[] array of per-cpu
 * pointers.
 */
typedef struct cpu_data
{
	struct cpu_data		*cpu_this;		/* pointer to myself */
	thread_t		cpu_active_thread;
	void			*cpu_int_state;		/* interrupt state */
	vm_offset_t		cpu_active_stack;	/* kernel stack base */
	vm_offset_t		cpu_kernel_stack;	/* kernel stack top */
	vm_offset_t		cpu_int_stack_top;
	int			cpu_preemption_level;
	int			cpu_simple_lock_count;
	int			cpu_interrupt_level;
	int			cpu_number;		/* Logical CPU */
	int			cpu_phys_number;	/* Physical CPU */
	cpu_id_t		cpu_id;			/* Platform Expert */
	int			cpu_signals;		/* IPI events */
	int			cpu_mcount_off;		/* mcount recursion */
	ast_t			cpu_pending_ast;
	int			cpu_type;
	int			cpu_subtype;
	int			cpu_threadtype;
	int			cpu_running;
	rtclock_timer_t		rtclock_timer;
	boolean_t		cpu_is64bit;
	task_map_t		cpu_task_map;
	addr64_t		cpu_task_cr3;
	addr64_t		cpu_active_cr3;
	addr64_t		cpu_kernel_cr3;
	cpu_uber_t		cpu_uber;
	void			*cpu_chud;
	void			*cpu_console_buf;
	struct x86_lcpu		lcpu;
	struct processor	*cpu_processor;
	struct cpu_pmap		*cpu_pmap;
	struct cpu_desc_table	*cpu_desc_tablep;
	struct fake_descriptor	*cpu_ldtp;
	cpu_desc_index_t	cpu_desc_index;
	int			cpu_ldt;
#ifdef MACH_KDB
	/* XXX Untested: */
	int			cpu_db_pass_thru;
	vm_offset_t		cpu_db_stacks;
	void			*cpu_kdb_saved_state;
	spl_t			cpu_kdb_saved_ipl;
	int			cpu_kdb_is_slave;
	int			cpu_kdb_active;
#endif /* MACH_KDB */
	boolean_t		cpu_iflag;
	boolean_t		cpu_boot_complete;
	int			cpu_hibernate;

	vm_offset_t		cpu_copywindow_base;
	uint64_t		*cpu_copywindow_pdp;

	vm_offset_t		cpu_physwindow_base;
	uint64_t		*cpu_physwindow_ptep;
	void 			*cpu_hi_iss;
	boolean_t		cpu_tlb_invalid;
	uint32_t		cpu_hwIntCnt[256];	/* Interrupt counts */
	uint64_t		cpu_dr7; /* debug control register */
	uint64_t		cpu_int_event_time;	/* intr entry/exit time */
	vmx_cpu_t		cpu_vmx;		/* wonderful world of virtualization */
	struct mca_state	*cpu_mca_state;		/* State at MC fault */
	uint64_t		cpu_uber_arg_store;	/* Double mapped address
							 * of current thread's
							 * uu_arg array.
							 */
	uint64_t		cpu_uber_arg_store_valid; /* Double mapped
							   * address of pcb
							   * arg store
							   * validity flag.
							   */
	rtc_nanotime_t		*cpu_nanotime;		/* Nanotime info */
							  
} cpu_data_t;

extern cpu_data_t	*cpu_data_ptr[];  
extern cpu_data_t	cpu_data_master;  

/* Macro to generate inline bodies to retrieve per-cpu data fields. */
#ifndef offsetof
#define offsetof(TYPE,MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
#endif /* offsetof */
#define CPU_DATA_GET(member,type)					\
	type ret;							\
	__asm__ volatile ("movl %%gs:%P1,%0"				\
		: "=r" (ret)						\
		: "i" (offsetof(cpu_data_t,member)));			\
	return ret;

/*
 * Everyone within the osfmk part of the kernel can use the fast
 * inline versions of these routines.  Everyone outside, must call
 * the real thing,
 */
static inline thread_t
get_active_thread(void)
{
	CPU_DATA_GET(cpu_active_thread,thread_t)
}
#define current_thread_fast()		get_active_thread()
#define current_thread()		current_thread_fast()

static inline boolean_t
get_is64bit(void)
{
	CPU_DATA_GET(cpu_is64bit, boolean_t)
}
#define cpu_mode_is64bit()		get_is64bit()

static inline int
get_preemption_level(void)
{
	CPU_DATA_GET(cpu_preemption_level,int)
}
static inline int
get_simple_lock_count(void)
{
	CPU_DATA_GET(cpu_simple_lock_count,int)
}
static inline int
get_interrupt_level(void)
{
	CPU_DATA_GET(cpu_interrupt_level,int)
}
static inline int
get_cpu_number(void)
{
	CPU_DATA_GET(cpu_number,int)
}
static inline int
get_cpu_phys_number(void)
{
	CPU_DATA_GET(cpu_phys_number,int)
}

static inline void
disable_preemption(void)
{
	__asm__ volatile ("incl %%gs:%P0"
			:
			: "i" (offsetof(cpu_data_t, cpu_preemption_level)));
}

static inline void
enable_preemption(void)
{
	assert(get_preemption_level() > 0);

	__asm__ volatile ("decl %%gs:%P0		\n\t"
			  "jne 1f			\n\t"
			  "call _kernel_preempt_check	\n\t"
			  "1:"
			: /* no outputs */
			: "i" (offsetof(cpu_data_t, cpu_preemption_level))
			: "eax", "ecx", "edx", "cc", "memory");
}

static inline void
enable_preemption_no_check(void)
{
	assert(get_preemption_level() > 0);

	__asm__ volatile ("decl %%gs:%P0"
			: /* no outputs */
			: "i" (offsetof(cpu_data_t, cpu_preemption_level))
			: "cc", "memory");
}

static inline void
mp_disable_preemption(void)
{
	disable_preemption();
}

static inline void
mp_enable_preemption(void)
{
	enable_preemption();
}

static inline void
mp_enable_preemption_no_check(void)
{
	enable_preemption_no_check();
}

static inline cpu_data_t *
current_cpu_datap(void)
{
	CPU_DATA_GET(cpu_this, cpu_data_t *);
}

static inline cpu_data_t *
cpu_datap(int cpu)
{
	assert(cpu_data_ptr[cpu]);
	return cpu_data_ptr[cpu];
}

extern cpu_data_t *cpu_data_alloc(boolean_t is_boot_cpu);

#else	/* !defined(__GNUC__) */

#endif	/* defined(__GNUC__) */

#endif	/* I386_CPU_DATA */
Commit	Line	Data
1c79356b	1	/*
c910b4d9	2	* Copyright (c) 2000-2008 Apple Inc. All rights reserved.
1c79356b	3	*
2d21ac55	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
1c79356b	5	*
2d21ac55 A	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
8f6c56a5	14	*
2d21ac55 A	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5 A	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
8f6c56a5 A	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55 A	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
8f6c56a5	25	*
2d21ac55	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
1c79356b A	27	*/
	28	/*
	29	* @OSF_COPYRIGHT@
	30	*
	31	*/
	32
	33	#ifndef I386_CPU_DATA
	34	#define I386_CPU_DATA
	35
1c79356b A	36	#include <mach_assert.h>
	37
	38	#if defined(__GNUC__)
	39
	40	#include <kern/assert.h>
	41	#include <kern/kern_types.h>
91447636	42	#include <kern/processor.h>
0c530ab8	43	#include <kern/pms.h>
55e303ae	44	#include <pexpert/pexpert.h>
0c530ab8 A	45	#include <mach/i386/thread_status.h>
	46	#include <i386/rtclock.h>
	47	#include <i386/pmCPU.h>
2d21ac55 A	48	#include <i386/cpu_topology.h>
	49
	50	#include <i386/vmx/vmx_cpu.h>
91447636 A	51
	52	/*
	53	* Data structures referenced (anonymously) from per-cpu data:
	54	*/
91447636	55	struct cpu_cons_buffer;
0c530ab8	56	struct cpu_desc_table;
2d21ac55	57	struct mca_state;
91447636 A	58
	59
	60	/*
	61	* Data structures embedded in per-cpu data:
	62	*/
	63	typedef struct rtclock_timer {
c910b4d9 A	64	queue_head_t queue;
	65	uint64_t deadline;
	66	boolean_t is_set;
	67	boolean_t has_expired;
91447636 A	68	} rtclock_timer_t;
91447636 A	69
2d21ac55	70
91447636 A	71	typedef struct {
	72	struct i386_tss *cdi_ktss;
	73	#if MACH_KDB
	74	struct i386_tss *cdi_dbtss;
	75	#endif /* MACH_KDB */
	76	struct fake_descriptor *cdi_gdt;
	77	struct fake_descriptor *cdi_idt;
	78	struct fake_descriptor *cdi_ldt;
0c530ab8	79	vm_offset_t cdi_sstk;
91447636 A	80	} cpu_desc_index_t;
91447636 A	81
0c530ab8 A	82	typedef enum {
	83	TASK_MAP_32BIT, /* 32-bit, compatibility mode */
	84	TASK_MAP_64BIT, /* 64-bit, separate address space */
	85	TASK_MAP_64BIT_SHARED /* 64-bit, kernel-shared addr space */
	86	} task_map_t;
	87
	88	/*
	89	* This structure is used on entry into the (uber-)kernel on syscall from
	90	* a 64-bit user. It contains the address of the machine state save area
	91	* for the current thread and a temporary place to save the user's rsp
	92	* before loading this address into rsp.
	93	*/
	94	typedef struct {
	95	addr64_t cu_isf; /* thread->pcb->iss.isf */
	96	uint64_t cu_tmp; /* temporary scratch */
	97	addr64_t cu_user_gs_base;
	98	} cpu_uber_t;
91447636 A	99
	100	/*
	101	* Per-cpu data.
	102	*
	103	* Each processor has a per-cpu data area which is dereferenced through the
	104	* current_cpu_datap() macro. For speed, the %gs segment is based here, and
	105	* using this, inlines provides single-instruction access to frequently used
	106	* members - such as get_cpu_number()/cpu_number(), and get_active_thread()/
	107	* current_thread().
	108	*
	109	* Cpu data owned by another processor can be accessed using the
	110	* cpu_datap(cpu_number) macro which uses the cpu_data_ptr[] array of per-cpu
	111	* pointers.
	112	*/
	113	typedef struct cpu_data
	114	{
	115	struct cpu_data cpu_this; / pointer to myself */
	116	thread_t cpu_active_thread;
0c530ab8 A	117	void cpu_int_state; / interrupt state */
	118	vm_offset_t cpu_active_stack; /* kernel stack base */
	119	vm_offset_t cpu_kernel_stack; /* kernel stack top */
91447636 A	120	vm_offset_t cpu_int_stack_top;
	121	int cpu_preemption_level;
	122	int cpu_simple_lock_count;
	123	int cpu_interrupt_level;
	124	int cpu_number; /* Logical CPU */
	125	int cpu_phys_number; /* Physical CPU */
	126	cpu_id_t cpu_id; /* Platform Expert */
	127	int cpu_signals; /* IPI events */
	128	int cpu_mcount_off; /* mcount recursion */
	129	ast_t cpu_pending_ast;
	130	int cpu_type;
	131	int cpu_subtype;
	132	int cpu_threadtype;
	133	int cpu_running;
0c530ab8	134	rtclock_timer_t rtclock_timer;
2d21ac55 A	135	boolean_t cpu_is64bit;
	136	task_map_t cpu_task_map;
	137	addr64_t cpu_task_cr3;
	138	addr64_t cpu_active_cr3;
	139	addr64_t cpu_kernel_cr3;
0c530ab8 A	140	cpu_uber_t cpu_uber;
0c530ab8 A	141	void *cpu_chud;
6601e61a	142	void *cpu_console_buf;
2d21ac55	143	struct x86_lcpu lcpu;
91447636 A	144	struct processor *cpu_processor;
91447636 A	145	struct cpu_pmap *cpu_pmap;
0c530ab8 A	146	struct cpu_desc_table *cpu_desc_tablep;
0c530ab8 A	147	struct fake_descriptor *cpu_ldtp;
91447636	148	cpu_desc_index_t cpu_desc_index;
0c530ab8	149	int cpu_ldt;
91447636 A	150	#ifdef MACH_KDB
	151	/* XXX Untested: */
	152	int cpu_db_pass_thru;
2d21ac55 A	153	vm_offset_t cpu_db_stacks;
	154	void *cpu_kdb_saved_state;
	155	spl_t cpu_kdb_saved_ipl;
91447636 A	156	int cpu_kdb_is_slave;
	157	int cpu_kdb_active;
	158	#endif /* MACH_KDB */
0c530ab8 A	159	boolean_t cpu_iflag;
	160	boolean_t cpu_boot_complete;
	161	int cpu_hibernate;
2d21ac55 A	162
	163	vm_offset_t cpu_copywindow_base;
	164	uint64_t *cpu_copywindow_pdp;
	165
	166	vm_offset_t cpu_physwindow_base;
	167	uint64_t *cpu_physwindow_ptep;
	168	void *cpu_hi_iss;
	169	boolean_t cpu_tlb_invalid;
593a1d5f	170	uint32_t cpu_hwIntCnt[256]; /* Interrupt counts */
0c530ab8	171	uint64_t cpu_dr7; /* debug control register */
2d21ac55 A	172	uint64_t cpu_int_event_time; /* intr entry/exit time */
	173	vmx_cpu_t cpu_vmx; /* wonderful world of virtualization */
	174	struct mca_state cpu_mca_state; / State at MC fault */
	175	uint64_t cpu_uber_arg_store; /* Double mapped address
	176	* of current thread's
	177	* uu_arg array.
	178	*/
	179	uint64_t cpu_uber_arg_store_valid; /* Double mapped
	180	* address of pcb
	181	* arg store
	182	* validity flag.
	183	*/
593a1d5f	184	rtc_nanotime_t cpu_nanotime; / Nanotime info */
2d21ac55	185
55e303ae	186	} cpu_data_t;
1c79356b	187
91447636 A	188	extern cpu_data_t *cpu_data_ptr[];
91447636 A	189	extern cpu_data_t cpu_data_master;
9bccf70c	190
55e303ae	191	/* Macro to generate inline bodies to retrieve per-cpu data fields. */
2d21ac55	192	#ifndef offsetof
55e303ae	193	#define offsetof(TYPE,MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
2d21ac55	194	#endif /* offsetof */
91447636	195	#define CPU_DATA_GET(member,type) \
55e303ae A	196	type ret; \
	197	__asm__ volatile ("movl %%gs:%P1,%0" \
	198	: "=r" (ret) \
91447636	199	: "i" (offsetof(cpu_data_t,member))); \
55e303ae	200	return ret;
9bccf70c	201
1c79356b A	202	/*
	203	* Everyone within the osfmk part of the kernel can use the fast
	204	* inline versions of these routines. Everyone outside, must call
	205	* the real thing,
	206	*/
91447636 A	207	static inline thread_t
91447636 A	208	get_active_thread(void)
1c79356b	209	{
91447636	210	CPU_DATA_GET(cpu_active_thread,thread_t)
1c79356b	211	}
91447636 A	212	#define current_thread_fast() get_active_thread()
91447636 A	213	#define current_thread() current_thread_fast()
1c79356b	214
0c530ab8 A	215	static inline boolean_t
	216	get_is64bit(void)
	217	{
	218	CPU_DATA_GET(cpu_is64bit, boolean_t)
	219	}
	220	#define cpu_mode_is64bit() get_is64bit()
	221
91447636 A	222	static inline int
91447636 A	223	get_preemption_level(void)
1c79356b	224	{
91447636	225	CPU_DATA_GET(cpu_preemption_level,int)
55e303ae	226	}
91447636 A	227	static inline int
91447636 A	228	get_simple_lock_count(void)
55e303ae	229	{
91447636	230	CPU_DATA_GET(cpu_simple_lock_count,int)
55e303ae	231	}
91447636 A	232	static inline int
91447636 A	233	get_interrupt_level(void)
55e303ae	234	{
91447636	235	CPU_DATA_GET(cpu_interrupt_level,int)
55e303ae	236	}
91447636 A	237	static inline int
91447636 A	238	get_cpu_number(void)
55e303ae A	239	{
	240	CPU_DATA_GET(cpu_number,int)
	241	}
91447636 A	242	static inline int
91447636 A	243	get_cpu_phys_number(void)
55e303ae A	244	{
55e303ae A	245	CPU_DATA_GET(cpu_phys_number,int)
1c79356b	246	}
1c79356b	247
91447636 A	248	static inline void
91447636 A	249	disable_preemption(void)
1c79356b	250	{
91447636 A	251	__asm__ volatile ("incl %%gs:%P0"
	252	:
	253	: "i" (offsetof(cpu_data_t, cpu_preemption_level)));
	254	}
1c79356b	255
91447636 A	256	static inline void
	257	enable_preemption(void)
	258	{
55e303ae A	259	assert(get_preemption_level() > 0);
55e303ae A	260
91447636 A	261	__asm__ volatile ("decl %%gs:%P0 \n\t"
	262	"jne 1f \n\t"
	263	"call _kernel_preempt_check \n\t"
	264	"1:"
1c79356b	265	: /* no outputs */
91447636 A	266	: "i" (offsetof(cpu_data_t, cpu_preemption_level))
91447636 A	267	: "eax", "ecx", "edx", "cc", "memory");
1c79356b A	268	}
1c79356b A	269
91447636 A	270	static inline void
91447636 A	271	enable_preemption_no_check(void)
1c79356b	272	{
1c79356b	273	assert(get_preemption_level() > 0);
1c79356b	274
91447636	275	__asm__ volatile ("decl %%gs:%P0"
1c79356b	276	: /* no outputs */
91447636	277	: "i" (offsetof(cpu_data_t, cpu_preemption_level))
1c79356b	278	: "cc", "memory");
1c79356b A	279	}
1c79356b A	280
91447636 A	281	static inline void
91447636 A	282	mp_disable_preemption(void)
1c79356b	283	{
1c79356b	284	disable_preemption();
1c79356b A	285	}
1c79356b A	286
91447636 A	287	static inline void
91447636 A	288	mp_enable_preemption(void)
1c79356b	289	{
1c79356b	290	enable_preemption();
1c79356b A	291	}
1c79356b A	292
91447636 A	293	static inline void
91447636 A	294	mp_enable_preemption_no_check(void)
1c79356b	295	{
1c79356b	296	enable_preemption_no_check();
1c79356b A	297	}
1c79356b A	298
91447636 A	299	static inline cpu_data_t *
	300	current_cpu_datap(void)
	301	{
	302	CPU_DATA_GET(cpu_this, cpu_data_t *);
	303	}
	304
	305	static inline cpu_data_t *
	306	cpu_datap(int cpu)
	307	{
	308	assert(cpu_data_ptr[cpu]);
	309	return cpu_data_ptr[cpu];
	310	}
	311
	312	extern cpu_data_t *cpu_data_alloc(boolean_t is_boot_cpu);
1c79356b A	313
	314	#else /* !defined(__GNUC__) */
	315
	316	#endif /* defined(__GNUC__) */
	317
	318	#endif /* I386_CPU_DATA */