[apple/xnu.git] / osfmk / i386 / cpu_data.h

/*
 * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
/*
 * @OSF_COPYRIGHT@
 * 
 */

#ifndef	I386_CPU_DATA
#define I386_CPU_DATA

#include <mach_assert.h>

#include <kern/assert.h>
#include <kern/kern_types.h>
#include <kern/queue.h>
#include <kern/processor.h>
#include <kern/pms.h>
#include <pexpert/pexpert.h>
#include <mach/i386/thread_status.h>
#include <mach/i386/vm_param.h>
#include <i386/locks.h>
#include <i386/rtclock_protos.h>
#include <i386/pmCPU.h>
#include <i386/cpu_topology.h>

#if CONFIG_VMX
#include <i386/vmx/vmx_cpu.h>
#endif

#include <machine/pal_routines.h>

/*
 * Data structures referenced (anonymously) from per-cpu data:
 */
struct cpu_cons_buffer;
struct cpu_desc_table;
struct mca_state;
struct prngContext;

/*
 * Data structures embedded in per-cpu data:
 */
typedef struct rtclock_timer {
	mpqueue_head_t		queue;
	uint64_t		deadline;
	uint64_t		when_set;
	boolean_t		has_expired;
} rtclock_timer_t;


typedef struct {
	struct x86_64_tss	*cdi_ktss;
	struct __attribute__((packed)) {
		uint16_t size;
		void *ptr;
	} cdi_gdt, cdi_idt;
	struct fake_descriptor	*cdi_ldt;
	vm_offset_t		cdi_sstk;
} cpu_desc_index_t;

typedef enum {
	TASK_MAP_32BIT,			/* 32-bit user, compatibility mode */ 
	TASK_MAP_64BIT,			/* 64-bit user thread, shared space */ 
} task_map_t;


/*
 * This structure is used on entry into the (uber-)kernel on syscall from
 * a 64-bit user. It contains the address of the machine state save area
 * for the current thread and a temporary place to save the user's rsp
 * before loading this address into rsp.
 */
typedef struct {
	addr64_t	cu_isf;		/* thread->pcb->iss.isf */
	uint64_t	cu_tmp;		/* temporary scratch */	
	addr64_t	cu_user_gs_base;
} cpu_uber_t;

typedef	uint16_t	pcid_t;
typedef	uint8_t		pcid_ref_t;

#define CPU_RTIME_BINS (12)
#define CPU_ITIME_BINS (CPU_RTIME_BINS)

/*
 * Per-cpu data.
 *
 * Each processor has a per-cpu data area which is dereferenced through the
 * current_cpu_datap() macro. For speed, the %gs segment is based here, and
 * using this, inlines provides single-instruction access to frequently used
 * members - such as get_cpu_number()/cpu_number(), and get_active_thread()/
 * current_thread(). 
 * 
 * Cpu data owned by another processor can be accessed using the
 * cpu_datap(cpu_number) macro which uses the cpu_data_ptr[] array of per-cpu
 * pointers.
 */
typedef struct cpu_data
{
	struct pal_cpu_data	cpu_pal_data;		/* PAL-specific data */
#define				cpu_pd cpu_pal_data	/* convenience alias */
	struct cpu_data		*cpu_this;		/* pointer to myself */
	thread_t		cpu_active_thread;
	thread_t		cpu_nthread;
	volatile int		cpu_preemption_level;
	int			cpu_number;		/* Logical CPU */
	void			*cpu_int_state;		/* interrupt state */
	vm_offset_t		cpu_active_stack;	/* kernel stack base */
	vm_offset_t		cpu_kernel_stack;	/* kernel stack top */
	vm_offset_t		cpu_int_stack_top;
	int			cpu_interrupt_level;
	int			cpu_phys_number;	/* Physical CPU */
	cpu_id_t		cpu_id;			/* Platform Expert */
	volatile int		cpu_signals;		/* IPI events */
	volatile int		cpu_prior_signals;	/* Last set of events,
							 * debugging
							 */
	ast_t			cpu_pending_ast;
	volatile int		cpu_running;
	boolean_t		cpu_fixed_pmcs_enabled;
	rtclock_timer_t		rtclock_timer;
	volatile addr64_t	cpu_active_cr3 __attribute((aligned(64)));
	union {
		volatile uint32_t cpu_tlb_invalid;
		struct {
			volatile uint16_t cpu_tlb_invalid_local;
			volatile uint16_t cpu_tlb_invalid_global;
		};
	};
	volatile task_map_t	cpu_task_map;
	volatile addr64_t	cpu_task_cr3;
	addr64_t		cpu_kernel_cr3;
	cpu_uber_t		cpu_uber;
	void			*cpu_chud;
	void			*cpu_console_buf;
	struct x86_lcpu		lcpu;
	struct processor	*cpu_processor;
#if NCOPY_WINDOWS > 0
	struct cpu_pmap		*cpu_pmap;
#endif
	struct cpu_desc_table	*cpu_desc_tablep;
	struct fake_descriptor	*cpu_ldtp;
	cpu_desc_index_t	cpu_desc_index;
	int			cpu_ldt;
#if NCOPY_WINDOWS > 0
	vm_offset_t		cpu_copywindow_base;
	uint64_t		*cpu_copywindow_pdp;

	vm_offset_t		cpu_physwindow_base;
	uint64_t		*cpu_physwindow_ptep;
#endif

#define HWINTCNT_SIZE 256
	uint32_t		cpu_hwIntCnt[HWINTCNT_SIZE];	/* Interrupt counts */
 	uint64_t		cpu_hwIntpexits[HWINTCNT_SIZE];
	uint64_t		cpu_hwIntcexits[HWINTCNT_SIZE];
	uint64_t		cpu_dr7; /* debug control register */
	uint64_t		cpu_int_event_time;	/* intr entry/exit time */
	pal_rtc_nanotime_t	*cpu_nanotime;		/* Nanotime info */
#if KPC
	/* double-buffered performance counter data */
	uint64_t                *cpu_kpc_buf[2];
	/* PMC shadow and reload value buffers */
	uint64_t                *cpu_kpc_shadow;
	uint64_t                *cpu_kpc_reload;
#endif
	uint32_t		cpu_pmap_pcid_enabled;
	pcid_t			cpu_active_pcid;
	pcid_t			cpu_last_pcid;
	volatile pcid_ref_t	*cpu_pmap_pcid_coherentp;
	volatile pcid_ref_t	*cpu_pmap_pcid_coherentp_kernel;
#define	PMAP_PCID_MAX_PCID      (0x1000)
	pcid_t			cpu_pcid_free_hint;
	pcid_ref_t		cpu_pcid_refcounts[PMAP_PCID_MAX_PCID];
	pmap_t			cpu_pcid_last_pmap_dispatched[PMAP_PCID_MAX_PCID];
#ifdef	PCID_STATS
	uint64_t		cpu_pmap_pcid_flushes;
	uint64_t		cpu_pmap_pcid_preserves;
#endif
	uint64_t		cpu_aperf;
	uint64_t		cpu_mperf;
	uint64_t		cpu_c3res;
	uint64_t		cpu_c6res;
	uint64_t		cpu_c7res;
	uint64_t		cpu_itime_total;
	uint64_t		cpu_rtime_total;
	uint64_t		cpu_ixtime;
	uint64_t                cpu_idle_exits;
 	uint64_t		cpu_rtimes[CPU_RTIME_BINS];
 	uint64_t		cpu_itimes[CPU_ITIME_BINS];
 	uint64_t		cpu_cur_insns;
 	uint64_t		cpu_cur_ucc;
 	uint64_t		cpu_cur_urc;
	uint64_t		cpu_gpmcs[4];
	uint64_t                cpu_max_observed_int_latency;
	int                     cpu_max_observed_int_latency_vector;
	volatile boolean_t	cpu_NMI_acknowledged;
	uint64_t		debugger_entry_time;
	uint64_t		debugger_ipi_time;
	/* A separate nested interrupt stack flag, to account
	 * for non-nested interrupts arriving while on the interrupt stack
	 * Currently only occurs when AICPM enables interrupts on the
	 * interrupt stack during processor offlining.
	 */
	uint32_t		cpu_nested_istack;
	uint32_t		cpu_nested_istack_events;
	x86_saved_state64_t	*cpu_fatal_trap_state;
	x86_saved_state64_t	*cpu_post_fatal_trap_state;
#if CONFIG_VMX
	vmx_cpu_t		cpu_vmx;		/* wonderful world of virtualization */
#endif
#if CONFIG_MCA
	struct mca_state	*cpu_mca_state;		/* State at MC fault */
#endif
	struct prngContext	*cpu_prng;		/* PRNG's context */
 	int			cpu_type;
 	int			cpu_subtype;
 	int			cpu_threadtype;
 	boolean_t		cpu_iflag;
 	boolean_t		cpu_boot_complete;
 	int			cpu_hibernate;
} cpu_data_t;

extern cpu_data_t	*cpu_data_ptr[];  

/* Macro to generate inline bodies to retrieve per-cpu data fields. */
#if defined(__clang__)
#define GS_RELATIVE volatile __attribute__((address_space(256)))
#ifndef offsetof
#define offsetof(TYPE,MEMBER) __builtin_offsetof(TYPE,MEMBER)
#endif

#define CPU_DATA_GET(member,type)										\
	cpu_data_t GS_RELATIVE *cpu_data =							\
		(cpu_data_t GS_RELATIVE *)0UL;									\
	type ret;															\
	ret = cpu_data->member;												\
	return ret;

#define CPU_DATA_GET_INDEX(member,index,type)							\
	cpu_data_t GS_RELATIVE *cpu_data =							\
		(cpu_data_t GS_RELATIVE *)0UL;									\
	type ret;															\
	ret = cpu_data->member[index];										\
	return ret;

#define CPU_DATA_SET(member,value)										\
	cpu_data_t GS_RELATIVE *cpu_data =							\
		(cpu_data_t GS_RELATIVE *)0UL;									\
	cpu_data->member = value;

#define CPU_DATA_XCHG(member,value,type)								\
	cpu_data_t GS_RELATIVE *cpu_data =							\
		(cpu_data_t GS_RELATIVE *)0UL;									\
	type ret;															\
	ret = cpu_data->member;												\
	cpu_data->member = value;											\
	return ret;

#else /* !defined(__clang__) */

#ifndef offsetof
#define offsetof(TYPE,MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
#endif /* offsetof */
#define CPU_DATA_GET(member,type)					\
	type ret;							\
	__asm__ volatile ("mov %%gs:%P1,%0"				\
		: "=r" (ret)						\
		: "i" (offsetof(cpu_data_t,member)));			\
	return ret;

#define CPU_DATA_GET_INDEX(member,index,type)	\
	type ret;							\
	__asm__ volatile ("mov %%gs:(%1),%0"				\
		: "=r" (ret)						\
		: "r" (offsetof(cpu_data_t,member[index])));			\
	return ret;

#define CPU_DATA_SET(member,value)					\
	__asm__ volatile ("mov %0,%%gs:%P1"				\
		:							\
		: "r" (value), "i" (offsetof(cpu_data_t,member)));

#define CPU_DATA_XCHG(member,value,type)				\
	type ret;							\
	__asm__ volatile ("xchg %0,%%gs:%P1"				\
		: "=r" (ret)						\
		: "i" (offsetof(cpu_data_t,member)), "0" (value));	\
	return ret;

#endif /* !defined(__clang__) */

/*
 * Everyone within the osfmk part of the kernel can use the fast
 * inline versions of these routines.  Everyone outside, must call
 * the real thing,
 */
static inline thread_t
get_active_thread(void)
{
	CPU_DATA_GET(cpu_active_thread,thread_t)
}
#define current_thread_fast()		get_active_thread()
#define current_thread()		current_thread_fast()

#define cpu_mode_is64bit()		TRUE

static inline int
get_preemption_level(void)
{
	CPU_DATA_GET(cpu_preemption_level,int)
}
static inline int
get_interrupt_level(void)
{
	CPU_DATA_GET(cpu_interrupt_level,int)
}
static inline int
get_cpu_number(void)
{
	CPU_DATA_GET(cpu_number,int)
}
static inline int
get_cpu_phys_number(void)
{
	CPU_DATA_GET(cpu_phys_number,int)
}


static inline void
disable_preemption(void)
{
#if defined(__clang__)
	cpu_data_t GS_RELATIVE *cpu_data = (cpu_data_t GS_RELATIVE *)0UL;
	cpu_data->cpu_preemption_level++;
#else
	__asm__ volatile ("incl %%gs:%P0"
			:
			: "i" (offsetof(cpu_data_t, cpu_preemption_level)));
#endif
}

static inline void
enable_preemption(void)
{
	assert(get_preemption_level() > 0);

#if defined(__clang__)
	cpu_data_t GS_RELATIVE *cpu_data = (cpu_data_t GS_RELATIVE *)0UL;
	if (0 == --cpu_data->cpu_preemption_level)
		kernel_preempt_check();
#else
	__asm__ volatile ("decl %%gs:%P0		\n\t"
			  "jne 1f			\n\t"
			  "call _kernel_preempt_check	\n\t"
			  "1:"
			: /* no outputs */
			: "i" (offsetof(cpu_data_t, cpu_preemption_level))
			: "eax", "ecx", "edx", "cc", "memory");
#endif
}

static inline void
enable_preemption_no_check(void)
{
	assert(get_preemption_level() > 0);

#if defined(__clang__)
	cpu_data_t GS_RELATIVE *cpu_data = (cpu_data_t GS_RELATIVE *)0UL;
	cpu_data->cpu_preemption_level--;
#else
	__asm__ volatile ("decl %%gs:%P0"
			: /* no outputs */
			: "i" (offsetof(cpu_data_t, cpu_preemption_level))
			: "cc", "memory");
#endif
}

static inline void
mp_disable_preemption(void)
{
	disable_preemption();
}

static inline void
mp_enable_preemption(void)
{
	enable_preemption();
}

static inline void
mp_enable_preemption_no_check(void)
{
	enable_preemption_no_check();
}

static inline cpu_data_t *
current_cpu_datap(void)
{
	CPU_DATA_GET(cpu_this, cpu_data_t *);
}

static inline cpu_data_t *
cpu_datap(int cpu)
{
	return cpu_data_ptr[cpu];
}

extern cpu_data_t *cpu_data_alloc(boolean_t is_boot_cpu);
extern void cpu_data_realloc(void);

#endif	/* I386_CPU_DATA */
Commit	Line	Data
1c79356b	1	/*
39236c6e	2	* Copyright (c) 2000-2012 Apple Inc. All rights reserved.
1c79356b	3	*
2d21ac55	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
1c79356b	5	*
2d21ac55 A	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
8f6c56a5	14	*
2d21ac55 A	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5 A	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
8f6c56a5 A	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55 A	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
8f6c56a5	25	*
2d21ac55	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
1c79356b A	27	*/
	28	/*
	29	* @OSF_COPYRIGHT@
	30	*
	31	*/
	32
	33	#ifndef I386_CPU_DATA
	34	#define I386_CPU_DATA
	35
1c79356b A	36	#include <mach_assert.h>
1c79356b A	37
1c79356b A	38	#include <kern/assert.h>
1c79356b A	39	#include <kern/kern_types.h>
b0d623f7	40	#include <kern/queue.h>
91447636	41	#include <kern/processor.h>
0c530ab8	42	#include <kern/pms.h>
55e303ae	43	#include <pexpert/pexpert.h>
0c530ab8	44	#include <mach/i386/thread_status.h>
b0d623f7	45	#include <mach/i386/vm_param.h>
fe8ab488	46	#include <i386/locks.h>
6d2010ae	47	#include <i386/rtclock_protos.h>
0c530ab8	48	#include <i386/pmCPU.h>
2d21ac55 A	49	#include <i386/cpu_topology.h>
2d21ac55 A	50
b0d623f7	51	#if CONFIG_VMX
2d21ac55	52	#include <i386/vmx/vmx_cpu.h>
b0d623f7	53	#endif
91447636	54
6d2010ae A	55	#include <machine/pal_routines.h>
6d2010ae A	56
91447636 A	57	/*
	58	* Data structures referenced (anonymously) from per-cpu data:
	59	*/
91447636	60	struct cpu_cons_buffer;
0c530ab8	61	struct cpu_desc_table;
2d21ac55	62	struct mca_state;
fe8ab488	63	struct prngContext;
91447636	64
91447636 A	65	/*
	66	* Data structures embedded in per-cpu data:
	67	*/
	68	typedef struct rtclock_timer {
6d2010ae	69	mpqueue_head_t queue;
c910b4d9	70	uint64_t deadline;
6d2010ae	71	uint64_t when_set;
c910b4d9	72	boolean_t has_expired;
91447636 A	73	} rtclock_timer_t;
91447636 A	74
2d21ac55	75
91447636	76	typedef struct {
39236c6e	77	struct x86_64_tss *cdi_ktss;
b0d623f7 A	78	struct __attribute__((packed)) {
	79	uint16_t size;
	80	void *ptr;
	81	} cdi_gdt, cdi_idt;
	82	struct fake_descriptor *cdi_ldt;
39236c6e	83	vm_offset_t cdi_sstk;
b0d623f7 A	84	} cpu_desc_index_t;
	85
	86	typedef enum {
	87	TASK_MAP_32BIT, /* 32-bit user, compatibility mode */
	88	TASK_MAP_64BIT, /* 64-bit user thread, shared space */
	89	} task_map_t;
	90
b0d623f7	91
0c530ab8 A	92	/*
	93	* This structure is used on entry into the (uber-)kernel on syscall from
	94	* a 64-bit user. It contains the address of the machine state save area
	95	* for the current thread and a temporary place to save the user's rsp
	96	* before loading this address into rsp.
	97	*/
	98	typedef struct {
	99	addr64_t cu_isf; /* thread->pcb->iss.isf */
	100	uint64_t cu_tmp; /* temporary scratch */
6d2010ae	101	addr64_t cu_user_gs_base;
0c530ab8	102	} cpu_uber_t;
91447636	103
6d2010ae A	104	typedef uint16_t pcid_t;
6d2010ae A	105	typedef uint8_t pcid_ref_t;
bd504ef0 A	106
	107	#define CPU_RTIME_BINS (12)
	108	#define CPU_ITIME_BINS (CPU_RTIME_BINS)
	109
91447636 A	110	/*
	111	* Per-cpu data.
	112	*
	113	* Each processor has a per-cpu data area which is dereferenced through the
	114	* current_cpu_datap() macro. For speed, the %gs segment is based here, and
	115	* using this, inlines provides single-instruction access to frequently used
	116	* members - such as get_cpu_number()/cpu_number(), and get_active_thread()/
	117	* current_thread().
	118	*
	119	* Cpu data owned by another processor can be accessed using the
	120	* cpu_datap(cpu_number) macro which uses the cpu_data_ptr[] array of per-cpu
	121	* pointers.
	122	*/
	123	typedef struct cpu_data
	124	{
6d2010ae A	125	struct pal_cpu_data cpu_pal_data; /* PAL-specific data */
6d2010ae A	126	#define cpu_pd cpu_pal_data /* convenience alias */
91447636 A	127	struct cpu_data cpu_this; / pointer to myself */
91447636 A	128	thread_t cpu_active_thread;
39236c6e A	129	thread_t cpu_nthread;
39236c6e A	130	volatile int cpu_preemption_level;
6d2010ae	131	int cpu_number; /* Logical CPU */
0c530ab8 A	132	void cpu_int_state; / interrupt state */
	133	vm_offset_t cpu_active_stack; /* kernel stack base */
	134	vm_offset_t cpu_kernel_stack; /* kernel stack top */
91447636	135	vm_offset_t cpu_int_stack_top;
91447636	136	int cpu_interrupt_level;
91447636 A	137	int cpu_phys_number; /* Physical CPU */
91447636 A	138	cpu_id_t cpu_id; /* Platform Expert */
39236c6e A	139	volatile int cpu_signals; /* IPI events */
39236c6e A	140	volatile int cpu_prior_signals; /* Last set of events,
060df5ea A	141	* debugging
060df5ea A	142	*/
91447636	143	ast_t cpu_pending_ast;
bd504ef0 A	144	volatile int cpu_running;
bd504ef0 A	145	boolean_t cpu_fixed_pmcs_enabled;
0c530ab8	146	rtclock_timer_t rtclock_timer;
6d2010ae A	147	volatile addr64_t cpu_active_cr3 __attribute((aligned(64)));
	148	union {
	149	volatile uint32_t cpu_tlb_invalid;
	150	struct {
	151	volatile uint16_t cpu_tlb_invalid_local;
	152	volatile uint16_t cpu_tlb_invalid_global;
	153	};
	154	};
	155	volatile task_map_t cpu_task_map;
b0d623f7	156	volatile addr64_t cpu_task_cr3;
2d21ac55	157	addr64_t cpu_kernel_cr3;
0c530ab8 A	158	cpu_uber_t cpu_uber;
0c530ab8 A	159	void *cpu_chud;
6601e61a	160	void *cpu_console_buf;
2d21ac55	161	struct x86_lcpu lcpu;
91447636	162	struct processor *cpu_processor;
b0d623f7	163	#if NCOPY_WINDOWS > 0
91447636	164	struct cpu_pmap *cpu_pmap;
b0d623f7	165	#endif
0c530ab8 A	166	struct cpu_desc_table *cpu_desc_tablep;
0c530ab8 A	167	struct fake_descriptor *cpu_ldtp;
91447636	168	cpu_desc_index_t cpu_desc_index;
0c530ab8	169	int cpu_ldt;
b0d623f7	170	#if NCOPY_WINDOWS > 0
2d21ac55 A	171	vm_offset_t cpu_copywindow_base;
	172	uint64_t *cpu_copywindow_pdp;
	173
	174	vm_offset_t cpu_physwindow_base;
	175	uint64_t *cpu_physwindow_ptep;
b0d623f7 A	176	#endif
b0d623f7 A	177
6d2010ae A	178	#define HWINTCNT_SIZE 256
6d2010ae A	179	uint32_t cpu_hwIntCnt[HWINTCNT_SIZE]; /* Interrupt counts */
bd504ef0 A	180	uint64_t cpu_hwIntpexits[HWINTCNT_SIZE];
bd504ef0 A	181	uint64_t cpu_hwIntcexits[HWINTCNT_SIZE];
0c530ab8	182	uint64_t cpu_dr7; /* debug control register */
2d21ac55	183	uint64_t cpu_int_event_time; /* intr entry/exit time */
6d2010ae	184	pal_rtc_nanotime_t cpu_nanotime; / Nanotime info */
39236c6e A	185	#if KPC
	186	/* double-buffered performance counter data */
	187	uint64_t *cpu_kpc_buf[2];
	188	/* PMC shadow and reload value buffers */
	189	uint64_t *cpu_kpc_shadow;
	190	uint64_t *cpu_kpc_reload;
	191	#endif
6d2010ae A	192	uint32_t cpu_pmap_pcid_enabled;
	193	pcid_t cpu_active_pcid;
	194	pcid_t cpu_last_pcid;
	195	volatile pcid_ref_t *cpu_pmap_pcid_coherentp;
	196	volatile pcid_ref_t *cpu_pmap_pcid_coherentp_kernel;
	197	#define PMAP_PCID_MAX_PCID (0x1000)
	198	pcid_t cpu_pcid_free_hint;
	199	pcid_ref_t cpu_pcid_refcounts[PMAP_PCID_MAX_PCID];
	200	pmap_t cpu_pcid_last_pmap_dispatched[PMAP_PCID_MAX_PCID];
	201	#ifdef PCID_STATS
	202	uint64_t cpu_pmap_pcid_flushes;
	203	uint64_t cpu_pmap_pcid_preserves;
	204	#endif
4b17d6b6 A	205	uint64_t cpu_aperf;
	206	uint64_t cpu_mperf;
	207	uint64_t cpu_c3res;
	208	uint64_t cpu_c6res;
	209	uint64_t cpu_c7res;
	210	uint64_t cpu_itime_total;
	211	uint64_t cpu_rtime_total;
4b17d6b6	212	uint64_t cpu_ixtime;
bd504ef0 A	213	uint64_t cpu_idle_exits;
	214	uint64_t cpu_rtimes[CPU_RTIME_BINS];
	215	uint64_t cpu_itimes[CPU_ITIME_BINS];
	216	uint64_t cpu_cur_insns;
	217	uint64_t cpu_cur_ucc;
	218	uint64_t cpu_cur_urc;
a1c7dba1	219	uint64_t cpu_gpmcs[4];
6d2010ae A	220	uint64_t cpu_max_observed_int_latency;
6d2010ae A	221	int cpu_max_observed_int_latency_vector;
39236c6e	222	volatile boolean_t cpu_NMI_acknowledged;
060df5ea	223	uint64_t debugger_entry_time;
bd504ef0	224	uint64_t debugger_ipi_time;
060df5ea A	225	/* A separate nested interrupt stack flag, to account
	226	* for non-nested interrupts arriving while on the interrupt stack
	227	* Currently only occurs when AICPM enables interrupts on the
	228	* interrupt stack during processor offlining.
	229	*/
	230	uint32_t cpu_nested_istack;
	231	uint32_t cpu_nested_istack_events;
6d2010ae A	232	x86_saved_state64_t *cpu_fatal_trap_state;
6d2010ae A	233	x86_saved_state64_t *cpu_post_fatal_trap_state;
bd504ef0 A	234	#if CONFIG_VMX
	235	vmx_cpu_t cpu_vmx; /* wonderful world of virtualization */
	236	#endif
	237	#if CONFIG_MCA
	238	struct mca_state cpu_mca_state; / State at MC fault */
	239	#endif
fe8ab488	240	struct prngContext cpu_prng; / PRNG's context */
bd504ef0 A	241	int cpu_type;
	242	int cpu_subtype;
	243	int cpu_threadtype;
	244	boolean_t cpu_iflag;
	245	boolean_t cpu_boot_complete;
	246	int cpu_hibernate;
55e303ae	247	} cpu_data_t;
1c79356b	248
91447636	249	extern cpu_data_t *cpu_data_ptr[];
9bccf70c	250
55e303ae	251	/* Macro to generate inline bodies to retrieve per-cpu data fields. */
39236c6e A	252	#if defined(__clang__)
	253	#define GS_RELATIVE volatile __attribute__((address_space(256)))
	254	#ifndef offsetof
	255	#define offsetof(TYPE,MEMBER) __builtin_offsetof(TYPE,MEMBER)
	256	#endif
	257
	258	#define CPU_DATA_GET(member,type) \
	259	cpu_data_t GS_RELATIVE *cpu_data = \
	260	(cpu_data_t GS_RELATIVE *)0UL; \
	261	type ret; \
	262	ret = cpu_data->member; \
	263	return ret;
	264
	265	#define CPU_DATA_GET_INDEX(member,index,type) \
	266	cpu_data_t GS_RELATIVE *cpu_data = \
	267	(cpu_data_t GS_RELATIVE *)0UL; \
	268	type ret; \
	269	ret = cpu_data->member[index]; \
	270	return ret;
	271
	272	#define CPU_DATA_SET(member,value) \
	273	cpu_data_t GS_RELATIVE *cpu_data = \
	274	(cpu_data_t GS_RELATIVE *)0UL; \
	275	cpu_data->member = value;
	276
	277	#define CPU_DATA_XCHG(member,value,type) \
	278	cpu_data_t GS_RELATIVE *cpu_data = \
	279	(cpu_data_t GS_RELATIVE *)0UL; \
	280	type ret; \
	281	ret = cpu_data->member; \
	282	cpu_data->member = value; \
	283	return ret;
	284
	285	#else /* !defined(__clang__) */
	286
2d21ac55	287	#ifndef offsetof
55e303ae	288	#define offsetof(TYPE,MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
2d21ac55	289	#endif /* offsetof */
91447636	290	#define CPU_DATA_GET(member,type) \
55e303ae	291	type ret; \
b0d623f7	292	__asm__ volatile ("mov %%gs:%P1,%0" \
55e303ae	293	: "=r" (ret) \
91447636	294	: "i" (offsetof(cpu_data_t,member))); \
55e303ae	295	return ret;
9bccf70c	296
6d2010ae A	297	#define CPU_DATA_GET_INDEX(member,index,type) \
	298	type ret; \
	299	__asm__ volatile ("mov %%gs:(%1),%0" \
	300	: "=r" (ret) \
	301	: "r" (offsetof(cpu_data_t,member[index]))); \
	302	return ret;
	303
	304	#define CPU_DATA_SET(member,value) \
	305	__asm__ volatile ("mov %0,%%gs:%P1" \
	306	: \
	307	: "r" (value), "i" (offsetof(cpu_data_t,member)));
39236c6e	308
6d2010ae A	309	#define CPU_DATA_XCHG(member,value,type) \
	310	type ret; \
	311	__asm__ volatile ("xchg %0,%%gs:%P1" \
	312	: "=r" (ret) \
	313	: "i" (offsetof(cpu_data_t,member)), "0" (value)); \
	314	return ret;
	315
39236c6e A	316	#endif /* !defined(__clang__) */
39236c6e A	317
1c79356b A	318	/*
	319	* Everyone within the osfmk part of the kernel can use the fast
	320	* inline versions of these routines. Everyone outside, must call
	321	* the real thing,
	322	*/
91447636 A	323	static inline thread_t
91447636 A	324	get_active_thread(void)
1c79356b	325	{
91447636	326	CPU_DATA_GET(cpu_active_thread,thread_t)
1c79356b	327	}
91447636 A	328	#define current_thread_fast() get_active_thread()
91447636 A	329	#define current_thread() current_thread_fast()
1c79356b	330
b0d623f7	331	#define cpu_mode_is64bit() TRUE
0c530ab8	332
91447636 A	333	static inline int
91447636 A	334	get_preemption_level(void)
1c79356b	335	{
91447636	336	CPU_DATA_GET(cpu_preemption_level,int)
55e303ae	337	}
91447636	338	static inline int
91447636	339	get_interrupt_level(void)
55e303ae	340	{
91447636	341	CPU_DATA_GET(cpu_interrupt_level,int)
55e303ae	342	}
91447636 A	343	static inline int
91447636 A	344	get_cpu_number(void)
55e303ae A	345	{
	346	CPU_DATA_GET(cpu_number,int)
	347	}
91447636 A	348	static inline int
91447636 A	349	get_cpu_phys_number(void)
55e303ae A	350	{
55e303ae A	351	CPU_DATA_GET(cpu_phys_number,int)
1c79356b	352	}
1c79356b	353
b0d623f7	354
91447636 A	355	static inline void
91447636 A	356	disable_preemption(void)
1c79356b	357	{
39236c6e A	358	#if defined(__clang__)
	359	cpu_data_t GS_RELATIVE cpu_data = (cpu_data_t GS_RELATIVE )0UL;
	360	cpu_data->cpu_preemption_level++;
	361	#else
91447636 A	362	__asm__ volatile ("incl %%gs:%P0"
	363	:
	364	: "i" (offsetof(cpu_data_t, cpu_preemption_level)));
39236c6e	365	#endif
91447636	366	}
1c79356b	367
91447636 A	368	static inline void
	369	enable_preemption(void)
	370	{
55e303ae A	371	assert(get_preemption_level() > 0);
55e303ae A	372
39236c6e A	373	#if defined(__clang__)
	374	cpu_data_t GS_RELATIVE cpu_data = (cpu_data_t GS_RELATIVE )0UL;
	375	if (0 == --cpu_data->cpu_preemption_level)
	376	kernel_preempt_check();
	377	#else
91447636 A	378	__asm__ volatile ("decl %%gs:%P0 \n\t"
	379	"jne 1f \n\t"
	380	"call _kernel_preempt_check \n\t"
	381	"1:"
1c79356b	382	: /* no outputs */
91447636 A	383	: "i" (offsetof(cpu_data_t, cpu_preemption_level))
91447636 A	384	: "eax", "ecx", "edx", "cc", "memory");
39236c6e	385	#endif
1c79356b A	386	}
1c79356b A	387
91447636 A	388	static inline void
91447636 A	389	enable_preemption_no_check(void)
1c79356b	390	{
1c79356b	391	assert(get_preemption_level() > 0);
1c79356b	392
39236c6e A	393	#if defined(__clang__)
	394	cpu_data_t GS_RELATIVE cpu_data = (cpu_data_t GS_RELATIVE )0UL;
	395	cpu_data->cpu_preemption_level--;
	396	#else
91447636	397	__asm__ volatile ("decl %%gs:%P0"
1c79356b	398	: /* no outputs */
91447636	399	: "i" (offsetof(cpu_data_t, cpu_preemption_level))
1c79356b	400	: "cc", "memory");
39236c6e	401	#endif
1c79356b A	402	}
1c79356b A	403
91447636 A	404	static inline void
91447636 A	405	mp_disable_preemption(void)
1c79356b	406	{
1c79356b	407	disable_preemption();
1c79356b A	408	}
1c79356b A	409
91447636 A	410	static inline void
91447636 A	411	mp_enable_preemption(void)
1c79356b	412	{
1c79356b	413	enable_preemption();
1c79356b A	414	}
1c79356b A	415
91447636 A	416	static inline void
91447636 A	417	mp_enable_preemption_no_check(void)
1c79356b	418	{
1c79356b	419	enable_preemption_no_check();
1c79356b A	420	}
1c79356b A	421
91447636 A	422	static inline cpu_data_t *
	423	current_cpu_datap(void)
	424	{
	425	CPU_DATA_GET(cpu_this, cpu_data_t *);
	426	}
	427
	428	static inline cpu_data_t *
	429	cpu_datap(int cpu)
	430	{
91447636 A	431	return cpu_data_ptr[cpu];
	432	}
	433
	434	extern cpu_data_t *cpu_data_alloc(boolean_t is_boot_cpu);
316670eb	435	extern void cpu_data_realloc(void);
1c79356b	436
1c79356b	437	#endif /* I386_CPU_DATA */