/*
- * Copyright (c) 2000-2008 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
} rtclock_timer_t;
-#if defined(__i386__)
-
-typedef struct {
- struct i386_tss *cdi_ktss;
-#if MACH_KDB
- struct i386_tss *cdi_dbtss;
-#endif /* MACH_KDB */
- struct __attribute__((packed)) {
- uint16_t size;
- struct fake_descriptor *ptr;
- } cdi_gdt, cdi_idt;
- struct fake_descriptor *cdi_ldt;
- vm_offset_t cdi_sstk;
-} cpu_desc_index_t;
-
-typedef enum {
- TASK_MAP_32BIT, /* 32-bit, compatibility mode */
- TASK_MAP_64BIT, /* 64-bit, separate address space */
- TASK_MAP_64BIT_SHARED /* 64-bit, kernel-shared addr space */
-} task_map_t;
-
-#elif defined(__x86_64__)
-
-
typedef struct {
- struct x86_64_tss *cdi_ktss;
-#if MACH_KDB
- struct x86_64_tss *cdi_dbtss;
-#endif /* MACH_KDB */
+ struct x86_64_tss *cdi_ktss;
struct __attribute__((packed)) {
uint16_t size;
void *ptr;
} cdi_gdt, cdi_idt;
struct fake_descriptor *cdi_ldt;
- vm_offset_t cdi_sstk;
+ vm_offset_t cdi_sstk;
} cpu_desc_index_t;
typedef enum {
TASK_MAP_64BIT, /* 64-bit user thread, shared space */
} task_map_t;
-#else
-#error Unsupported architecture
-#endif
/*
* This structure is used on entry into the (uber-)kernel on syscall from
typedef uint16_t pcid_t;
typedef uint8_t pcid_ref_t;
+
+#define CPU_RTIME_BINS (12)
+#define CPU_ITIME_BINS (CPU_RTIME_BINS)
+
/*
* Per-cpu data.
*
#define cpu_pd cpu_pal_data /* convenience alias */
struct cpu_data *cpu_this; /* pointer to myself */
thread_t cpu_active_thread;
- int cpu_preemption_level;
+ thread_t cpu_nthread;
+ volatile int cpu_preemption_level;
int cpu_number; /* Logical CPU */
void *cpu_int_state; /* interrupt state */
vm_offset_t cpu_active_stack; /* kernel stack base */
int cpu_interrupt_level;
int cpu_phys_number; /* Physical CPU */
cpu_id_t cpu_id; /* Platform Expert */
- int cpu_signals; /* IPI events */
- int cpu_prior_signals; /* Last set of events,
+ volatile int cpu_signals; /* IPI events */
+ volatile int cpu_prior_signals; /* Last set of events,
* debugging
*/
- int cpu_mcount_off; /* mcount recursion */
ast_t cpu_pending_ast;
- int cpu_type;
- int cpu_subtype;
- int cpu_threadtype;
- int cpu_running;
+ volatile int cpu_running;
+ boolean_t cpu_fixed_pmcs_enabled;
rtclock_timer_t rtclock_timer;
- boolean_t cpu_is64bit;
volatile addr64_t cpu_active_cr3 __attribute((aligned(64)));
union {
volatile uint32_t cpu_tlb_invalid;
struct fake_descriptor *cpu_ldtp;
cpu_desc_index_t cpu_desc_index;
int cpu_ldt;
-#ifdef MACH_KDB
- /* XXX Untested: */
- int cpu_db_pass_thru;
- vm_offset_t cpu_db_stacks;
- void *cpu_kdb_saved_state;
- spl_t cpu_kdb_saved_ipl;
- int cpu_kdb_is_slave;
- int cpu_kdb_active;
-#endif /* MACH_KDB */
- boolean_t cpu_iflag;
- boolean_t cpu_boot_complete;
- int cpu_hibernate;
#if NCOPY_WINDOWS > 0
vm_offset_t cpu_copywindow_base;
uint64_t *cpu_copywindow_pdp;
vm_offset_t cpu_physwindow_base;
uint64_t *cpu_physwindow_ptep;
#endif
- void *cpu_hi_iss;
#define HWINTCNT_SIZE 256
uint32_t cpu_hwIntCnt[HWINTCNT_SIZE]; /* Interrupt counts */
+ uint64_t cpu_hwIntpexits[HWINTCNT_SIZE];
+ uint64_t cpu_hwIntcexits[HWINTCNT_SIZE];
uint64_t cpu_dr7; /* debug control register */
uint64_t cpu_int_event_time; /* intr entry/exit time */
-#if CONFIG_VMX
- vmx_cpu_t cpu_vmx; /* wonderful world of virtualization */
-#endif
-#if CONFIG_MCA
- struct mca_state *cpu_mca_state; /* State at MC fault */
-#endif
- uint64_t cpu_uber_arg_store; /* Double mapped address
- * of current thread's
- * uu_arg array.
- */
- uint64_t cpu_uber_arg_store_valid; /* Double mapped
- * address of pcb
- * arg store
- * validity flag.
- */
pal_rtc_nanotime_t *cpu_nanotime; /* Nanotime info */
+#if CONFIG_COUNTERS
thread_t csw_old_thread;
thread_t csw_new_thread;
-#if defined(__x86_64__)
+#endif /* CONFIG COUNTERS */
+#if KPC
+ /* double-buffered performance counter data */
+ uint64_t *cpu_kpc_buf[2];
+ /* PMC shadow and reload value buffers */
+ uint64_t *cpu_kpc_shadow;
+ uint64_t *cpu_kpc_reload;
+#endif
uint32_t cpu_pmap_pcid_enabled;
pcid_t cpu_active_pcid;
pcid_t cpu_last_pcid;
uint64_t cpu_pmap_pcid_flushes;
uint64_t cpu_pmap_pcid_preserves;
#endif
-#endif /* x86_64 */
+ uint64_t cpu_aperf;
+ uint64_t cpu_mperf;
+ uint64_t cpu_c3res;
+ uint64_t cpu_c6res;
+ uint64_t cpu_c7res;
+ uint64_t cpu_itime_total;
+ uint64_t cpu_rtime_total;
+ uint64_t cpu_ixtime;
+ uint64_t cpu_idle_exits;
+ uint64_t cpu_rtimes[CPU_RTIME_BINS];
+ uint64_t cpu_itimes[CPU_ITIME_BINS];
+ uint64_t cpu_cur_insns;
+ uint64_t cpu_cur_ucc;
+ uint64_t cpu_cur_urc;
uint64_t cpu_max_observed_int_latency;
int cpu_max_observed_int_latency_vector;
- uint64_t debugger_entry_time;
volatile boolean_t cpu_NMI_acknowledged;
+ uint64_t debugger_entry_time;
+ uint64_t debugger_ipi_time;
/* A separate nested interrupt stack flag, to account
* for non-nested interrupts arriving while on the interrupt stack
* Currently only occurs when AICPM enables interrupts on the
uint32_t cpu_nested_istack_events;
x86_saved_state64_t *cpu_fatal_trap_state;
x86_saved_state64_t *cpu_post_fatal_trap_state;
+#if CONFIG_VMX
+ vmx_cpu_t cpu_vmx; /* wonderful world of virtualization */
+#endif
+#if CONFIG_MCA
+ struct mca_state *cpu_mca_state; /* State at MC fault */
+#endif
+ int cpu_type;
+ int cpu_subtype;
+ int cpu_threadtype;
+ boolean_t cpu_iflag;
+ boolean_t cpu_boot_complete;
+ int cpu_hibernate;
} cpu_data_t;
extern cpu_data_t *cpu_data_ptr[];
-extern cpu_data_t cpu_data_master;
/* Macro to generate inline bodies to retrieve per-cpu data fields. */
+#if defined(__clang__)
+#define GS_RELATIVE volatile __attribute__((address_space(256)))
+#ifndef offsetof
+#define offsetof(TYPE,MEMBER) __builtin_offsetof(TYPE,MEMBER)
+#endif
+
+#define CPU_DATA_GET(member,type) \
+ cpu_data_t GS_RELATIVE *cpu_data = \
+ (cpu_data_t GS_RELATIVE *)0UL; \
+ type ret; \
+ ret = cpu_data->member; \
+ return ret;
+
+#define CPU_DATA_GET_INDEX(member,index,type) \
+ cpu_data_t GS_RELATIVE *cpu_data = \
+ (cpu_data_t GS_RELATIVE *)0UL; \
+ type ret; \
+ ret = cpu_data->member[index]; \
+ return ret;
+
+#define CPU_DATA_SET(member,value) \
+ cpu_data_t GS_RELATIVE *cpu_data = \
+ (cpu_data_t GS_RELATIVE *)0UL; \
+ cpu_data->member = value;
+
+#define CPU_DATA_XCHG(member,value,type) \
+ cpu_data_t GS_RELATIVE *cpu_data = \
+ (cpu_data_t GS_RELATIVE *)0UL; \
+ type ret; \
+ ret = cpu_data->member; \
+ cpu_data->member = value; \
+ return ret;
+
+#else /* !defined(__clang__) */
+
#ifndef offsetof
#define offsetof(TYPE,MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
#endif /* offsetof */
__asm__ volatile ("mov %0,%%gs:%P1" \
: \
: "r" (value), "i" (offsetof(cpu_data_t,member)));
+
#define CPU_DATA_XCHG(member,value,type) \
type ret; \
__asm__ volatile ("xchg %0,%%gs:%P1" \
: "i" (offsetof(cpu_data_t,member)), "0" (value)); \
return ret;
+#endif /* !defined(__clang__) */
+
/*
* Everyone within the osfmk part of the kernel can use the fast
* inline versions of these routines. Everyone outside, must call
#define current_thread_fast() get_active_thread()
#define current_thread() current_thread_fast()
-static inline boolean_t
-get_is64bit(void)
-{
- CPU_DATA_GET(cpu_is64bit, boolean_t)
-}
-#if CONFIG_YONAH
-#define cpu_mode_is64bit() get_is64bit()
-#else
#define cpu_mode_is64bit() TRUE
-#endif
static inline int
get_preemption_level(void)
static inline void
disable_preemption(void)
{
+#if defined(__clang__)
+ cpu_data_t GS_RELATIVE *cpu_data = (cpu_data_t GS_RELATIVE *)0UL;
+ cpu_data->cpu_preemption_level++;
+#else
__asm__ volatile ("incl %%gs:%P0"
:
: "i" (offsetof(cpu_data_t, cpu_preemption_level)));
+#endif
}
static inline void
{
assert(get_preemption_level() > 0);
+#if defined(__clang__)
+ cpu_data_t GS_RELATIVE *cpu_data = (cpu_data_t GS_RELATIVE *)0UL;
+ if (0 == --cpu_data->cpu_preemption_level)
+ kernel_preempt_check();
+#else
__asm__ volatile ("decl %%gs:%P0 \n\t"
"jne 1f \n\t"
"call _kernel_preempt_check \n\t"
: /* no outputs */
: "i" (offsetof(cpu_data_t, cpu_preemption_level))
: "eax", "ecx", "edx", "cc", "memory");
+#endif
}
static inline void
{
assert(get_preemption_level() > 0);
+#if defined(__clang__)
+ cpu_data_t GS_RELATIVE *cpu_data = (cpu_data_t GS_RELATIVE *)0UL;
+ cpu_data->cpu_preemption_level--;
+#else
__asm__ volatile ("decl %%gs:%P0"
: /* no outputs */
: "i" (offsetof(cpu_data_t, cpu_preemption_level))
: "cc", "memory");
+#endif
}
static inline void
}
extern cpu_data_t *cpu_data_alloc(boolean_t is_boot_cpu);
+extern void cpu_data_realloc(void);
#endif /* I386_CPU_DATA */
projects / apple / xnu.git / blobdiff