X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/060df5ea7c632b1ac8cc8aac1fb59758165c2084..99c3a10404e5d1ef94397ab4df5a8b74711fc4d3:/osfmk/i386/cpu_data.h diff --git a/osfmk/i386/cpu_data.h b/osfmk/i386/cpu_data.h index 63eb4446b..3473fbd3a 100644 --- a/osfmk/i386/cpu_data.h +++ b/osfmk/i386/cpu_data.h @@ -43,7 +43,7 @@ #include #include #include -#include +#include #include #include @@ -51,6 +51,8 @@ #include #endif +#include + /* * Data structures referenced (anonymously) from per-cpu data: */ @@ -58,14 +60,13 @@ struct cpu_cons_buffer; struct cpu_desc_table; struct mca_state; - /* * Data structures embedded in per-cpu data: */ typedef struct rtclock_timer { - queue_head_t queue; + mpqueue_head_t queue; uint64_t deadline; - boolean_t is_set; + uint64_t when_set; boolean_t has_expired; } rtclock_timer_t; @@ -74,9 +75,6 @@ typedef struct rtclock_timer { typedef struct { struct i386_tss *cdi_ktss; -#if MACH_KDB - struct i386_tss *cdi_dbtss; -#endif /* MACH_KDB */ struct __attribute__((packed)) { uint16_t size; struct fake_descriptor *ptr; @@ -96,9 +94,6 @@ typedef enum { typedef struct { struct x86_64_tss *cdi_ktss; -#if MACH_KDB - struct x86_64_tss *cdi_dbtss; -#endif /* MACH_KDB */ struct __attribute__((packed)) { uint16_t size; void *ptr; @@ -125,9 +120,11 @@ typedef enum { typedef struct { addr64_t cu_isf; /* thread->pcb->iss.isf */ uint64_t cu_tmp; /* temporary scratch */ - addr64_t cu_user_gs_base; + addr64_t cu_user_gs_base; } cpu_uber_t; +typedef uint16_t pcid_t; +typedef uint8_t pcid_ref_t; /* * Per-cpu data. * @@ -143,16 +140,17 @@ typedef struct { */ typedef struct cpu_data { + struct pal_cpu_data cpu_pal_data; /* PAL-specific data */ +#define cpu_pd cpu_pal_data /* convenience alias */ struct cpu_data *cpu_this; /* pointer to myself */ thread_t cpu_active_thread; + int cpu_preemption_level; + int cpu_number; /* Logical CPU */ void *cpu_int_state; /* interrupt state */ vm_offset_t cpu_active_stack; /* kernel stack base */ vm_offset_t cpu_kernel_stack; /* kernel stack top */ vm_offset_t cpu_int_stack_top; - int cpu_preemption_level; - int cpu_simple_lock_count; int cpu_interrupt_level; - int cpu_number; /* Logical CPU */ int cpu_phys_number; /* Physical CPU */ cpu_id_t cpu_id; /* Platform Expert */ int cpu_signals; /* IPI events */ @@ -167,9 +165,16 @@ typedef struct cpu_data int cpu_running; rtclock_timer_t rtclock_timer; boolean_t cpu_is64bit; - task_map_t cpu_task_map; + volatile addr64_t cpu_active_cr3 __attribute((aligned(64))); + union { + volatile uint32_t cpu_tlb_invalid; + struct { + volatile uint16_t cpu_tlb_invalid_local; + volatile uint16_t cpu_tlb_invalid_global; + }; + }; + volatile task_map_t cpu_task_map; volatile addr64_t cpu_task_cr3; - volatile addr64_t cpu_active_cr3; addr64_t cpu_kernel_cr3; cpu_uber_t cpu_uber; void *cpu_chud; @@ -183,32 +188,20 @@ typedef struct cpu_data struct fake_descriptor *cpu_ldtp; cpu_desc_index_t cpu_desc_index; int cpu_ldt; -#ifdef MACH_KDB - /* XXX Untested: */ - int cpu_db_pass_thru; - vm_offset_t cpu_db_stacks; - void *cpu_kdb_saved_state; - spl_t cpu_kdb_saved_ipl; - int cpu_kdb_is_slave; - int cpu_kdb_active; -#endif /* MACH_KDB */ boolean_t cpu_iflag; boolean_t cpu_boot_complete; int cpu_hibernate; - #if NCOPY_WINDOWS > 0 vm_offset_t cpu_copywindow_base; uint64_t *cpu_copywindow_pdp; vm_offset_t cpu_physwindow_base; uint64_t *cpu_physwindow_ptep; - void *cpu_hi_iss; #endif + void *cpu_hi_iss; - - - volatile boolean_t cpu_tlb_invalid; - uint32_t cpu_hwIntCnt[256]; /* Interrupt counts */ +#define HWINTCNT_SIZE 256 + uint32_t cpu_hwIntCnt[HWINTCNT_SIZE]; /* Interrupt counts */ uint64_t cpu_dr7; /* debug control register */ uint64_t cpu_int_event_time; /* intr entry/exit time */ #if CONFIG_VMX @@ -226,11 +219,28 @@ typedef struct cpu_data * arg store * validity flag. */ - rtc_nanotime_t *cpu_nanotime; /* Nanotime info */ + pal_rtc_nanotime_t *cpu_nanotime; /* Nanotime info */ +#if CONFIG_COUNTERS thread_t csw_old_thread; thread_t csw_new_thread; - uint64_t cpu_max_observed_int_latency; - int cpu_max_observed_int_latency_vector; +#endif /* CONFIG COUNTERS */ +#if defined(__x86_64__) + uint32_t cpu_pmap_pcid_enabled; + pcid_t cpu_active_pcid; + pcid_t cpu_last_pcid; + volatile pcid_ref_t *cpu_pmap_pcid_coherentp; + volatile pcid_ref_t *cpu_pmap_pcid_coherentp_kernel; +#define PMAP_PCID_MAX_PCID (0x1000) + pcid_t cpu_pcid_free_hint; + pcid_ref_t cpu_pcid_refcounts[PMAP_PCID_MAX_PCID]; + pmap_t cpu_pcid_last_pmap_dispatched[PMAP_PCID_MAX_PCID]; +#ifdef PCID_STATS + uint64_t cpu_pmap_pcid_flushes; + uint64_t cpu_pmap_pcid_preserves; +#endif +#endif /* x86_64 */ + uint64_t cpu_max_observed_int_latency; + int cpu_max_observed_int_latency_vector; uint64_t debugger_entry_time; volatile boolean_t cpu_NMI_acknowledged; /* A separate nested interrupt stack flag, to account @@ -240,6 +250,8 @@ typedef struct cpu_data */ uint32_t cpu_nested_istack; uint32_t cpu_nested_istack_events; + x86_saved_state64_t *cpu_fatal_trap_state; + x86_saved_state64_t *cpu_post_fatal_trap_state; } cpu_data_t; extern cpu_data_t *cpu_data_ptr[]; @@ -256,6 +268,24 @@ extern cpu_data_t cpu_data_master; : "i" (offsetof(cpu_data_t,member))); \ return ret; +#define CPU_DATA_GET_INDEX(member,index,type) \ + type ret; \ + __asm__ volatile ("mov %%gs:(%1),%0" \ + : "=r" (ret) \ + : "r" (offsetof(cpu_data_t,member[index]))); \ + return ret; + +#define CPU_DATA_SET(member,value) \ + __asm__ volatile ("mov %0,%%gs:%P1" \ + : \ + : "r" (value), "i" (offsetof(cpu_data_t,member))); +#define CPU_DATA_XCHG(member,value,type) \ + type ret; \ + __asm__ volatile ("xchg %0,%%gs:%P1" \ + : "=r" (ret) \ + : "i" (offsetof(cpu_data_t,member)), "0" (value)); \ + return ret; + /* * Everyone within the osfmk part of the kernel can use the fast * inline versions of these routines. Everyone outside, must call @@ -269,14 +299,14 @@ get_active_thread(void) #define current_thread_fast() get_active_thread() #define current_thread() current_thread_fast() -#if defined(__i386__) static inline boolean_t get_is64bit(void) { CPU_DATA_GET(cpu_is64bit, boolean_t) } +#if CONFIG_YONAH #define cpu_mode_is64bit() get_is64bit() -#elif defined(__x86_64__) +#else #define cpu_mode_is64bit() TRUE #endif @@ -286,11 +316,6 @@ get_preemption_level(void) CPU_DATA_GET(cpu_preemption_level,int) } static inline int -get_simple_lock_count(void) -{ - CPU_DATA_GET(cpu_simple_lock_count,int) -} -static inline int get_interrupt_level(void) { CPU_DATA_GET(cpu_interrupt_level,int) @@ -371,5 +396,6 @@ cpu_datap(int cpu) } extern cpu_data_t *cpu_data_alloc(boolean_t is_boot_cpu); +extern void cpu_data_realloc(void); #endif /* I386_CPU_DATA */