X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/c0fea4742e91338fffdcf79f86a7c1d5e2b97eb1..22ba694c5857e62b5a553b1505dcf2e509177f28:/osfmk/i386/cpu_data.h diff --git a/osfmk/i386/cpu_data.h b/osfmk/i386/cpu_data.h index 8ad34142a..e0bb1a7e4 100644 --- a/osfmk/i386/cpu_data.h +++ b/osfmk/i386/cpu_data.h @@ -1,23 +1,29 @@ /* - * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2012 Apple Inc. All rights reserved. * - * @APPLE_LICENSE_HEADER_START@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * - * The contents of this file constitute Original Code as defined in and - * are subject to the Apple Public Source License Version 1.1 (the - * "License"). You may not use this file except in compliance with the - * License. Please obtain a copy of the License at - * http://www.apple.com/publicsource and read it before using this file. + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. * - * This Original Code and all software distributed under the License are - * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the - * License for the specific language governing rights and limitations - * under the License. + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. * - * @APPLE_LICENSE_HEADER_END@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * @OSF_COPYRIGHT@ @@ -29,58 +35,58 @@ #include -#if defined(__GNUC__) - #include #include +#include #include #include #include #include -#include +#include +#include #include +#include + +#if CONFIG_VMX +#include +#endif + +#include /* * Data structures referenced (anonymously) from per-cpu data: */ -struct cpu_core; struct cpu_cons_buffer; struct cpu_desc_table; - +struct mca_state; /* * Data structures embedded in per-cpu data: */ typedef struct rtclock_timer { - uint64_t deadline; - boolean_t is_set; - boolean_t has_expired; + mpqueue_head_t queue; + uint64_t deadline; + uint64_t when_set; + boolean_t has_expired; } rtclock_timer_t; -typedef struct { - uint64_t tsc_base; /* timestamp */ - uint64_t ns_base; /* nanoseconds */ - uint32_t scale; /* tsc -> nanosec multiplier */ - uint32_t shift; /* tsc -> nanosec shift/div */ -} rtc_nanotime_t; typedef struct { - struct i386_tss *cdi_ktss; -#if MACH_KDB - struct i386_tss *cdi_dbtss; -#endif /* MACH_KDB */ - struct fake_descriptor *cdi_gdt; - struct fake_descriptor *cdi_idt; - struct fake_descriptor *cdi_ldt; + struct x86_64_tss *cdi_ktss; + struct __attribute__((packed)) { + uint16_t size; + void *ptr; + } cdi_gdt, cdi_idt; + struct fake_descriptor *cdi_ldt; vm_offset_t cdi_sstk; } cpu_desc_index_t; typedef enum { - TASK_MAP_32BIT, /* 32-bit, compatibility mode */ - TASK_MAP_64BIT, /* 64-bit, separate address space */ - TASK_MAP_64BIT_SHARED /* 64-bit, kernel-shared addr space */ + TASK_MAP_32BIT, /* 32-bit user, compatibility mode */ + TASK_MAP_64BIT, /* 64-bit user thread, shared space */ } task_map_t; + /* * This structure is used on entry into the (uber-)kernel on syscall from * a 64-bit user. It contains the address of the machine state save area @@ -90,9 +96,15 @@ typedef enum { typedef struct { addr64_t cu_isf; /* thread->pcb->iss.isf */ uint64_t cu_tmp; /* temporary scratch */ - addr64_t cu_user_gs_base; + addr64_t cu_user_gs_base; } cpu_uber_t; +typedef uint16_t pcid_t; +typedef uint8_t pcid_ref_t; + +#define CPU_RTIME_BINS (12) +#define CPU_ITIME_BINS (CPU_RTIME_BINS) + /* * Per-cpu data. * @@ -108,86 +120,201 @@ typedef struct { */ typedef struct cpu_data { + struct pal_cpu_data cpu_pal_data; /* PAL-specific data */ +#define cpu_pd cpu_pal_data /* convenience alias */ struct cpu_data *cpu_this; /* pointer to myself */ thread_t cpu_active_thread; + thread_t cpu_nthread; + volatile int cpu_preemption_level; + int cpu_number; /* Logical CPU */ void *cpu_int_state; /* interrupt state */ vm_offset_t cpu_active_stack; /* kernel stack base */ vm_offset_t cpu_kernel_stack; /* kernel stack top */ vm_offset_t cpu_int_stack_top; - int cpu_preemption_level; - int cpu_simple_lock_count; int cpu_interrupt_level; - int cpu_number; /* Logical CPU */ int cpu_phys_number; /* Physical CPU */ cpu_id_t cpu_id; /* Platform Expert */ - int cpu_signals; /* IPI events */ - int cpu_mcount_off; /* mcount recursion */ + volatile int cpu_signals; /* IPI events */ + volatile int cpu_prior_signals; /* Last set of events, + * debugging + */ ast_t cpu_pending_ast; - int cpu_type; - int cpu_subtype; - int cpu_threadtype; - int cpu_running; - uint64_t rtclock_intr_deadline; + volatile int cpu_running; + boolean_t cpu_fixed_pmcs_enabled; rtclock_timer_t rtclock_timer; - boolean_t cpu_is64bit; - task_map_t cpu_task_map; - addr64_t cpu_task_cr3; - addr64_t cpu_active_cr3; - addr64_t cpu_kernel_cr3; + volatile addr64_t cpu_active_cr3 __attribute((aligned(64))); + union { + volatile uint32_t cpu_tlb_invalid; + struct { + volatile uint16_t cpu_tlb_invalid_local; + volatile uint16_t cpu_tlb_invalid_global; + }; + }; + volatile task_map_t cpu_task_map; + volatile addr64_t cpu_task_cr3; + addr64_t cpu_kernel_cr3; cpu_uber_t cpu_uber; void *cpu_chud; void *cpu_console_buf; - struct cpu_core *cpu_core; /* cpu's parent core */ + struct x86_lcpu lcpu; struct processor *cpu_processor; +#if NCOPY_WINDOWS > 0 struct cpu_pmap *cpu_pmap; +#endif struct cpu_desc_table *cpu_desc_tablep; struct fake_descriptor *cpu_ldtp; cpu_desc_index_t cpu_desc_index; int cpu_ldt; -#ifdef MACH_KDB - /* XXX Untested: */ - int cpu_db_pass_thru; - vm_offset_t cpu_db_stacks; - void *cpu_kdb_saved_state; - spl_t cpu_kdb_saved_ipl; - int cpu_kdb_is_slave; - int cpu_kdb_active; -#endif /* MACH_KDB */ - boolean_t cpu_iflag; - boolean_t cpu_boot_complete; - int cpu_hibernate; - pmsd pms; /* Power Management Stepper control */ - uint64_t rtcPop; /* when the etimer wants a timer pop */ - - vm_offset_t cpu_copywindow_base; - uint64_t *cpu_copywindow_pdp; - - vm_offset_t cpu_physwindow_base; - uint64_t *cpu_physwindow_ptep; - void *cpu_hi_iss; - boolean_t cpu_tlb_invalid; - - uint64_t *cpu_pmHpet; /* Address of the HPET for this processor */ - uint32_t cpu_pmHpetVec; /* Interrupt vector for HPET for this processor */ -/* Statistics */ - pmStats_t cpu_pmStats; /* Power management data */ - uint32_t cpu_hwIntCnt[256]; /* Interrupt counts */ - +#if NCOPY_WINDOWS > 0 + vm_offset_t cpu_copywindow_base; + uint64_t *cpu_copywindow_pdp; + + vm_offset_t cpu_physwindow_base; + uint64_t *cpu_physwindow_ptep; +#endif + +#define HWINTCNT_SIZE 256 + uint32_t cpu_hwIntCnt[HWINTCNT_SIZE]; /* Interrupt counts */ + uint64_t cpu_hwIntpexits[HWINTCNT_SIZE]; + uint64_t cpu_hwIntcexits[HWINTCNT_SIZE]; uint64_t cpu_dr7; /* debug control register */ + uint64_t cpu_int_event_time; /* intr entry/exit time */ + pal_rtc_nanotime_t *cpu_nanotime; /* Nanotime info */ +#if CONFIG_COUNTERS + thread_t csw_old_thread; + thread_t csw_new_thread; +#endif /* CONFIG COUNTERS */ +#if KPC + /* double-buffered performance counter data */ + uint64_t *cpu_kpc_buf[2]; + /* PMC shadow and reload value buffers */ + uint64_t *cpu_kpc_shadow; + uint64_t *cpu_kpc_reload; +#endif + uint32_t cpu_pmap_pcid_enabled; + pcid_t cpu_active_pcid; + pcid_t cpu_last_pcid; + volatile pcid_ref_t *cpu_pmap_pcid_coherentp; + volatile pcid_ref_t *cpu_pmap_pcid_coherentp_kernel; +#define PMAP_PCID_MAX_PCID (0x1000) + pcid_t cpu_pcid_free_hint; + pcid_ref_t cpu_pcid_refcounts[PMAP_PCID_MAX_PCID]; + pmap_t cpu_pcid_last_pmap_dispatched[PMAP_PCID_MAX_PCID]; +#ifdef PCID_STATS + uint64_t cpu_pmap_pcid_flushes; + uint64_t cpu_pmap_pcid_preserves; +#endif + uint64_t cpu_aperf; + uint64_t cpu_mperf; + uint64_t cpu_c3res; + uint64_t cpu_c6res; + uint64_t cpu_c7res; + uint64_t cpu_itime_total; + uint64_t cpu_rtime_total; + uint64_t cpu_ixtime; + uint64_t cpu_idle_exits; + uint64_t cpu_rtimes[CPU_RTIME_BINS]; + uint64_t cpu_itimes[CPU_ITIME_BINS]; + uint64_t cpu_cur_insns; + uint64_t cpu_cur_ucc; + uint64_t cpu_cur_urc; + uint64_t cpu_max_observed_int_latency; + int cpu_max_observed_int_latency_vector; + volatile boolean_t cpu_NMI_acknowledged; + uint64_t debugger_entry_time; + uint64_t debugger_ipi_time; + /* A separate nested interrupt stack flag, to account + * for non-nested interrupts arriving while on the interrupt stack + * Currently only occurs when AICPM enables interrupts on the + * interrupt stack during processor offlining. + */ + uint32_t cpu_nested_istack; + uint32_t cpu_nested_istack_events; + x86_saved_state64_t *cpu_fatal_trap_state; + x86_saved_state64_t *cpu_post_fatal_trap_state; +#if CONFIG_VMX + vmx_cpu_t cpu_vmx; /* wonderful world of virtualization */ +#endif +#if CONFIG_MCA + struct mca_state *cpu_mca_state; /* State at MC fault */ +#endif + int cpu_type; + int cpu_subtype; + int cpu_threadtype; + boolean_t cpu_iflag; + boolean_t cpu_boot_complete; + int cpu_hibernate; } cpu_data_t; extern cpu_data_t *cpu_data_ptr[]; -extern cpu_data_t cpu_data_master; /* Macro to generate inline bodies to retrieve per-cpu data fields. */ +#if defined(__clang__) +#define GS_RELATIVE volatile __attribute__((address_space(256))) +#ifndef offsetof +#define offsetof(TYPE,MEMBER) __builtin_offsetof(TYPE,MEMBER) +#endif + +#define CPU_DATA_GET(member,type) \ + cpu_data_t GS_RELATIVE *cpu_data = \ + (cpu_data_t GS_RELATIVE *)0UL; \ + type ret; \ + ret = cpu_data->member; \ + return ret; + +#define CPU_DATA_GET_INDEX(member,index,type) \ + cpu_data_t GS_RELATIVE *cpu_data = \ + (cpu_data_t GS_RELATIVE *)0UL; \ + type ret; \ + ret = cpu_data->member[index]; \ + return ret; + +#define CPU_DATA_SET(member,value) \ + cpu_data_t GS_RELATIVE *cpu_data = \ + (cpu_data_t GS_RELATIVE *)0UL; \ + cpu_data->member = value; + +#define CPU_DATA_XCHG(member,value,type) \ + cpu_data_t GS_RELATIVE *cpu_data = \ + (cpu_data_t GS_RELATIVE *)0UL; \ + type ret; \ + ret = cpu_data->member; \ + cpu_data->member = value; \ + return ret; + +#else /* !defined(__clang__) */ + +#ifndef offsetof #define offsetof(TYPE,MEMBER) ((size_t) &((TYPE *)0)->MEMBER) +#endif /* offsetof */ #define CPU_DATA_GET(member,type) \ type ret; \ - __asm__ volatile ("movl %%gs:%P1,%0" \ + __asm__ volatile ("mov %%gs:%P1,%0" \ : "=r" (ret) \ : "i" (offsetof(cpu_data_t,member))); \ return ret; +#define CPU_DATA_GET_INDEX(member,index,type) \ + type ret; \ + __asm__ volatile ("mov %%gs:(%1),%0" \ + : "=r" (ret) \ + : "r" (offsetof(cpu_data_t,member[index]))); \ + return ret; + +#define CPU_DATA_SET(member,value) \ + __asm__ volatile ("mov %0,%%gs:%P1" \ + : \ + : "r" (value), "i" (offsetof(cpu_data_t,member))); + +#define CPU_DATA_XCHG(member,value,type) \ + type ret; \ + __asm__ volatile ("xchg %0,%%gs:%P1" \ + : "=r" (ret) \ + : "i" (offsetof(cpu_data_t,member)), "0" (value)); \ + return ret; + +#endif /* !defined(__clang__) */ + /* * Everyone within the osfmk part of the kernel can use the fast * inline versions of these routines. Everyone outside, must call @@ -201,12 +328,7 @@ get_active_thread(void) #define current_thread_fast() get_active_thread() #define current_thread() current_thread_fast() -static inline boolean_t -get_is64bit(void) -{ - CPU_DATA_GET(cpu_is64bit, boolean_t) -} -#define cpu_mode_is64bit() get_is64bit() +#define cpu_mode_is64bit() TRUE static inline int get_preemption_level(void) @@ -214,11 +336,6 @@ get_preemption_level(void) CPU_DATA_GET(cpu_preemption_level,int) } static inline int -get_simple_lock_count(void) -{ - CPU_DATA_GET(cpu_simple_lock_count,int) -} -static inline int get_interrupt_level(void) { CPU_DATA_GET(cpu_interrupt_level,int) @@ -233,18 +350,19 @@ get_cpu_phys_number(void) { CPU_DATA_GET(cpu_phys_number,int) } -static inline struct -cpu_core * get_cpu_core(void) -{ - CPU_DATA_GET(cpu_core,struct cpu_core *) -} + static inline void disable_preemption(void) { +#if defined(__clang__) + cpu_data_t GS_RELATIVE *cpu_data = (cpu_data_t GS_RELATIVE *)0UL; + cpu_data->cpu_preemption_level++; +#else __asm__ volatile ("incl %%gs:%P0" : : "i" (offsetof(cpu_data_t, cpu_preemption_level))); +#endif } static inline void @@ -252,6 +370,11 @@ enable_preemption(void) { assert(get_preemption_level() > 0); +#if defined(__clang__) + cpu_data_t GS_RELATIVE *cpu_data = (cpu_data_t GS_RELATIVE *)0UL; + if (0 == --cpu_data->cpu_preemption_level) + kernel_preempt_check(); +#else __asm__ volatile ("decl %%gs:%P0 \n\t" "jne 1f \n\t" "call _kernel_preempt_check \n\t" @@ -259,6 +382,7 @@ enable_preemption(void) : /* no outputs */ : "i" (offsetof(cpu_data_t, cpu_preemption_level)) : "eax", "ecx", "edx", "cc", "memory"); +#endif } static inline void @@ -266,10 +390,15 @@ enable_preemption_no_check(void) { assert(get_preemption_level() > 0); +#if defined(__clang__) + cpu_data_t GS_RELATIVE *cpu_data = (cpu_data_t GS_RELATIVE *)0UL; + cpu_data->cpu_preemption_level--; +#else __asm__ volatile ("decl %%gs:%P0" : /* no outputs */ : "i" (offsetof(cpu_data_t, cpu_preemption_level)) : "cc", "memory"); +#endif } static inline void @@ -299,14 +428,10 @@ current_cpu_datap(void) static inline cpu_data_t * cpu_datap(int cpu) { - assert(cpu_data_ptr[cpu]); return cpu_data_ptr[cpu]; } extern cpu_data_t *cpu_data_alloc(boolean_t is_boot_cpu); - -#else /* !defined(__GNUC__) */ - -#endif /* defined(__GNUC__) */ +extern void cpu_data_realloc(void); #endif /* I386_CPU_DATA */