X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/0a7de7458d150b5d4dffc935ba399be265ef0a1a..refs/heads/master:/osfmk/i386/cpuid.c diff --git a/osfmk/i386/cpuid.c b/osfmk/i386/cpuid.c index 75da80e5f..90844be06 100644 --- a/osfmk/i386/cpuid.c +++ b/osfmk/i386/cpuid.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2019 Apple Inc. All rights reserved. + * Copyright (c) 2000-2020 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -33,6 +33,7 @@ #include #include +#include int force_tecs_at_idle; int tecs_mode_supported; @@ -212,7 +213,17 @@ static cpuid_cache_descriptor_t intel_cpuid_leaf2_descriptor_table[] = { #define INTEL_LEAF2_DESC_NUM (sizeof(intel_cpuid_leaf2_descriptor_table) / \ sizeof(cpuid_cache_descriptor_t)) +boolean_t cpuid_tsx_disabled = false; /* true if XNU disabled TSX */ +boolean_t cpuid_tsx_supported = false; + static void do_cwas(i386_cpu_info_t *cpuinfo, boolean_t on_slave); +static void cpuid_do_precpuid_was(void); + +#if DEBUG || DEVELOPMENT +static void cpuid_vmm_detect_pv_interface(i386_vmm_info_t *info_p, const char *signature, + bool (*)(i386_vmm_info_t*, const uint32_t, const uint32_t)); +static bool cpuid_vmm_detect_applepv_features(i386_vmm_info_t *info_p, const uint32_t base, const uint32_t max_leaf); +#endif /* DEBUG || DEVELOPMENT */ static inline cpuid_cache_descriptor_t * cpuid_leaf2_find(uint8_t value) @@ -250,19 +261,34 @@ static void do_cwas(i386_cpu_info_t *cpuinfo, boolean_t on_slave) { extern int force_thread_policy_tecs; + cwa_classifier_e wa_reqd; /* * Workaround for reclaiming perf counter 3 due to TSX memory ordering erratum. * This workaround does not support being forcibly set (since an MSR must be * enumerated, lest we #GP when forced to access it.) - * When RTM_FORCE_FORCE is enabled all RTM transactions on the logical CPU will - * forcefully abort, but the general purpose counter 3 will report correct values. + * + * Note that if disabling TSX is supported, disablement is prefered over forcing + * TSX transactions to abort. */ - if (cpuid_wa_required(CPU_INTEL_TSXFA) == CWA_ON) { + if (cpuid_wa_required(CPU_INTEL_TSXDA) == CWA_ON) { + /* This must be executed on all logical processors */ + wrmsr64(MSR_IA32_TSX_CTRL, MSR_IA32_TSXCTRL_TSX_CPU_CLEAR | MSR_IA32_TSXCTRL_RTM_DISABLE); + } else if (cpuid_wa_required(CPU_INTEL_TSXFA) == CWA_ON) { + /* This must be executed on all logical processors */ wrmsr64(MSR_IA32_TSX_FORCE_ABORT, rdmsr64(MSR_IA32_TSX_FORCE_ABORT) | MSR_IA32_TSXFA_RTM_FORCE_ABORT); } + if (((wa_reqd = cpuid_wa_required(CPU_INTEL_SRBDS)) & CWA_ON) != 0 && + ((wa_reqd & CWA_FORCE_ON) == CWA_ON || + (cpuinfo->cpuid_leaf7_extfeatures & CPUID_LEAF7_EXTFEATURE_SRBDS_CTRL) != 0)) { + /* This must be executed on all logical processors */ + uint64_t mcuoptctrl = rdmsr64(MSR_IA32_MCU_OPT_CTRL); + mcuoptctrl |= MSR_IA32_MCUOPTCTRL_RNGDS_MITG_DIS; + wrmsr64(MSR_IA32_MCU_OPT_CTRL, mcuoptctrl); + } + if (on_slave) { return; } @@ -276,7 +302,7 @@ do_cwas(i386_cpu_info_t *cpuinfo, boolean_t on_slave) force_tecs_at_idle = 1; } - /*FALLTHROUGH*/ + OS_FALLTHROUGH; case CWA_ON: tecs_mode_supported = 1; break; @@ -323,7 +349,7 @@ cpuid_set_cache_info( i386_cpu_info_t * info_p ) if ((cpuid_result[j] >> 31) == 1) { /* bit31 is validity */ continue; } - ((uint32_t *) info_p->cache_info)[j] = cpuid_result[j]; + ((uint32_t *)(void *)info_p->cache_info)[j] = cpuid_result[j]; } /* first byte gives number of cpuid calls to get all descriptors */ for (i = 1; i < info_p->cache_info[0]; i++) { @@ -335,7 +361,7 @@ cpuid_set_cache_info( i386_cpu_info_t * info_p ) if ((cpuid_result[j] >> 31) == 1) { continue; } - ((uint32_t *) info_p->cache_info)[4 * i + j] = + ((uint32_t *)(void *)info_p->cache_info)[4 * i + j] = cpuid_result[j]; } } @@ -808,6 +834,8 @@ cpuid_set_generic_info(i386_cpu_info_t *info_p) info_p->cpuid_leaf7_features = quad(reg[ecx], reg[ebx]); info_p->cpuid_leaf7_extfeatures = reg[edx]; + cpuid_tsx_supported = (reg[ebx] & (CPUID_LEAF7_FEATURE_HLE | CPUID_LEAF7_FEATURE_RTM)) != 0; + DBG(" Feature Leaf7:\n"); DBG(" EBX : 0x%x\n", reg[ebx]); DBG(" ECX : 0x%x\n", reg[ecx]); @@ -872,15 +900,18 @@ cpuid_set_cpufamily(i386_cpu_info_t *info_p) break; case CPUID_MODEL_SKYLAKE: case CPUID_MODEL_SKYLAKE_DT: -#if !defined(RC_HIDE_XNU_J137) case CPUID_MODEL_SKYLAKE_W: -#endif cpufamily = CPUFAMILY_INTEL_SKYLAKE; break; case CPUID_MODEL_KABYLAKE: case CPUID_MODEL_KABYLAKE_DT: cpufamily = CPUFAMILY_INTEL_KABYLAKE; break; + case CPUID_MODEL_ICELAKE: + case CPUID_MODEL_ICELAKE_H: + case CPUID_MODEL_ICELAKE_DT: + cpufamily = CPUFAMILY_INTEL_ICELAKE; + break; } break; } @@ -899,6 +930,9 @@ cpuid_set_info(void) i386_cpu_info_t *info_p = &cpuid_cpu_info; boolean_t enable_x86_64h = TRUE; + /* Perform pre-cpuid workarounds (since their effects impact values returned via cpuid) */ + cpuid_do_precpuid_was(); + cpuid_set_generic_info(info_p); /* verify we are running on a supported CPU */ @@ -929,43 +963,54 @@ cpuid_set_info(void) } /* cpuid_set_cache_info must be invoked after set_generic_info */ - if (info_p->cpuid_cpufamily == CPUFAMILY_INTEL_PENRYN) { - cpuid_set_cache_info(info_p); - } - /* * Find the number of enabled cores and threads * (which determines whether SMT/Hyperthreading is active). */ - switch (info_p->cpuid_cpufamily) { - case CPUFAMILY_INTEL_PENRYN: - info_p->core_count = info_p->cpuid_cores_per_package; - info_p->thread_count = info_p->cpuid_logical_per_package; - break; - case CPUFAMILY_INTEL_WESTMERE: { - uint64_t msr = rdmsr64(MSR_CORE_THREAD_COUNT); - info_p->core_count = bitfield32((uint32_t)msr, 19, 16); - info_p->thread_count = bitfield32((uint32_t)msr, 15, 0); - break; - } - default: { - uint64_t msr = rdmsr64(MSR_CORE_THREAD_COUNT); - if (msr == 0) { - /* Provide a non-zero default for some VMMs */ - msr = (1 << 16) + 1; - } - info_p->core_count = bitfield32((uint32_t)msr, 31, 16); - info_p->thread_count = bitfield32((uint32_t)msr, 15, 0); - break; - } - } - if (info_p->core_count == 0) { - info_p->core_count = info_p->cpuid_cores_per_package; - info_p->thread_count = info_p->cpuid_logical_per_package; - } - if (info_p->cpuid_cpufamily != CPUFAMILY_INTEL_PENRYN) { + /* + * Not all VMMs emulate MSR_CORE_THREAD_COUNT (0x35). + */ + if (0 != (info_p->cpuid_features & CPUID_FEATURE_VMM) && + PE_parse_boot_argn("-nomsr35h", NULL, 0)) { + info_p->core_count = 1; + info_p->thread_count = 1; cpuid_set_cache_info(info_p); + } else { + switch (info_p->cpuid_cpufamily) { + case CPUFAMILY_INTEL_PENRYN: + cpuid_set_cache_info(info_p); + info_p->core_count = info_p->cpuid_cores_per_package; + info_p->thread_count = info_p->cpuid_logical_per_package; + break; + case CPUFAMILY_INTEL_WESTMERE: { + /* + * This should be the same as Nehalem but an A0 silicon bug returns + * invalid data in the top 12 bits. Hence, we use only bits [19..16] + * rather than [31..16] for core count - which actually can't exceed 8. + */ + uint64_t msr = rdmsr64(MSR_CORE_THREAD_COUNT); + if (0 == msr) { + /* Provide a non-zero default for some VMMs */ + msr = (1 << 16) | 1; + } + info_p->core_count = bitfield32((uint32_t)msr, 19, 16); + info_p->thread_count = bitfield32((uint32_t)msr, 15, 0); + cpuid_set_cache_info(info_p); + break; + } + default: { + uint64_t msr = rdmsr64(MSR_CORE_THREAD_COUNT); + if (0 == msr) { + /* Provide a non-zero default for some VMMs */ + msr = (1 << 16) | 1; + } + info_p->core_count = bitfield32((uint32_t)msr, 31, 16); + info_p->thread_count = bitfield32((uint32_t)msr, 15, 0); + cpuid_set_cache_info(info_p); + break; + } + } } DBG("cpuid_set_info():\n"); @@ -976,6 +1021,9 @@ cpuid_set_info(void) info_p->cpuid_model_string = ""; /* deprecated */ + /* Init CPU LBRs */ + i386_lbr_init(info_p, true); + do_cwas(info_p, FALSE); } @@ -1093,7 +1141,11 @@ static struct table { {CPUID_LEAF7_FEATURE_OSPKE, "OSPKE"}, {CPUID_LEAF7_FEATURE_WAITPKG, "WAITPKG"}, {CPUID_LEAF7_FEATURE_GFNI, "GFNI"}, - {CPUID_LEAF7_FEATURE_AVX512VPCDQ, "AVX512VPCDQ"}, + {CPUID_LEAF7_FEATURE_VAES, "VAES"}, + {CPUID_LEAF7_FEATURE_VPCLMULQDQ, "VPCLMULQDQ"}, + {CPUID_LEAF7_FEATURE_AVX512VNNI, "AVX512VNNI"}, + {CPUID_LEAF7_FEATURE_AVX512BITALG, "AVX512BITALG"}, + {CPUID_LEAF7_FEATURE_AVX512VPCDQ, "AVX512VPOPCNTDQ"}, {CPUID_LEAF7_FEATURE_RDPID, "RDPID"}, {CPUID_LEAF7_FEATURE_CLDEMOTE, "CLDEMOTE"}, {CPUID_LEAF7_FEATURE_MOVDIRI, "MOVDIRI"}, @@ -1104,6 +1156,7 @@ static struct table { leaf7_extfeature_map[] = { { CPUID_LEAF7_EXTFEATURE_AVX5124VNNIW, "AVX5124VNNIW" }, { CPUID_LEAF7_EXTFEATURE_AVX5124FMAPS, "AVX5124FMAPS" }, + { CPUID_LEAF7_EXTFEATURE_FSREPMOV, "FSREPMOV" }, { CPUID_LEAF7_EXTFEATURE_MDCLEAR, "MDCLEAR" }, { CPUID_LEAF7_EXTFEATURE_TSXFA, "TSXFA" }, { CPUID_LEAF7_EXTFEATURE_IBRS, "IBRS" }, @@ -1290,13 +1343,42 @@ cpuid_leaf7_extfeatures(void) return cpuid_info()->cpuid_leaf7_extfeatures; } +const char * +cpuid_vmm_family_string(void) +{ + switch (cpuid_vmm_info()->cpuid_vmm_family) { + case CPUID_VMM_FAMILY_NONE: + return "None"; + + case CPUID_VMM_FAMILY_VMWARE: + return "VMWare"; + + case CPUID_VMM_FAMILY_PARALLELS: + return "Parallels"; + + case CPUID_VMM_FAMILY_HYVE: + return "xHyve"; + + case CPUID_VMM_FAMILY_HVF: + return "HVF"; + + case CPUID_VMM_FAMILY_KVM: + return "KVM"; + + case CPUID_VMM_FAMILY_UNKNOWN: + /*FALLTHROUGH*/ + default: + return "Unknown VMM"; + } +} + static i386_vmm_info_t *_cpuid_vmm_infop = NULL; static i386_vmm_info_t _cpuid_vmm_info; static void cpuid_init_vmm_info(i386_vmm_info_t *info_p) { - uint32_t reg[4]; + uint32_t reg[4], maxbasic_regs[4]; uint32_t max_vmm_leaf; bzero(info_p, sizeof(*info_p)); @@ -1307,8 +1389,27 @@ cpuid_init_vmm_info(i386_vmm_info_t *info_p) DBG("cpuid_init_vmm_info(%p)\n", info_p); + /* + * Get the highest basic leaf value, then save the cpuid details for that leaf + * for comparison with the [ostensible] VMM leaf. + */ + cpuid_fn(0, reg); + cpuid_fn(reg[eax], maxbasic_regs); + /* do cpuid 0x40000000 to get VMM vendor */ cpuid_fn(0x40000000, reg); + + /* + * If leaf 0x40000000 is non-existent, cpuid will return the values as + * if the highest basic leaf was requested, so compare to those values + * we just retrieved to see if no vmm is present. + */ + if (bcmp(reg, maxbasic_regs, sizeof(reg)) == 0) { + info_p->cpuid_vmm_family = CPUID_VMM_FAMILY_NONE; + DBG(" vmm_vendor : NONE\n"); + return; + } + max_vmm_leaf = reg[eax]; bcopy((char *)®[ebx], &info_p->cpuid_vmm_vendor[0], 4); bcopy((char *)®[ecx], &info_p->cpuid_vmm_vendor[4], 4); @@ -1318,9 +1419,18 @@ cpuid_init_vmm_info(i386_vmm_info_t *info_p) if (0 == strcmp(info_p->cpuid_vmm_vendor, CPUID_VMM_ID_VMWARE)) { /* VMware identification string: kb.vmware.com/kb/1009458 */ info_p->cpuid_vmm_family = CPUID_VMM_FAMILY_VMWARE; - } else if (0 == strcmp(info_p->cpuid_vmm_vendor, CPUID_VMM_ID_PARALLELS)) { + } else if (0 == bcmp(info_p->cpuid_vmm_vendor, CPUID_VMM_ID_PARALLELS, 12)) { /* Parallels identification string */ info_p->cpuid_vmm_family = CPUID_VMM_FAMILY_PARALLELS; + } else if (0 == bcmp(info_p->cpuid_vmm_vendor, CPUID_VMM_ID_HYVE, 12)) { + /* bhyve/xhyve identification string */ + info_p->cpuid_vmm_family = CPUID_VMM_FAMILY_HYVE; + } else if (0 == bcmp(info_p->cpuid_vmm_vendor, CPUID_VMM_ID_HVF, 12)) { + /* HVF identification string */ + info_p->cpuid_vmm_family = CPUID_VMM_FAMILY_HVF; + } else if (0 == bcmp(info_p->cpuid_vmm_vendor, CPUID_VMM_ID_KVM, 12)) { + /* KVM identification string */ + info_p->cpuid_vmm_family = CPUID_VMM_FAMILY_KVM; } else { info_p->cpuid_vmm_family = CPUID_VMM_FAMILY_UNKNOWN; } @@ -1333,6 +1443,10 @@ cpuid_init_vmm_info(i386_vmm_info_t *info_p) info_p->cpuid_vmm_bus_frequency = reg[ebx]; } +#if DEBUG || DEVELOPMENT + cpuid_vmm_detect_pv_interface(info_p, APPLEPV_SIGNATURE, &cpuid_vmm_detect_applepv_features); +#endif + DBG(" vmm_vendor : %s\n", info_p->cpuid_vmm_vendor); DBG(" vmm_family : %u\n", info_p->cpuid_vmm_family); DBG(" vmm_bus_frequency : %u\n", info_p->cpuid_vmm_bus_frequency); @@ -1361,13 +1475,22 @@ cpuid_vmm_family(void) return cpuid_vmm_info()->cpuid_vmm_family; } +#if DEBUG || DEVELOPMENT +uint64_t +cpuid_vmm_get_applepv_features(void) +{ + return cpuid_vmm_info()->cpuid_vmm_applepv_features; +} +#endif /* DEBUG || DEVELOPMENT */ + cwa_classifier_e cpuid_wa_required(cpu_wa_e wa) { + i386_cpu_info_t *info_p = &cpuid_cpu_info; static uint64_t bootarg_cpu_wa_enables = 0; static uint64_t bootarg_cpu_wa_disables = 0; static int bootargs_overrides_processed = 0; - i386_cpu_info_t *info_p = &cpuid_cpu_info; + uint32_t reg[4]; if (!bootargs_overrides_processed) { if (!PE_parse_boot_argn("cwae", &bootarg_cpu_wa_enables, sizeof(bootarg_cpu_wa_enables))) { @@ -1414,7 +1537,12 @@ cpuid_wa_required(cpu_wa_e wa) case CPU_INTEL_TSXFA: /* - * If this CPU supports RTM and supports FORCE_ABORT, return that + * Note that if TSX was disabled in cpuid_do_precpuid_was(), the cached cpuid + * info will indicate that RTM is *not* supported and this workaround will not + * be enabled. + */ + /* + * Otherwise, if the CPU supports both TSX(HLE) and FORCE_ABORT, return that * the workaround should be enabled. */ if ((info_p->cpuid_leaf7_extfeatures & CPUID_LEAF7_EXTFEATURE_TSXFA) != 0 && @@ -1423,9 +1551,131 @@ cpuid_wa_required(cpu_wa_e wa) } break; + case CPU_INTEL_TSXDA: + /* + * Since this workaround might be requested before cpuid_set_info() is complete, + * we need to invoke cpuid directly when looking for the required bits. + */ + cpuid_fn(0x7, reg); + if (reg[edx] & CPUID_LEAF7_EXTFEATURE_ACAPMSR) { + uint64_t archcap_msr = rdmsr64(MSR_IA32_ARCH_CAPABILITIES); + /* + * If this CPU supports TSX (HLE being the proxy for TSX detection) AND it does + * not include a hardware fix for TAA and it supports the TSX_CTRL MSR, disable TSX entirely. + * (Note this can be overridden (above) if the cwad boot-arg's value has bit 2 set.) + */ + if ((reg[ebx] & CPUID_LEAF7_FEATURE_HLE) != 0 && + (archcap_msr & (MSR_IA32_ARCH_CAPABILITIES_TAA_NO | MSR_IA32_ARCH_CAPABILITIES_TSX_CTRL)) + == MSR_IA32_ARCH_CAPABILITIES_TSX_CTRL) { + return CWA_ON; + } + } + break; + + case CPU_INTEL_SRBDS: + /* + * SRBDS mitigations are enabled by default. CWA_ON returned here indicates + * the caller should disable the mitigation. Mitigations should be disabled + * at least for CPUs that advertise MDS_NO *and* (either TAA_NO is set OR TSX + * has been disabled). + */ + if ((info_p->cpuid_leaf7_extfeatures & CPUID_LEAF7_EXTFEATURE_SRBDS_CTRL) != 0) { + if ((info_p->cpuid_leaf7_extfeatures & CPUID_LEAF7_EXTFEATURE_ACAPMSR) != 0) { + uint64_t archcap_msr = rdmsr64(MSR_IA32_ARCH_CAPABILITIES); + if ((archcap_msr & MSR_IA32_ARCH_CAPABILITIES_MDS_NO) != 0 && + ((archcap_msr & MSR_IA32_ARCH_CAPABILITIES_TAA_NO) != 0 || + cpuid_tsx_disabled)) { + return CWA_ON; + } + } + } + break; + default: break; } return CWA_OFF; } + +static void +cpuid_do_precpuid_was(void) +{ + /* + * Note that care must be taken not to use any data from the cached cpuid data since it is + * likely uninitialized at this point. That includes calling functions that make use of + * that data as well. + */ + + /* Note the TSX disablement, we do not support force-on since it depends on MSRs being present */ + if (cpuid_wa_required(CPU_INTEL_TSXDA) == CWA_ON) { + /* This must be executed on all logical processors */ + wrmsr64(MSR_IA32_TSX_CTRL, MSR_IA32_TSXCTRL_TSX_CPU_CLEAR | MSR_IA32_TSXCTRL_RTM_DISABLE); + cpuid_tsx_disabled = true; + } +} + + +#if DEBUG || DEVELOPMENT + +/* + * Hunt for Apple Paravirtualization support in the hypervisor class leaves [0x4000_0000-0x4001_0000]. + * Hypervisor interfaces are expected to be found at 0x100 boundaries for compatibility. + */ + +static bool +cpuid_vmm_detect_applepv_features(i386_vmm_info_t *info_p, const uint32_t base, const uint32_t max_leaf) +{ + if ((max_leaf - base) < APPLEPV_LEAF_INDEX_MAX) { + return false; + } + + /* + * Issue cpuid to make sure the interface supports "AH#1" features. + * This avoids a possible collision with "Hv#1" used by Hyper-V. + */ + uint32_t reg[4]; + char interface[5]; + cpuid_fn(base + APPLEPV_INTERFACE_LEAF_INDEX, reg); + memcpy(&interface[0], ®[eax], 4); + interface[4] = '\0'; + if (0 == strcmp(interface, APPLEPV_INTERFACE)) { + cpuid_fn(base + APPLEPV_FEATURES_LEAF_INDEX, reg); + info_p->cpuid_vmm_applepv_features = quad(reg[ecx], reg[edx]); + return true; + } + return false; +} + +static void +cpuid_vmm_detect_pv_interface(i386_vmm_info_t *info_p, const char *signature, + bool (*searcher)(i386_vmm_info_t*, const uint32_t, const uint32_t)) +{ + int hcalls; + if (PE_parse_boot_argn("hcalls", &hcalls, sizeof(hcalls)) && + hcalls == 0) { + return; + } + + assert(info_p); + /* + * Look for PV interface matching signature + */ + for (uint32_t base = 0x40000100; base < 0x40010000; base += 0x100) { + uint32_t reg[4]; + char vendor[13]; + + cpuid_fn(base, reg); + memcpy(&vendor[0], ®[ebx], 4); + memcpy(&vendor[4], ®[ecx], 4); + memcpy(&vendor[8], ®[edx], 4); + vendor[12] = '\0'; + if ((0 == strcmp(vendor, signature)) && + (reg[eax] - base) < 0x100 && + (*searcher)(info_p, base, reg[eax])) { + break; + } + } +} + +#endif /* DEBUG || DEVELOPMENT */