X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/39236c6e673c41db228275375ab7fdb0f837b292..d26ffc64f583ab2d29df48f13518685602bc8832:/osfmk/i386/vmx/vmx_cpu.c?ds=sidebyside diff --git a/osfmk/i386/vmx/vmx_cpu.c b/osfmk/i386/vmx/vmx_cpu.c index 27c1b3fc9..49b36f846 100644 --- a/osfmk/i386/vmx/vmx_cpu.c +++ b/osfmk/i386/vmx/vmx_cpu.c @@ -41,7 +41,9 @@ int vmx_use_count = 0; boolean_t vmx_exclusive = FALSE; -decl_simple_lock_data(static,vmx_use_count_lock) + +lck_grp_t *vmx_lck_grp = NULL; +lck_mtx_t *vmx_lck_mtx = NULL; /* ----------------------------------------------------------------------------- vmx_is_available() @@ -64,6 +66,7 @@ vmxon_is_enabled(void) (rdmsr64(MSR_IA32_FEATURE_CONTROL) & MSR_IA32_FEATCTL_VMXON)); } +#if MACH_ASSERT /* ----------------------------------------------------------------------------- vmx_is_cr0_valid() Is CR0 valid for executing VMXON on this CPU? @@ -86,8 +89,10 @@ vmx_is_cr4_valid(vmx_specs_t *specs) return (0 == ((~cr4 & specs->cr4_fixed_0)|(cr4 & ~specs->cr4_fixed_1))); } +#endif + static void -vmx_init(void) +vmx_enable(void) { uint64_t msr_image; @@ -104,6 +109,18 @@ vmx_init(void) (msr_image | MSR_IA32_FEATCTL_VMXON | MSR_IA32_FEATCTL_LOCK)); + + set_cr4(get_cr4() | CR4_VMXE); +} + +void +vmx_init() +{ + vmx_lck_grp = lck_grp_alloc_init("vmx", LCK_GRP_ATTR_NULL); + assert(vmx_lck_grp); + + vmx_lck_mtx = lck_mtx_alloc_init(vmx_lck_grp, LCK_ATTR_NULL); + assert(vmx_lck_mtx); } /* ----------------------------------------------------------------------------- @@ -114,20 +131,16 @@ vmx_init(void) the remainder of the vmx_specs_t uninitialized. -------------------------------------------------------------------------- */ void -vmx_get_specs() +vmx_cpu_init() { vmx_specs_t *specs = ¤t_cpu_datap()->cpu_vmx.specs; - uint64_t msr_image; - - /* this is called once for every CPU, but the lock doesn't care :-) */ - simple_lock_init(&vmx_use_count_lock, 0); - vmx_init(); + vmx_enable(); - /* - * if we have read the data on boot, we won't read it - * again on wakeup, otherwise *bad* things will happen - */ + VMX_KPRINTF("[%d]vmx_cpu_init() initialized: %d\n", + cpu_number(), specs->initialized); + + /* if we have read the data on boot, we won't read it again on wakeup */ if (specs->initialized) return; else @@ -135,53 +148,21 @@ vmx_get_specs() /* See if VMX is present, return if it is not */ specs->vmx_present = vmx_is_available() && vmxon_is_enabled(); + VMX_KPRINTF("[%d]vmx_cpu_init() vmx_present: %d\n", + cpu_number(), specs->vmx_present); if (!specs->vmx_present) return; -#define bitfield(x,f) ((x >> f##_BIT) & f##_MASK) - /* Obtain and decode VMX general capabilities */ - msr_image = rdmsr64(MSR_IA32_VMX_BASIC); - specs->vmcs_id = (uint32_t)(msr_image & VMX_VCR_VMCS_REV_ID); - specs->vmcs_mem_type = bitfield(msr_image, VMX_VCR_VMCS_MEM_TYPE) != 0; - specs->vmcs_size = bitfield(msr_image, VMX_VCR_VMCS_SIZE); - - /* Obtain allowed settings for pin-based execution controls */ - msr_image = rdmsr64(MSR_IA32_VMXPINBASED_CTLS); - specs->pin_exctls_0 = (uint32_t)(msr_image & 0xFFFFFFFF); - specs->pin_exctls_1 = (uint32_t)(msr_image >> 32); - - /* Obtain allowed settings for processor-based execution controls */ - msr_image = rdmsr64(MSR_IA32_PROCBASED_CTLS); - specs->proc_exctls_0 = (uint32_t)(msr_image & 0xFFFFFFFF); - specs->proc_exctls_1 = (uint32_t)(msr_image >> 32); - - /* Obtain allowed settings for VM-exit controls */ - msr_image = rdmsr64(MSR_IA32_VMX_EXIT_CTLS); - specs->exit_ctls_0 = (uint32_t)(msr_image & 0xFFFFFFFF); - specs->exit_ctls_1 = (uint32_t)(msr_image >> 32); - - /* Obtain allowed settings for VM-entry controls */ - msr_image = rdmsr64(MSR_IA32_VMX_ENTRY_CTLS); - specs->enter_ctls_0 = (uint32_t)(msr_image & 0xFFFFFFFF); - specs->enter_ctls_0 = (uint32_t)(msr_image >> 32); - - /* Obtain and decode miscellaneous capabilities */ - msr_image = rdmsr64(MSR_IA32_VMX_MISC); - specs->act_halt = bitfield(msr_image, VMX_VCR_ACT_HLT) != 0; - specs->act_shutdown = bitfield(msr_image, VMX_VCR_ACT_SHUTDOWN) != 0; - specs->act_SIPI = bitfield(msr_image, VMX_VCR_ACT_SIPI) != 0; - specs->act_CSTATE = bitfield(msr_image, VMX_VCR_ACT_CSTATE) != 0; - specs->cr3_targs = bitfield(msr_image, VMX_VCR_CR3_TARGS); - specs->max_msrs = (uint32_t)(512 * (1 + bitfield(msr_image, VMX_VCR_MAX_MSRS))); - specs->mseg_id = (uint32_t)bitfield(msr_image, VMX_VCR_MSEG_ID); - +#define rdmsr_mask(msr, mask) (uint32_t)(rdmsr64(msr) & (mask)) + specs->vmcs_id = rdmsr_mask(MSR_IA32_VMX_BASIC, VMX_VCR_VMCS_REV_ID); + /* Obtain VMX-fixed bits in CR0 */ - specs->cr0_fixed_0 = (uint32_t)rdmsr64(MSR_IA32_VMX_CR0_FIXED0) & 0xFFFFFFFF; - specs->cr0_fixed_1 = (uint32_t)rdmsr64(MSR_IA32_VMX_CR0_FIXED1) & 0xFFFFFFFF; + specs->cr0_fixed_0 = rdmsr_mask(MSR_IA32_VMX_CR0_FIXED0, 0xFFFFFFFF); + specs->cr0_fixed_1 = rdmsr_mask(MSR_IA32_VMX_CR0_FIXED1, 0xFFFFFFFF); /* Obtain VMX-fixed bits in CR4 */ - specs->cr4_fixed_0 = (uint32_t)rdmsr64(MSR_IA32_VMX_CR4_FIXED0) & 0xFFFFFFFF; - specs->cr4_fixed_1 = (uint32_t)rdmsr64(MSR_IA32_VMX_CR4_FIXED1) & 0xFFFFFFFF; + specs->cr4_fixed_0 = rdmsr_mask(MSR_IA32_VMX_CR4_FIXED0, 0xFFFFFFFF); + specs->cr4_fixed_1 = rdmsr_mask(MSR_IA32_VMX_CR4_FIXED1, 0xFFFFFFFF); } /* ----------------------------------------------------------------------------- @@ -195,8 +176,9 @@ vmx_on(void *arg __unused) addr64_t vmxon_region_paddr; int result; - vmx_init(); - + VMX_KPRINTF("[%d]vmx_on() entry state: %d\n", + cpu_number(), cpu->specs.vmx_on); + assert(cpu->specs.vmx_present); if (NULL == cpu->vmxon_region) @@ -206,16 +188,20 @@ vmx_on(void *arg __unused) /* * Enable VMX operation. */ - set_cr4(get_cr4() | CR4_VMXE); + if (FALSE == cpu->specs.vmx_on) { + assert(vmx_is_cr0_valid(&cpu->specs)); + assert(vmx_is_cr4_valid(&cpu->specs)); - assert(vmx_is_cr0_valid(&cpu->specs)); - assert(vmx_is_cr4_valid(&cpu->specs)); - - result = __vmxon(vmxon_region_paddr); + result = __vmxon(vmxon_region_paddr); + + if (result != VMX_SUCCEED) { + panic("vmx_on: unexpected return %d from __vmxon()", result); + } - if (result != VMX_SUCCEED) { - panic("vmx_on: unexpected return %d from __vmxon()", result); + cpu->specs.vmx_on = TRUE; } + VMX_KPRINTF("[%d]vmx_on() return state: %d\n", + cpu_number(), cpu->specs.vmx_on); } /* ----------------------------------------------------------------------------- @@ -225,16 +211,25 @@ vmx_on(void *arg __unused) static void vmx_off(void *arg __unused) { + vmx_cpu_t *cpu = ¤t_cpu_datap()->cpu_vmx; int result; - /* Tell the CPU to release the VMXON region */ - result = __vmxoff(); + VMX_KPRINTF("[%d]vmx_off() entry state: %d\n", + cpu_number(), cpu->specs.vmx_on); + + if (TRUE == cpu->specs.vmx_on) { + /* Tell the CPU to release the VMXON region */ + result = __vmxoff(); - if (result != VMX_SUCCEED) { - panic("vmx_off: unexpected return %d from __vmxoff()", result); + if (result != VMX_SUCCEED) { + panic("vmx_off: unexpected return %d from __vmxoff()", result); + } + + cpu->specs.vmx_on = FALSE; } - set_cr4(get_cr4() & ~CR4_VMXE); + VMX_KPRINTF("[%d]vmx_off() return state: %d\n", + cpu_number(), cpu->specs.vmx_on); } /* ----------------------------------------------------------------------------- @@ -282,10 +277,10 @@ static boolean_t vmx_globally_available(void) { unsigned int i; - + unsigned int ncpus = ml_get_max_cpus(); boolean_t available = TRUE; - for (i=0; icpu_vmx; if (!cpu->specs.vmx_present) @@ -304,31 +299,33 @@ int host_vmxon(boolean_t exclusive) { int error; - boolean_t do_it = FALSE; /* do the cpu sync outside of the area holding the lock */ + + assert(0 == get_preemption_level()); if (!vmx_globally_available()) return VMX_UNSUPPORTED; - simple_lock(&vmx_use_count_lock); + lck_mtx_lock(vmx_lck_mtx); - if (vmx_exclusive) { + if (vmx_exclusive || (exclusive && vmx_use_count)) { error = VMX_INUSE; } else { - vmx_use_count++; - if (vmx_use_count == 1) /* was turned off before */ - do_it = TRUE; - vmx_exclusive = exclusive; + if (0 == vmx_use_count) { + vmx_allocate_vmxon_regions(); + vmx_exclusive = exclusive; + vmx_use_count = 1; + mp_cpus_call(CPUMASK_ALL, ASYNC, vmx_on, NULL); + + } else { + vmx_use_count++; + } VMX_KPRINTF("VMX use count: %d\n", vmx_use_count); error = VMX_OK; } - simple_unlock(&vmx_use_count_lock); + lck_mtx_unlock(vmx_lck_mtx); - if (do_it) { - vmx_allocate_vmxon_regions(); - mp_rendezvous(NULL, vmx_on, NULL, NULL); - } return error; } @@ -339,24 +336,21 @@ host_vmxon(boolean_t exclusive) void host_vmxoff() { - boolean_t do_it = FALSE; /* do the cpu sync outside of the area holding the lock */ + assert(0 == get_preemption_level()); - simple_lock(&vmx_use_count_lock); + lck_mtx_lock(vmx_lck_mtx); - if (vmx_use_count) { - vmx_use_count--; + if (1 == vmx_use_count) { vmx_exclusive = FALSE; - if (!vmx_use_count) - do_it = TRUE; - } - - simple_unlock(&vmx_use_count_lock); - - if (do_it) { - mp_rendezvous(NULL, vmx_off, NULL, NULL); + vmx_use_count = 0; + mp_cpus_call(CPUMASK_ALL, ASYNC, vmx_off, NULL); vmx_free_vmxon_regions(); + } else { + vmx_use_count--; } + lck_mtx_unlock(vmx_lck_mtx); + VMX_KPRINTF("VMX use count: %d\n", vmx_use_count); } @@ -369,6 +363,7 @@ void vmx_suspend() { VMX_KPRINTF("vmx_suspend\n"); + if (vmx_use_count) vmx_off(NULL); } @@ -378,10 +373,49 @@ vmx_suspend() Restore the previous VT state. Called when CPU comes back online. -------------------------------------------------------------------------- */ void -vmx_resume() +vmx_resume(boolean_t is_wake_from_hibernate) { VMX_KPRINTF("vmx_resume\n"); - vmx_init(); /* init VMX on CPU #0 */ - if (vmx_use_count) - vmx_on(NULL); + + vmx_enable(); + + if (vmx_use_count == 0) + return; + + /* + * When resuming from hiberate on the boot cpu, + * we must mark VMX as off since that's the state at wake-up + * because the restored state in memory records otherwise. + * This results in vmx_on() doing the right thing. + */ + if (is_wake_from_hibernate) { + vmx_cpu_t *cpu = ¤t_cpu_datap()->cpu_vmx; + cpu->specs.vmx_on = FALSE; + } + + vmx_on(NULL); +} + +/* ----------------------------------------------------------------------------- + vmx_hv_support() + Determine if the VMX feature set is sufficent for kernel HV support. + -------------------------------------------------------------------------- */ +boolean_t +vmx_hv_support() +{ + if (!vmx_is_available()) + return FALSE; + +#define CHK(msr, shift, mask) if (!VMX_CAP(msr, shift, mask)) return FALSE; + + /* 'EPT' and 'Unrestricted Mode' are part of the secondary processor-based + * VM-execution controls */ + CHK(MSR_IA32_VMX_BASIC, 0, VMX_BASIC_TRUE_CTLS) + CHK(MSR_IA32_VMX_TRUE_PROCBASED_CTLS, 32, VMX_TRUE_PROCBASED_SECONDARY_CTLS) + + /* if we have these, check for 'EPT' and 'Unrestricted Mode' */ + CHK(MSR_IA32_VMX_PROCBASED_CTLS2, 32, VMX_PROCBASED_CTLS2_EPT) + CHK(MSR_IA32_VMX_PROCBASED_CTLS2, 32, VMX_PROCBASED_CTLS2_UNRESTRICTED) + + return TRUE; }