[apple/xnu.git] / osfmk / i386 / vmx / vmx_cpu.c

/*
 * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <pexpert/pexpert.h>
#include <i386/cpuid.h>
#include <i386/cpu_data.h>
#include <i386/mp.h>
#include <i386/proc_reg.h>
#include <i386/vmx.h>
#include <i386/vmx/vmx_asm.h>
#include <i386/vmx/vmx_shims.h>
#include <i386/vmx/vmx_cpu.h>
#include <mach/mach_host.h>             /* for host_info() */

#define VMX_KPRINTF(x...) /* kprintf("vmx: " x) */

int vmx_use_count = 0;
boolean_t vmx_exclusive = FALSE;
decl_simple_lock_data(static,vmx_use_count_lock)

/* -----------------------------------------------------------------------------
   vmx_is_available()
	Is the VMX facility available on this CPU?
   -------------------------------------------------------------------------- */
static inline boolean_t
vmx_is_available(void)
{
	return (0 != (cpuid_features() & CPUID_FEATURE_VMX));
}

/* -----------------------------------------------------------------------------
   vmxon_is_enabled()
	Is the VMXON instruction enabled on this CPU?
   -------------------------------------------------------------------------- */
static inline boolean_t
vmxon_is_enabled(void)
{
	return (vmx_is_available() &&
		(rdmsr64(MSR_IA32_FEATURE_CONTROL) & MSR_IA32_FEATCTL_VMXON));
}

/* -----------------------------------------------------------------------------
   vmx_is_cr0_valid()
	Is CR0 valid for executing VMXON on this CPU?
   -------------------------------------------------------------------------- */
static inline boolean_t
vmx_is_cr0_valid(vmx_specs_t *specs)
{
	uintptr_t cr0 = get_cr0();
	return (0 == ((~cr0 & specs->cr0_fixed_0)|(cr0 & ~specs->cr0_fixed_1)));
}

/* -----------------------------------------------------------------------------
   vmx_is_cr4_valid()
	Is CR4 valid for executing VMXON on this CPU?
   -------------------------------------------------------------------------- */
static inline boolean_t
vmx_is_cr4_valid(vmx_specs_t *specs)
{
	uintptr_t cr4 = get_cr4();
	return (0 == ((~cr4 & specs->cr4_fixed_0)|(cr4 & ~specs->cr4_fixed_1)));
}

static void
vmx_init(void)
{
	uint64_t msr_image;

	if (!vmx_is_available())
		return;

	/*
	 * We don't count on EFI initializing MSR_IA32_FEATURE_CONTROL
	 * and turning VMXON on and locking the bit, so we do that now.
	 */
	msr_image = rdmsr64(MSR_IA32_FEATURE_CONTROL);
	if (0 == ((msr_image & MSR_IA32_FEATCTL_LOCK)))
		wrmsr64(MSR_IA32_FEATURE_CONTROL,
			(msr_image |
			 MSR_IA32_FEATCTL_VMXON |
			 MSR_IA32_FEATCTL_LOCK));
}

/* -----------------------------------------------------------------------------
   vmx_get_specs()
	Obtain VMX facility specifications for this CPU and
	enter them into the vmx_specs_t structure. If VMX is not available or
	disabled on this CPU, set vmx_present to false and return leaving
	the remainder of the vmx_specs_t uninitialized. 
   -------------------------------------------------------------------------- */
void
vmx_get_specs()
{
	vmx_specs_t *specs = &current_cpu_datap()->cpu_vmx.specs;
	uint64_t msr_image;
	
	/* this is called once for every CPU, but the lock doesn't care :-) */
	simple_lock_init(&vmx_use_count_lock, 0);

	vmx_init();

	/*
	 * if we have read the data on boot, we won't read it
	 *  again on wakeup, otherwise *bad* things will happen
	 */
	if (specs->initialized)
		return;
	else
		specs->initialized = TRUE;

	/* See if VMX is present, return if it is not */
	specs->vmx_present = vmx_is_available() && vmxon_is_enabled();
	if (!specs->vmx_present)
		return;

#define bitfield(x,f)	((x >> f##_BIT) & f##_MASK)
	/* Obtain and decode VMX general capabilities */	
	msr_image = rdmsr64(MSR_IA32_VMX_BASIC);
	specs->vmcs_id       = (uint32_t)(msr_image & VMX_VCR_VMCS_REV_ID);
	specs->vmcs_mem_type = bitfield(msr_image, VMX_VCR_VMCS_MEM_TYPE) != 0;
	specs->vmcs_size = bitfield(msr_image, VMX_VCR_VMCS_SIZE);
							  
	/* Obtain allowed settings for pin-based execution controls */
	msr_image = rdmsr64(MSR_IA32_VMXPINBASED_CTLS);
	specs->pin_exctls_0 = (uint32_t)(msr_image & 0xFFFFFFFF);
	specs->pin_exctls_1 = (uint32_t)(msr_image >> 32);
	
	/* Obtain allowed settings for processor-based execution controls */
	msr_image = rdmsr64(MSR_IA32_PROCBASED_CTLS);
	specs->proc_exctls_0 = (uint32_t)(msr_image & 0xFFFFFFFF);
	specs->proc_exctls_1 = (uint32_t)(msr_image >> 32);
	
	/* Obtain allowed settings for VM-exit controls */
	msr_image = rdmsr64(MSR_IA32_VMX_EXIT_CTLS);
	specs->exit_ctls_0 = (uint32_t)(msr_image & 0xFFFFFFFF);
	specs->exit_ctls_1 = (uint32_t)(msr_image >> 32);
	
	/* Obtain allowed settings for VM-entry controls */
	msr_image = rdmsr64(MSR_IA32_VMX_ENTRY_CTLS);
	specs->enter_ctls_0 = (uint32_t)(msr_image & 0xFFFFFFFF);
	specs->enter_ctls_0 = (uint32_t)(msr_image >> 32);
	
	/* Obtain and decode miscellaneous capabilities */
	msr_image = rdmsr64(MSR_IA32_VMX_MISC);
	specs->act_halt     = bitfield(msr_image, VMX_VCR_ACT_HLT) != 0;
	specs->act_shutdown = bitfield(msr_image, VMX_VCR_ACT_SHUTDOWN) != 0;
	specs->act_SIPI     = bitfield(msr_image, VMX_VCR_ACT_SIPI) != 0;
	specs->act_CSTATE   = bitfield(msr_image, VMX_VCR_ACT_CSTATE) != 0;
	specs->cr3_targs    = bitfield(msr_image, VMX_VCR_CR3_TARGS);
	specs->max_msrs     = (uint32_t)(512 * (1 + bitfield(msr_image, VMX_VCR_MAX_MSRS)));
	specs->mseg_id      = (uint32_t)bitfield(msr_image, VMX_VCR_MSEG_ID);
	
	/* Obtain VMX-fixed bits in CR0 */
	specs->cr0_fixed_0 = (uint32_t)rdmsr64(MSR_IA32_VMX_CR0_FIXED0) & 0xFFFFFFFF;
	specs->cr0_fixed_1 = (uint32_t)rdmsr64(MSR_IA32_VMX_CR0_FIXED1) & 0xFFFFFFFF;
	
	/* Obtain VMX-fixed bits in CR4 */
	specs->cr4_fixed_0 = (uint32_t)rdmsr64(MSR_IA32_VMX_CR4_FIXED0) & 0xFFFFFFFF;
	specs->cr4_fixed_1 = (uint32_t)rdmsr64(MSR_IA32_VMX_CR4_FIXED1) & 0xFFFFFFFF;
}

/* -----------------------------------------------------------------------------
   vmx_on()
	Enter VMX root operation on this CPU.
   -------------------------------------------------------------------------- */
static void
vmx_on(void *arg __unused)
{
	vmx_cpu_t *cpu = &current_cpu_datap()->cpu_vmx;
	addr64_t vmxon_region_paddr;
	int result;

	vmx_init();
	
	assert(cpu->specs.vmx_present);

	if (NULL == cpu->vmxon_region)
		panic("vmx_on: VMXON region not allocated");
	vmxon_region_paddr = vmx_paddr(cpu->vmxon_region);

	/*
	 * Enable VMX operation.
	 */
	set_cr4(get_cr4() | CR4_VMXE);
	
	assert(vmx_is_cr0_valid(&cpu->specs));
	assert(vmx_is_cr4_valid(&cpu->specs));
	
#if defined(__i386__)
	if (!cpu_mode_is64bit())
		result = VMX_FAIL_INVALID; /* Not supported in legacy mode */
	else
#endif
	result = __vmxon(vmxon_region_paddr);

	if (result != VMX_SUCCEED) {
		panic("vmx_on: unexpected return %d from __vmxon()", result);
	}
}

/* -----------------------------------------------------------------------------
   vmx_off()
	Leave VMX root operation on this CPU.
   -------------------------------------------------------------------------- */
static void
vmx_off(void *arg __unused)
{
	int result;
	
	/* Tell the CPU to release the VMXON region */
#if defined(__i386__)
	if (!cpu_mode_is64bit())
		result = VMX_FAIL_INVALID; /* Not supported in legacy mode */
	else
#endif
	result = __vmxoff();

	if (result != VMX_SUCCEED) {
		panic("vmx_off: unexpected return %d from __vmxoff()", result);
	}
}

/* -----------------------------------------------------------------------------
   vmx_allocate_vmxon_regions()
	Allocate, clear and init VMXON regions for all CPUs.
   -------------------------------------------------------------------------- */
static void
vmx_allocate_vmxon_regions(void)
{
	unsigned int i;
	
	for (i=0; i<real_ncpus; i++) {
		vmx_cpu_t *cpu = &cpu_datap(i)->cpu_vmx;

		/* The size is defined to be always <= 4K, so we just allocate a page */
		cpu->vmxon_region = vmx_pcalloc();
		if (NULL == cpu->vmxon_region)
			panic("vmx_allocate_vmxon_regions: unable to allocate VMXON region");
		*(uint32_t*)(cpu->vmxon_region) = cpu->specs.vmcs_id;
	}
}

/* -----------------------------------------------------------------------------
   vmx_free_vmxon_regions()
	Free VMXON regions for all CPUs.
   -------------------------------------------------------------------------- */
static void
vmx_free_vmxon_regions(void)
{
	unsigned int i;

	for (i=0; i<real_ncpus; i++) {
		vmx_cpu_t *cpu = &cpu_datap(i)->cpu_vmx;

		vmx_pfree(cpu->vmxon_region);
		cpu->vmxon_region = NULL;
	}
}

/* -----------------------------------------------------------------------------
   vmx_globally_available()
	Checks whether VT can be turned on for all CPUs.
   -------------------------------------------------------------------------- */
static boolean_t
vmx_globally_available(void)
{
	unsigned int i;
	
	boolean_t available = TRUE;

	for (i=0; i<real_ncpus; i++) {
		vmx_cpu_t *cpu = &cpu_datap(i)->cpu_vmx;

		if (!cpu->specs.vmx_present)
			available = FALSE;
	}
	VMX_KPRINTF("VMX available: %d\n", available);
	return available;
}


/* -----------------------------------------------------------------------------
   vmx_turn_on()
	Turn on VT operation on all CPUs.
   -------------------------------------------------------------------------- */
int
host_vmxon(boolean_t exclusive)
{
	int error;
	boolean_t do_it = FALSE; /* do the cpu sync outside of the area holding the lock */

	if (!vmx_globally_available())
		return VMX_UNSUPPORTED;

	simple_lock(&vmx_use_count_lock);

	if (vmx_exclusive) {
		error = VMX_INUSE;
	} else {
		vmx_use_count++;
		if (vmx_use_count == 1) /* was turned off before */
			do_it = TRUE;
		vmx_exclusive = exclusive;

		VMX_KPRINTF("VMX use count: %d\n", vmx_use_count);
		error = VMX_OK;
	}

	simple_unlock(&vmx_use_count_lock);

	if (do_it) {
		vmx_allocate_vmxon_regions();
		mp_rendezvous(NULL, vmx_on, NULL, NULL);
	}
	return error;
}

/* -----------------------------------------------------------------------------
   vmx_turn_off()
	Turn off VT operation on all CPUs.
   -------------------------------------------------------------------------- */
void
host_vmxoff()
{
	boolean_t do_it = FALSE; /* do the cpu sync outside of the area holding the lock */

	simple_lock(&vmx_use_count_lock);

	if (vmx_use_count) {
		vmx_use_count--;
		vmx_exclusive = FALSE;
		if (!vmx_use_count)
			do_it = TRUE;
	}

	simple_unlock(&vmx_use_count_lock);

	if (do_it) {
		mp_rendezvous(NULL, vmx_off, NULL, NULL);
		vmx_free_vmxon_regions();
	}

	VMX_KPRINTF("VMX use count: %d\n", vmx_use_count);
}

/* -----------------------------------------------------------------------------
   vmx_suspend()
	Turn off VT operation on this CPU if it was on.
	Called when a CPU goes offline.
   -------------------------------------------------------------------------- */
void
vmx_suspend()
{
	VMX_KPRINTF("vmx_suspend\n");
	if (vmx_use_count)
		vmx_off(NULL);
}

/* -----------------------------------------------------------------------------
   vmx_suspend()
	Restore the previous VT state. Called when CPU comes back online.
   -------------------------------------------------------------------------- */
void
vmx_resume()
{
	VMX_KPRINTF("vmx_resume\n");
	vmx_init(); /* init VMX on CPU #0 */
	if (vmx_use_count)
		vmx_on(NULL);
}
Commit	Line	Data
2d21ac55 A	1	/*
	2	* Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	#include <pexpert/pexpert.h>
	30	#include <i386/cpuid.h>
	31	#include <i386/cpu_data.h>
b0d623f7	32	#include <i386/mp.h>
2d21ac55 A	33	#include <i386/proc_reg.h>
	34	#include <i386/vmx.h>
	35	#include <i386/vmx/vmx_asm.h>
	36	#include <i386/vmx/vmx_shims.h>
	37	#include <i386/vmx/vmx_cpu.h>
2d21ac55	38	#include <mach/mach_host.h> /* for host_info() */
2d21ac55 A	39
	40	#define VMX_KPRINTF(x...) /* kprintf("vmx: " x) */
	41
	42	int vmx_use_count = 0;
b0d623f7	43	boolean_t vmx_exclusive = FALSE;
2d21ac55 A	44	decl_simple_lock_data(static,vmx_use_count_lock)
	45
	46	/* -----------------------------------------------------------------------------
	47	vmx_is_available()
	48	Is the VMX facility available on this CPU?
	49	-------------------------------------------------------------------------- */
	50	static inline boolean_t
	51	vmx_is_available(void)
	52	{
	53	return (0 != (cpuid_features() & CPUID_FEATURE_VMX));
	54	}
	55
	56	/* -----------------------------------------------------------------------------
	57	vmxon_is_enabled()
	58	Is the VMXON instruction enabled on this CPU?
	59	-------------------------------------------------------------------------- */
	60	static inline boolean_t
	61	vmxon_is_enabled(void)
	62	{
	63	return (vmx_is_available() &&
	64	(rdmsr64(MSR_IA32_FEATURE_CONTROL) & MSR_IA32_FEATCTL_VMXON));
	65	}
	66
	67	/* -----------------------------------------------------------------------------
	68	vmx_is_cr0_valid()
	69	Is CR0 valid for executing VMXON on this CPU?
	70	-------------------------------------------------------------------------- */
	71	static inline boolean_t
	72	vmx_is_cr0_valid(vmx_specs_t *specs)
	73	{
b0d623f7	74	uintptr_t cr0 = get_cr0();
2d21ac55 A	75	return (0 == ((~cr0 & specs->cr0_fixed_0)\|(cr0 & ~specs->cr0_fixed_1)));
	76	}
	77
	78	/* -----------------------------------------------------------------------------
	79	vmx_is_cr4_valid()
	80	Is CR4 valid for executing VMXON on this CPU?
	81	-------------------------------------------------------------------------- */
	82	static inline boolean_t
	83	vmx_is_cr4_valid(vmx_specs_t *specs)
	84	{
b0d623f7	85	uintptr_t cr4 = get_cr4();
2d21ac55 A	86	return (0 == ((~cr4 & specs->cr4_fixed_0)\|(cr4 & ~specs->cr4_fixed_1)));
	87	}
	88
	89	static void
	90	vmx_init(void)
	91	{
	92	uint64_t msr_image;
	93
	94	if (!vmx_is_available())
	95	return;
	96
	97	/*
	98	* We don't count on EFI initializing MSR_IA32_FEATURE_CONTROL
	99	* and turning VMXON on and locking the bit, so we do that now.
	100	*/
	101	msr_image = rdmsr64(MSR_IA32_FEATURE_CONTROL);
	102	if (0 == ((msr_image & MSR_IA32_FEATCTL_LOCK)))
	103	wrmsr64(MSR_IA32_FEATURE_CONTROL,
	104	(msr_image \|
	105	MSR_IA32_FEATCTL_VMXON \|
	106	MSR_IA32_FEATCTL_LOCK));
	107	}
	108
	109	/* -----------------------------------------------------------------------------
	110	vmx_get_specs()
	111	Obtain VMX facility specifications for this CPU and
	112	enter them into the vmx_specs_t structure. If VMX is not available or
	113	disabled on this CPU, set vmx_present to false and return leaving
	114	the remainder of the vmx_specs_t uninitialized.
	115	-------------------------------------------------------------------------- */
	116	void
	117	vmx_get_specs()
	118	{
	119	vmx_specs_t *specs = &current_cpu_datap()->cpu_vmx.specs;
	120	uint64_t msr_image;
	121
	122	/* this is called once for every CPU, but the lock doesn't care :-) */
	123	simple_lock_init(&vmx_use_count_lock, 0);
	124
	125	vmx_init();
	126
	127	/*
	128	* if we have read the data on boot, we won't read it
	129	* again on wakeup, otherwise bad things will happen
	130	*/
	131	if (specs->initialized)
	132	return;
	133	else
	134	specs->initialized = TRUE;
	135
	136	/* See if VMX is present, return if it is not */
	137	specs->vmx_present = vmx_is_available() && vmxon_is_enabled();
	138	if (!specs->vmx_present)
	139	return;
	140
	141	#define bitfield(x,f) ((x >> f##_BIT) & f##_MASK)
	142	/* Obtain and decode VMX general capabilities */
	143	msr_image = rdmsr64(MSR_IA32_VMX_BASIC);
b0d623f7	144	specs->vmcs_id = (uint32_t)(msr_image & VMX_VCR_VMCS_REV_ID);
2d21ac55 A	145	specs->vmcs_mem_type = bitfield(msr_image, VMX_VCR_VMCS_MEM_TYPE) != 0;
	146	specs->vmcs_size = bitfield(msr_image, VMX_VCR_VMCS_SIZE);
	147
	148	/* Obtain allowed settings for pin-based execution controls */
	149	msr_image = rdmsr64(MSR_IA32_VMXPINBASED_CTLS);
b0d623f7 A	150	specs->pin_exctls_0 = (uint32_t)(msr_image & 0xFFFFFFFF);
b0d623f7 A	151	specs->pin_exctls_1 = (uint32_t)(msr_image >> 32);
2d21ac55 A	152
	153	/* Obtain allowed settings for processor-based execution controls */
	154	msr_image = rdmsr64(MSR_IA32_PROCBASED_CTLS);
b0d623f7 A	155	specs->proc_exctls_0 = (uint32_t)(msr_image & 0xFFFFFFFF);
b0d623f7 A	156	specs->proc_exctls_1 = (uint32_t)(msr_image >> 32);
2d21ac55 A	157
	158	/* Obtain allowed settings for VM-exit controls */
	159	msr_image = rdmsr64(MSR_IA32_VMX_EXIT_CTLS);
b0d623f7 A	160	specs->exit_ctls_0 = (uint32_t)(msr_image & 0xFFFFFFFF);
b0d623f7 A	161	specs->exit_ctls_1 = (uint32_t)(msr_image >> 32);
2d21ac55 A	162
	163	/* Obtain allowed settings for VM-entry controls */
	164	msr_image = rdmsr64(MSR_IA32_VMX_ENTRY_CTLS);
b0d623f7 A	165	specs->enter_ctls_0 = (uint32_t)(msr_image & 0xFFFFFFFF);
b0d623f7 A	166	specs->enter_ctls_0 = (uint32_t)(msr_image >> 32);
2d21ac55 A	167
	168	/* Obtain and decode miscellaneous capabilities */
	169	msr_image = rdmsr64(MSR_IA32_VMX_MISC);
	170	specs->act_halt = bitfield(msr_image, VMX_VCR_ACT_HLT) != 0;
	171	specs->act_shutdown = bitfield(msr_image, VMX_VCR_ACT_SHUTDOWN) != 0;
	172	specs->act_SIPI = bitfield(msr_image, VMX_VCR_ACT_SIPI) != 0;
	173	specs->act_CSTATE = bitfield(msr_image, VMX_VCR_ACT_CSTATE) != 0;
	174	specs->cr3_targs = bitfield(msr_image, VMX_VCR_CR3_TARGS);
b0d623f7 A	175	specs->max_msrs = (uint32_t)(512 * (1 + bitfield(msr_image, VMX_VCR_MAX_MSRS)));
b0d623f7 A	176	specs->mseg_id = (uint32_t)bitfield(msr_image, VMX_VCR_MSEG_ID);
2d21ac55 A	177
2d21ac55 A	178	/* Obtain VMX-fixed bits in CR0 */
b0d623f7 A	179	specs->cr0_fixed_0 = (uint32_t)rdmsr64(MSR_IA32_VMX_CR0_FIXED0) & 0xFFFFFFFF;
b0d623f7 A	180	specs->cr0_fixed_1 = (uint32_t)rdmsr64(MSR_IA32_VMX_CR0_FIXED1) & 0xFFFFFFFF;
2d21ac55 A	181
2d21ac55 A	182	/* Obtain VMX-fixed bits in CR4 */
b0d623f7 A	183	specs->cr4_fixed_0 = (uint32_t)rdmsr64(MSR_IA32_VMX_CR4_FIXED0) & 0xFFFFFFFF;
b0d623f7 A	184	specs->cr4_fixed_1 = (uint32_t)rdmsr64(MSR_IA32_VMX_CR4_FIXED1) & 0xFFFFFFFF;
2d21ac55 A	185	}
	186
	187	/* -----------------------------------------------------------------------------
	188	vmx_on()
	189	Enter VMX root operation on this CPU.
	190	-------------------------------------------------------------------------- */
	191	static void
6d2010ae	192	vmx_on(void *arg __unused)
2d21ac55 A	193	{
	194	vmx_cpu_t *cpu = &current_cpu_datap()->cpu_vmx;
	195	addr64_t vmxon_region_paddr;
	196	int result;
	197
	198	vmx_init();
	199
	200	assert(cpu->specs.vmx_present);
	201
	202	if (NULL == cpu->vmxon_region)
	203	panic("vmx_on: VMXON region not allocated");
	204	vmxon_region_paddr = vmx_paddr(cpu->vmxon_region);
	205
	206	/*
	207	* Enable VMX operation.
	208	*/
	209	set_cr4(get_cr4() \| CR4_VMXE);
	210
	211	assert(vmx_is_cr0_valid(&cpu->specs));
	212	assert(vmx_is_cr4_valid(&cpu->specs));
	213
316670eb A	214	#if defined(__i386__)
	215	if (!cpu_mode_is64bit())
	216	result = VMX_FAIL_INVALID; /* Not supported in legacy mode */
	217	else
	218	#endif
	219	result = __vmxon(vmxon_region_paddr);
	220
	221	if (result != VMX_SUCCEED) {
2d21ac55 A	222	panic("vmx_on: unexpected return %d from __vmxon()", result);
	223	}
	224	}
	225
	226	/* -----------------------------------------------------------------------------
	227	vmx_off()
	228	Leave VMX root operation on this CPU.
	229	-------------------------------------------------------------------------- */
	230	static void
6d2010ae	231	vmx_off(void *arg __unused)
2d21ac55 A	232	{
	233	int result;
	234
	235	/* Tell the CPU to release the VMXON region */
316670eb A	236	#if defined(__i386__)
	237	if (!cpu_mode_is64bit())
	238	result = VMX_FAIL_INVALID; /* Not supported in legacy mode */
	239	else
	240	#endif
	241	result = __vmxoff();
	242
	243	if (result != VMX_SUCCEED) {
2d21ac55 A	244	panic("vmx_off: unexpected return %d from __vmxoff()", result);
	245	}
	246	}
	247
	248	/* -----------------------------------------------------------------------------
	249	vmx_allocate_vmxon_regions()
	250	Allocate, clear and init VMXON regions for all CPUs.
	251	-------------------------------------------------------------------------- */
	252	static void
	253	vmx_allocate_vmxon_regions(void)
	254	{
	255	unsigned int i;
	256
	257	for (i=0; i<real_ncpus; i++) {
	258	vmx_cpu_t *cpu = &cpu_datap(i)->cpu_vmx;
	259
	260	/* The size is defined to be always <= 4K, so we just allocate a page */
	261	cpu->vmxon_region = vmx_pcalloc();
	262	if (NULL == cpu->vmxon_region)
	263	panic("vmx_allocate_vmxon_regions: unable to allocate VMXON region");
	264	(uint32_t)(cpu->vmxon_region) = cpu->specs.vmcs_id;
	265	}
	266	}
	267
	268	/* -----------------------------------------------------------------------------
	269	vmx_free_vmxon_regions()
	270	Free VMXON regions for all CPUs.
	271	-------------------------------------------------------------------------- */
	272	static void
	273	vmx_free_vmxon_regions(void)
	274	{
	275	unsigned int i;
	276
	277	for (i=0; i<real_ncpus; i++) {
	278	vmx_cpu_t *cpu = &cpu_datap(i)->cpu_vmx;
	279
	280	vmx_pfree(cpu->vmxon_region);
	281	cpu->vmxon_region = NULL;
	282	}
	283	}
	284
	285	/* -----------------------------------------------------------------------------
	286	vmx_globally_available()
	287	Checks whether VT can be turned on for all CPUs.
	288	-------------------------------------------------------------------------- */
	289	static boolean_t
	290	vmx_globally_available(void)
	291	{
	292	unsigned int i;
	293
	294	boolean_t available = TRUE;
	295
	296	for (i=0; i<real_ncpus; i++) {
	297	vmx_cpu_t *cpu = &cpu_datap(i)->cpu_vmx;
	298
	299	if (!cpu->specs.vmx_present)
	300	available = FALSE;
	301	}
	302	VMX_KPRINTF("VMX available: %d\n", available);
	303	return available;
	304	}
	305
	306
	307	/* -----------------------------------------------------------------------------
308	vmx_turn_on()
309	Turn on VT operation on all CPUs.
310	-------------------------------------------------------------------------- */
311	int
312	host_vmxon(boolean_t exclusive)
313	{
314	int error;
b0d623f7	315	boolean_t do_it = FALSE; /* do the cpu sync outside of the area holding the lock */
2d21ac55 A	316
	317	if (!vmx_globally_available())
	318	return VMX_UNSUPPORTED;
	319
	320	simple_lock(&vmx_use_count_lock);
	321
	322	if (vmx_exclusive) {
	323	error = VMX_INUSE;
b0d623f7 A	324	} else {
	325	vmx_use_count++;
	326	if (vmx_use_count == 1) /* was turned off before */
	327	do_it = TRUE;
	328	vmx_exclusive = exclusive;
	329
	330	VMX_KPRINTF("VMX use count: %d\n", vmx_use_count);
	331	error = VMX_OK;
2d21ac55	332	}
2d21ac55	333
2d21ac55 A	334	simple_unlock(&vmx_use_count_lock);
2d21ac55 A	335
b0d623f7 A	336	if (do_it) {
b0d623f7 A	337	vmx_allocate_vmxon_regions();
6d2010ae	338	mp_rendezvous(NULL, vmx_on, NULL, NULL);
b0d623f7	339	}
2d21ac55 A	340	return error;
	341	}
	342
	343	/* -----------------------------------------------------------------------------
	344	vmx_turn_off()
	345	Turn off VT operation on all CPUs.
	346	-------------------------------------------------------------------------- */
	347	void
	348	host_vmxoff()
	349	{
b0d623f7 A	350	boolean_t do_it = FALSE; /* do the cpu sync outside of the area holding the lock */
b0d623f7 A	351
2d21ac55 A	352	simple_lock(&vmx_use_count_lock);
	353
	354	if (vmx_use_count) {
	355	vmx_use_count--;
b0d623f7 A	356	vmx_exclusive = FALSE;
	357	if (!vmx_use_count)
	358	do_it = TRUE;
2d21ac55 A	359	}
	360
	361	simple_unlock(&vmx_use_count_lock);
	362
b0d623f7	363	if (do_it) {
6d2010ae	364	mp_rendezvous(NULL, vmx_off, NULL, NULL);
b0d623f7 A	365	vmx_free_vmxon_regions();
	366	}
	367
2d21ac55 A	368	VMX_KPRINTF("VMX use count: %d\n", vmx_use_count);
	369	}
	370
	371	/* -----------------------------------------------------------------------------
	372	vmx_suspend()
	373	Turn off VT operation on this CPU if it was on.
	374	Called when a CPU goes offline.
	375	-------------------------------------------------------------------------- */
	376	void
	377	vmx_suspend()
	378	{
	379	VMX_KPRINTF("vmx_suspend\n");
	380	if (vmx_use_count)
6d2010ae	381	vmx_off(NULL);
2d21ac55 A	382	}
	383
	384	/* -----------------------------------------------------------------------------
	385	vmx_suspend()
	386	Restore the previous VT state. Called when CPU comes back online.
	387	-------------------------------------------------------------------------- */
	388	void
	389	vmx_resume()
	390	{
	391	VMX_KPRINTF("vmx_resume\n");
c910b4d9	392	vmx_init(); /* init VMX on CPU #0 */
2d21ac55	393	if (vmx_use_count)
6d2010ae	394	vmx_on(NULL);
2d21ac55	395	}