[apple/xnu.git] / osfmk / i386 / perfmon.c

/*
 * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <mach/std_types.h>
#include <i386/cpu_data.h>
#include <i386/cpu_number.h>
#include <i386/perfmon.h>
#include <i386/proc_reg.h>
#include <i386/cpu_threads.h>
#include <i386/mp.h>
#include <i386/cpuid.h>
#include <i386/lock.h>
#include <vm/vm_kern.h>
#include <kern/task.h>

#if DEBUG
#define	DBG(x...)	kprintf(x)
#else
#define	DBG(x...)
#endif

decl_simple_lock_data(,pmc_lock)
static task_t	pmc_owner = TASK_NULL;
static int	pmc_thread_count = 0;

/* PMC Facility Owner:
 * TASK_NULL - no one owns it
 * kernel_task - owned by pmc
 * other task - owned by another task
 */

/*
 * Table of ESCRs and addresses associated with performance counters/CCCRs.
 * See Intel SDM Vol 3, Table 15-4 (section 15.9):
 */
static uint16_t pmc_escr_addr_table[18][8] = {
	[MSR_BPU_COUNTER0] {
		[MSR_BSU_ESCR0]		0x3a0,
		[MSR_FSB_ESCR0]		0x3a2,
		[MSR_MOB_ESCR0]		0x3aa,
		[MSR_PMH_ESCR0]		0x3ac,
		[MSR_BPU_ESCR0]		0x3b2,
		[MSR_IS_ESCR0]		0x3b4,
		[MSR_ITLB_ESCR0]	0x3b6,
		[MSR_IX_ESCR0]		0x3c8,
	},
	[MSR_BPU_COUNTER1] {
		[MSR_BSU_ESCR0]		0x3a0,
		[MSR_FSB_ESCR0]		0x3a2,
		[MSR_MOB_ESCR0]		0x3aa,
		[MSR_PMH_ESCR0]		0x3ac,
		[MSR_BPU_ESCR0]		0x3b2,
		[MSR_IS_ESCR0]		0x3b4,
		[MSR_ITLB_ESCR0]	0x3b6,
		[MSR_IX_ESCR0]		0x3c8,
	},
	[MSR_BPU_COUNTER2] {
		[MSR_BSU_ESCR1]		0x3a1,
		[MSR_FSB_ESCR1]		0x3a3,
		[MSR_MOB_ESCR1]		0x3ab,
		[MSR_PMH_ESCR1]		0x3ad,
		[MSR_BPU_ESCR1]		0x3b3,
		[MSR_IS_ESCR1]		0x3b5,
		[MSR_ITLB_ESCR1]	0x3b7,
		[MSR_IX_ESCR1]		0x3c9,
	},
	[MSR_BPU_COUNTER3] {
		[MSR_BSU_ESCR1]		0x3a1,
		[MSR_FSB_ESCR1]		0x3a3,
		[MSR_MOB_ESCR1]		0x3ab,
		[MSR_PMH_ESCR1]		0x3ad,
		[MSR_BPU_ESCR1]		0x3b3,
		[MSR_IS_ESCR1]		0x3b5,
		[MSR_ITLB_ESCR1]	0x3b7,
		[MSR_IX_ESCR1]		0x3c9,
	},
	[MSR_MS_COUNTER0] {
		[MSR_MS_ESCR1]		0x3c1,
		[MSR_TBPU_ESCR1]	0x3c3,
		[MSR_TC_ESCR1]		0x3c5,
	},
	[MSR_MS_COUNTER1] {
		[MSR_MS_ESCR1]		0x3c1,
		[MSR_TBPU_ESCR1]	0x3c3,
		[MSR_TC_ESCR1]		0x3c5,
	},
	[MSR_MS_COUNTER2] {
		[MSR_MS_ESCR1]		0x3c1,
		[MSR_TBPU_ESCR1]	0x3c3,
		[MSR_TC_ESCR1]		0x3c5,
	},
	[MSR_MS_COUNTER3] {
		[MSR_MS_ESCR1]		0x3c1,
		[MSR_TBPU_ESCR1]	0x3c3,
		[MSR_TC_ESCR1]		0x3c5,
	},
	[MSR_FLAME_COUNTER0] {
		[MSR_FIRM_ESCR0]	0x3a4,
		[MSR_FLAME_ESCR0]	0x3a6,
		[MSR_DAC_ESCR0]		0x3a8,
		[MSR_SAT_ESCR0]		0x3ae,
		[MSR_U2L_ESCR0]		0x3b0,
	},
	[MSR_FLAME_COUNTER1] {
		[MSR_FIRM_ESCR0]	0x3a4,
		[MSR_FLAME_ESCR0]	0x3a6,
		[MSR_DAC_ESCR0]		0x3a8,
		[MSR_SAT_ESCR0]		0x3ae,
		[MSR_U2L_ESCR0]		0x3b0,
	},
	[MSR_FLAME_COUNTER2] {
		[MSR_FIRM_ESCR1]	0x3a5,
		[MSR_FLAME_ESCR1]	0x3a7,
		[MSR_DAC_ESCR1]		0x3a9,
		[MSR_SAT_ESCR1]		0x3af,
		[MSR_U2L_ESCR1]		0x3b1,
	},
	[MSR_FLAME_COUNTER3] {
		[MSR_FIRM_ESCR1]	0x3a5,
		[MSR_FLAME_ESCR1]	0x3a7,
		[MSR_DAC_ESCR1]		0x3a9,
		[MSR_SAT_ESCR1]		0x3af,
		[MSR_U2L_ESCR1]		0x3b1,
	},
	[MSR_IQ_COUNTER0] {
		[MSR_CRU_ESCR0]		0x3b8,
		[MSR_CRU_ESCR2]		0x3cc,
		[MSR_CRU_ESCR4]		0x3e0,
		[MSR_IQ_ESCR0]		0x3ba,
		[MSR_RAT_ESCR0]		0x3bc,
		[MSR_SSU_ESCR0]		0x3be,
		[MSR_AFL_ESCR0]		0x3ca,
	},
	[MSR_IQ_COUNTER1] {
		[MSR_CRU_ESCR0]		0x3b8,
		[MSR_CRU_ESCR2]		0x3cc,
		[MSR_CRU_ESCR4]		0x3e0,
		[MSR_IQ_ESCR0]		0x3ba,
		[MSR_RAT_ESCR0]		0x3bc,
		[MSR_SSU_ESCR0]		0x3be,
		[MSR_AFL_ESCR0]		0x3ca,
	},
	[MSR_IQ_COUNTER2] {
		[MSR_CRU_ESCR1]		0x3b9,
		[MSR_CRU_ESCR3]		0x3cd,
		[MSR_CRU_ESCR5]		0x3e1,
		[MSR_IQ_ESCR1]		0x3bb,
		[MSR_RAT_ESCR1]		0x3bd,
		[MSR_AFL_ESCR1]		0x3cb,
	},
	[MSR_IQ_COUNTER3] {
		[MSR_CRU_ESCR1]		0x3b9,
		[MSR_CRU_ESCR3]		0x3cd,
		[MSR_CRU_ESCR5]		0x3e1,
		[MSR_IQ_ESCR1]		0x3bb,
		[MSR_RAT_ESCR1]		0x3bd,
		[MSR_AFL_ESCR1]		0x3cb,
	},
	[MSR_IQ_COUNTER4] {
		[MSR_CRU_ESCR0]		0x3b8,
		[MSR_CRU_ESCR2]		0x3cc,
		[MSR_CRU_ESCR4]		0x3e0,
		[MSR_IQ_ESCR0]		0x3ba,
		[MSR_RAT_ESCR0]		0x3bc,
		[MSR_SSU_ESCR0]		0x3be,
		[MSR_AFL_ESCR0]		0x3ca,
	},
	[MSR_IQ_COUNTER5] {
		[MSR_CRU_ESCR1]		0x3b9,
		[MSR_CRU_ESCR3]		0x3cd,
		[MSR_CRU_ESCR5]		0x3e1,
		[MSR_IQ_ESCR1]		0x3bb,
		[MSR_RAT_ESCR1]		0x3bd,
		[MSR_AFL_ESCR1]		0x3cb,
	},
};
#define PMC_ESCR_ADDR(id,esid)	pmc_escr_addr_table[id][esid]

typedef struct {
	pmc_id_t	id_max;			/* Maximum counter id */
	pmc_machine_t	machine_type;		/* P6 or P4/Xeon */
	uint32_t	msr_counter_base;	/* First counter MSR */
	uint32_t	msr_control_base;	/* First control MSR */
	boolean_t	reserved[18];		/* Max-sized arrays... */
	pmc_ovf_func_t	*ovf_func[18];
#ifdef DEBUG
	pmc_cccr_t	cccr_shadow[18];	/* Last cccr values set */
	pmc_counter_t	counter_shadow[18];	/* Last counter values set */
	uint32_t	ovfs_unexpected[18];	/* Count of unexpected intrs */
#endif
} pmc_table_t;

static pmc_machine_t
_pmc_machine_type(void)
{
	i386_cpu_info_t	*infop = cpuid_info();
        
	if (strncmp(infop->cpuid_vendor, CPUID_VID_INTEL, sizeof(CPUID_VID_INTEL)) != 0)
		return pmc_none;
	
	if (!pmc_is_available())
		return pmc_none;

	switch (infop->cpuid_family) {
	case 0x6:
		return pmc_P6;
	case 0xf:
		return pmc_P4_Xeon;
	default:
		return pmc_unknown;
	}
}

static void
pmc_p4_intr(void *state)
{
	pmc_table_t	*pmc_table = (pmc_table_t *) cpu_core()->pmc;
	uint32_t	cccr_addr;
	pmc_cccr_t	cccr;
	pmc_id_t	id;
	int		my_logical_cpu = cpu_to_logical_cpu(cpu_number());

	/*
	 * Scan through table for reserved counters with overflow and
	 * with a registered overflow function.
	 */
	for (id = 0; id <= pmc_table->id_max; id++) {
		if (!pmc_table->reserved[id])
			continue;
		cccr_addr = pmc_table->msr_control_base + id;
		cccr.u_u64 = rdmsr64(cccr_addr);
#ifdef DEBUG
		pmc_table->cccr_shadow[id] = cccr;
		*((uint64_t *) &pmc_table->counter_shadow[id]) =
			rdmsr64(pmc_table->msr_counter_base + id);
#endif
		if (cccr.u_htt.ovf == 0)
			continue;
		if ((cccr.u_htt.ovf_pmi_t0 == 1 && my_logical_cpu == 0) ||
		    (cccr.u_htt.ovf_pmi_t1 == 1 && my_logical_cpu == 1)) {
			if (pmc_table->ovf_func[id]) {
				(*pmc_table->ovf_func[id])(id, state);
				/* func expected to clear overflow */
				continue;
			}
		}
		/* Clear overflow for unexpected interrupt */
#ifdef DEBUG
		pmc_table->ovfs_unexpected[id]++;
#endif
	}
}

static void
pmc_p6_intr(void *state)
{
	pmc_table_t	*pmc_table = (pmc_table_t *) cpu_core()->pmc;
	pmc_id_t	id;

	/*
	 * Can't determine which counter has overflow
	 * so call all registered functions.
	 */
	for (id = 0; id <= pmc_table->id_max; id++)
		if (pmc_table->reserved[id] && pmc_table->ovf_func[id])
			(*pmc_table->ovf_func[id])(id, state);
}

void *
pmc_alloc(void)
{
	int		ret;
	pmc_table_t	*pmc_table;
	pmc_machine_t	pmc_type;

	pmc_type = _pmc_machine_type();
	if (pmc_type == pmc_none) {
		return NULL;
	}
	
		ret = kmem_alloc(kernel_map,
			(void *) &pmc_table, sizeof(pmc_table_t));
		if (ret != KERN_SUCCESS)
			panic("pmc_init() kmem_alloc returned %d\n", ret);
		bzero((void *)pmc_table, sizeof(pmc_table_t));

		pmc_table->machine_type = pmc_type;
		switch (pmc_type) {
		case pmc_P4_Xeon:
			pmc_table->id_max = 17;
			pmc_table->msr_counter_base = MSR_COUNTER_ADDR(0);
			pmc_table->msr_control_base = MSR_CCCR_ADDR(0);
			lapic_set_pmi_func(&pmc_p4_intr);
			break;
		case pmc_P6:
			pmc_table->id_max = 1;
			pmc_table->msr_counter_base = MSR_P6_COUNTER_ADDR(0);
			pmc_table->msr_control_base = MSR_P6_PES_ADDR(0);
			lapic_set_pmi_func(&pmc_p6_intr);
			break;
		default:
			break;
		}
	return (void *) pmc_table;
}


static inline pmc_table_t *
pmc_table_valid(pmc_id_t id)
{
	cpu_core_t	*my_core = cpu_core();
	pmc_table_t	*pmc_table;

	assert(my_core);
	
	pmc_table = (pmc_table_t *) my_core->pmc;
	return (pmc_table == NULL ||
		id > pmc_table->id_max ||
		!pmc_table->reserved[id]) ? NULL : pmc_table;
}

int
pmc_machine_type(pmc_machine_t *type)
{
	cpu_core_t	*my_core = cpu_core();
	pmc_table_t	*pmc_table;

	assert(my_core);

	pmc_table = (pmc_table_t *) my_core->pmc;
	if (pmc_table == NULL)
		return KERN_FAILURE;

	*type = pmc_table->machine_type;
	
	return KERN_SUCCESS;
}

int
pmc_reserve(pmc_id_t id)
{
	cpu_core_t	*my_core = cpu_core();
	pmc_table_t	*pmc_table;

	assert(my_core);

	pmc_table = (pmc_table_t *) my_core->pmc;
	if (pmc_table == NULL)
		return KERN_FAILURE;
	if (id > pmc_table->id_max)
		return KERN_INVALID_ARGUMENT;
	if (pmc_table->reserved[id])
		return KERN_FAILURE;

	pmc_table->reserved[id] = TRUE;

	return KERN_SUCCESS;
}

boolean_t
pmc_is_reserved(pmc_id_t id)
{
	return pmc_table_valid(id) != NULL;
}

int
pmc_free(pmc_id_t id)
{
	pmc_table_t	*pmc_table = pmc_table_valid(id);

	if (pmc_table == NULL)
		return KERN_INVALID_ARGUMENT;

	pmc_cccr_write(id, 0x0ULL);
	pmc_table->reserved[id] = FALSE;
	pmc_table->ovf_func[id] = NULL;

	return KERN_SUCCESS;
}

int
pmc_counter_read(pmc_id_t id, pmc_counter_t *val)
{
	pmc_table_t	*pmc_table = pmc_table_valid(id);

	if (pmc_table == NULL)
		return KERN_INVALID_ARGUMENT;
	
	*(uint64_t *)val = rdmsr64(pmc_table->msr_counter_base + id);

	return KERN_SUCCESS;
}

int
pmc_counter_write(pmc_id_t id, pmc_counter_t *val)
{
	pmc_table_t	*pmc_table = pmc_table_valid(id);

	if (pmc_table == NULL)
		return KERN_INVALID_ARGUMENT;
	
	wrmsr64(pmc_table->msr_counter_base + id, *(uint64_t *)val);

	return KERN_SUCCESS;
}

int
pmc_cccr_read(pmc_id_t id, pmc_cccr_t *cccr)
{
	pmc_table_t	*pmc_table = pmc_table_valid(id);

	if (pmc_table == NULL)
		return KERN_INVALID_ARGUMENT;
	
	if (pmc_table->machine_type != pmc_P4_Xeon)
		return KERN_FAILURE;
	
	*(uint64_t *)cccr = rdmsr64(pmc_table->msr_control_base + id);

	return KERN_SUCCESS;
}

int
pmc_cccr_write(pmc_id_t id, pmc_cccr_t *cccr)
{
	pmc_table_t	*pmc_table = pmc_table_valid(id);

	if (pmc_table == NULL)
		return KERN_INVALID_ARGUMENT;

	if (pmc_table->machine_type != pmc_P4_Xeon)
		return KERN_FAILURE;
	
	wrmsr64(pmc_table->msr_control_base + id, *(uint64_t *)cccr);

	return KERN_SUCCESS;
}

int
pmc_evtsel_read(pmc_id_t id, pmc_evtsel_t *evtsel)
{
	pmc_table_t	*pmc_table = pmc_table_valid(id);

	if (pmc_table == NULL)
		return KERN_INVALID_ARGUMENT;
	
	if (pmc_table->machine_type != pmc_P6)
		return KERN_FAILURE;
	
	*(uint64_t *)evtsel = rdmsr64(pmc_table->msr_control_base + id);

	return KERN_SUCCESS;
}

int
pmc_evtsel_write(pmc_id_t id, pmc_evtsel_t *evtsel)
{
	pmc_table_t	*pmc_table = pmc_table_valid(id);

	if (pmc_table == NULL)
		return KERN_INVALID_ARGUMENT;

	if (pmc_table->machine_type != pmc_P4_Xeon)
		return KERN_FAILURE;
	
	wrmsr64(pmc_table->msr_control_base + id, *(uint64_t *)evtsel);

	return KERN_SUCCESS;
}

int
pmc_escr_read(pmc_id_t id, pmc_escr_id_t esid, pmc_escr_t *escr)
{
	uint32_t	addr;
	pmc_table_t	*pmc_table = pmc_table_valid(id);

	if (pmc_table == NULL)
		return KERN_INVALID_ARGUMENT;

	if (pmc_table->machine_type != pmc_P4_Xeon)
		return KERN_FAILURE;
	
	if (esid > PMC_ESID_MAX)
		return KERN_INVALID_ARGUMENT;

	addr = PMC_ESCR_ADDR(id, esid);
	if (addr == 0)
		return KERN_INVALID_ARGUMENT;

	*(uint64_t *)escr = rdmsr64(addr);

	return KERN_SUCCESS;
}

int
pmc_escr_write(pmc_id_t id, pmc_escr_id_t esid, pmc_escr_t *escr)
{
	uint32_t	addr;
	pmc_table_t	*pmc_table = pmc_table_valid(id);

	if (pmc_table == NULL)
		return KERN_FAILURE;

	if (pmc_table->machine_type != pmc_P4_Xeon)
		return KERN_FAILURE;
	
	if (esid > PMC_ESID_MAX)
		return KERN_INVALID_ARGUMENT;

	addr = PMC_ESCR_ADDR(id, esid);
	if (addr == 0)
		return KERN_INVALID_ARGUMENT;

	wrmsr64(addr, *(uint64_t *)escr);

	return KERN_SUCCESS;
}

int
pmc_set_ovf_func(pmc_id_t id, pmc_ovf_func_t func)
{
	pmc_table_t	*pmc_table = pmc_table_valid(id);

	if (pmc_table == NULL)
		return KERN_INVALID_ARGUMENT;

	pmc_table->ovf_func[id] = func;

	return KERN_SUCCESS;
}

int
pmc_acquire(task_t task)
{
	kern_return_t retval = KERN_SUCCESS;
  
	simple_lock(&pmc_lock);
  
	if(pmc_owner == task) {
		DBG("pmc_acquire - "
		    "ACQUIRED: already owner\n");
		retval = KERN_SUCCESS;
		/* already own it */
	} else if(pmc_owner == TASK_NULL) { /* no one owns it */
		pmc_owner = task;
		pmc_thread_count = 0;
		DBG("pmc_acquire - "
		    "ACQUIRED: no current owner - made new owner\n");
		retval = KERN_SUCCESS;
	} else { /* someone already owns it */
		if(pmc_owner == kernel_task) {
			if(pmc_thread_count == 0) {
				/* kernel owns it but no threads using it */
				pmc_owner = task;
				pmc_thread_count = 0;
				DBG("pmc_acquire - "
				    "ACQUIRED: owned by kernel, no threads\n");
				retval = KERN_SUCCESS;
			} else {
				DBG("pmc_acquire - "
				    "DENIED: owned by kernel, in use\n");
				retval = KERN_RESOURCE_SHORTAGE;
			}
		} else { /* non-kernel owner */
			DBG("pmc_acquire - "	
			    "DENIED: owned by another task\n");
			retval = KERN_RESOURCE_SHORTAGE;
		}
	}
  
	simple_unlock(&pmc_lock);
	return retval;
}

int
pmc_release(task_t task)
{
	kern_return_t retval = KERN_SUCCESS;
	task_t old_pmc_owner = pmc_owner;
  
	simple_lock(&pmc_lock);
  
	if(task != pmc_owner) {
		retval = KERN_NO_ACCESS;
	} else {
		if(old_pmc_owner == kernel_task) {
			if(pmc_thread_count>0) {
				DBG("pmc_release - "
				    "NOT RELEASED: owned by kernel, in use\n");
				retval = KERN_NO_ACCESS;
			} else {
				DBG("pmc_release - "
				    "RELEASED: was owned by kernel\n");
				pmc_owner = TASK_NULL;
				retval = KERN_SUCCESS;
			}
		} else {
			DBG("pmc_release - "
			    "RELEASED: was owned by user\n");
			pmc_owner = TASK_NULL;
			retval = KERN_SUCCESS;
		}
	}

	simple_unlock(&pmc_lock);
	return retval;
}
Commit	Line	Data
91447636 A	1	/*
	2	* Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
	3	*
8f6c56a5	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
91447636	5	*
8f6c56a5 A	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
8ad349bb	24	* limitations under the License.
8f6c56a5 A	25	*
8f6c56a5 A	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
91447636 A	27	*/
	28
	29	#include <mach/std_types.h>
	30	#include <i386/cpu_data.h>
	31	#include <i386/cpu_number.h>
	32	#include <i386/perfmon.h>
	33	#include <i386/proc_reg.h>
	34	#include <i386/cpu_threads.h>
	35	#include <i386/mp.h>
	36	#include <i386/cpuid.h>
	37	#include <i386/lock.h>
	38	#include <vm/vm_kern.h>
4452a7af	39	#include <kern/task.h>
91447636	40
4452a7af	41	#if DEBUG
91447636 A	42	#define DBG(x...) kprintf(x)
	43	#else
	44	#define DBG(x...)
	45	#endif
	46
4452a7af A	47	decl_simple_lock_data(,pmc_lock)
	48	static task_t pmc_owner = TASK_NULL;
	49	static int pmc_thread_count = 0;
	50
	51	/* PMC Facility Owner:
	52	* TASK_NULL - no one owns it
	53	* kernel_task - owned by pmc
	54	* other task - owned by another task
	55	*/
	56
91447636 A	57	/*
	58	* Table of ESCRs and addresses associated with performance counters/CCCRs.
	59	* See Intel SDM Vol 3, Table 15-4 (section 15.9):
	60	*/
	61	static uint16_t pmc_escr_addr_table[18][8] = {
	62	[MSR_BPU_COUNTER0] {
	63	[MSR_BSU_ESCR0] 0x3a0,
	64	[MSR_FSB_ESCR0] 0x3a2,
	65	[MSR_MOB_ESCR0] 0x3aa,
	66	[MSR_PMH_ESCR0] 0x3ac,
	67	[MSR_BPU_ESCR0] 0x3b2,
	68	[MSR_IS_ESCR0] 0x3b4,
	69	[MSR_ITLB_ESCR0] 0x3b6,
	70	[MSR_IX_ESCR0] 0x3c8,
	71	},
	72	[MSR_BPU_COUNTER1] {
	73	[MSR_BSU_ESCR0] 0x3a0,
	74	[MSR_FSB_ESCR0] 0x3a2,
	75	[MSR_MOB_ESCR0] 0x3aa,
	76	[MSR_PMH_ESCR0] 0x3ac,
	77	[MSR_BPU_ESCR0] 0x3b2,
	78	[MSR_IS_ESCR0] 0x3b4,
	79	[MSR_ITLB_ESCR0] 0x3b6,
	80	[MSR_IX_ESCR0] 0x3c8,
	81	},
	82	[MSR_BPU_COUNTER2] {
	83	[MSR_BSU_ESCR1] 0x3a1,
	84	[MSR_FSB_ESCR1] 0x3a3,
	85	[MSR_MOB_ESCR1] 0x3ab,
	86	[MSR_PMH_ESCR1] 0x3ad,
	87	[MSR_BPU_ESCR1] 0x3b3,
	88	[MSR_IS_ESCR1] 0x3b5,
	89	[MSR_ITLB_ESCR1] 0x3b7,
	90	[MSR_IX_ESCR1] 0x3c9,
	91	},
	92	[MSR_BPU_COUNTER3] {
	93	[MSR_BSU_ESCR1] 0x3a1,
	94	[MSR_FSB_ESCR1] 0x3a3,
	95	[MSR_MOB_ESCR1] 0x3ab,
	96	[MSR_PMH_ESCR1] 0x3ad,
	97	[MSR_BPU_ESCR1] 0x3b3,
	98	[MSR_IS_ESCR1] 0x3b5,
	99	[MSR_ITLB_ESCR1] 0x3b7,
	100	[MSR_IX_ESCR1] 0x3c9,
	101	},
	102	[MSR_MS_COUNTER0] {
	103	[MSR_MS_ESCR1] 0x3c1,
	104	[MSR_TBPU_ESCR1] 0x3c3,
	105	[MSR_TC_ESCR1] 0x3c5,
	106	},
	107	[MSR_MS_COUNTER1] {
	108	[MSR_MS_ESCR1] 0x3c1,
	109	[MSR_TBPU_ESCR1] 0x3c3,
	110	[MSR_TC_ESCR1] 0x3c5,
	111	},
	112	[MSR_MS_COUNTER2] {
	113	[MSR_MS_ESCR1] 0x3c1,
	114	[MSR_TBPU_ESCR1] 0x3c3,
	115	[MSR_TC_ESCR1] 0x3c5,
	116	},
	117	[MSR_MS_COUNTER3] {
	118	[MSR_MS_ESCR1] 0x3c1,
	119	[MSR_TBPU_ESCR1] 0x3c3,
	120	[MSR_TC_ESCR1] 0x3c5,
121	},
122	[MSR_FLAME_COUNTER0] {
123	[MSR_FIRM_ESCR0] 0x3a4,
124	[MSR_FLAME_ESCR0] 0x3a6,
125	[MSR_DAC_ESCR0] 0x3a8,
126	[MSR_SAT_ESCR0] 0x3ae,
127	[MSR_U2L_ESCR0] 0x3b0,
128	},
129	[MSR_FLAME_COUNTER1] {
130	[MSR_FIRM_ESCR0] 0x3a4,
131	[MSR_FLAME_ESCR0] 0x3a6,
132	[MSR_DAC_ESCR0] 0x3a8,
133	[MSR_SAT_ESCR0] 0x3ae,
134	[MSR_U2L_ESCR0] 0x3b0,
135	},
136	[MSR_FLAME_COUNTER2] {
137	[MSR_FIRM_ESCR1] 0x3a5,
138	[MSR_FLAME_ESCR1] 0x3a7,
139	[MSR_DAC_ESCR1] 0x3a9,
140	[MSR_SAT_ESCR1] 0x3af,
141	[MSR_U2L_ESCR1] 0x3b1,
142	},
143	[MSR_FLAME_COUNTER3] {
144	[MSR_FIRM_ESCR1] 0x3a5,
145	[MSR_FLAME_ESCR1] 0x3a7,
146	[MSR_DAC_ESCR1] 0x3a9,
147	[MSR_SAT_ESCR1] 0x3af,
148	[MSR_U2L_ESCR1] 0x3b1,
149	},
150	[MSR_IQ_COUNTER0] {
151	[MSR_CRU_ESCR0] 0x3b8,
152	[MSR_CRU_ESCR2] 0x3cc,
153	[MSR_CRU_ESCR4] 0x3e0,
154	[MSR_IQ_ESCR0] 0x3ba,
155	[MSR_RAT_ESCR0] 0x3bc,
156	[MSR_SSU_ESCR0] 0x3be,
157	[MSR_AFL_ESCR0] 0x3ca,
158	},
159	[MSR_IQ_COUNTER1] {
160	[MSR_CRU_ESCR0] 0x3b8,
161	[MSR_CRU_ESCR2] 0x3cc,
162	[MSR_CRU_ESCR4] 0x3e0,
163	[MSR_IQ_ESCR0] 0x3ba,
164	[MSR_RAT_ESCR0] 0x3bc,
165	[MSR_SSU_ESCR0] 0x3be,
166	[MSR_AFL_ESCR0] 0x3ca,
167	},
168	[MSR_IQ_COUNTER2] {
169	[MSR_CRU_ESCR1] 0x3b9,
170	[MSR_CRU_ESCR3] 0x3cd,
171	[MSR_CRU_ESCR5] 0x3e1,
172	[MSR_IQ_ESCR1] 0x3bb,
173	[MSR_RAT_ESCR1] 0x3bd,
174	[MSR_AFL_ESCR1] 0x3cb,
175	},
176	[MSR_IQ_COUNTER3] {
177	[MSR_CRU_ESCR1] 0x3b9,
178	[MSR_CRU_ESCR3] 0x3cd,
179	[MSR_CRU_ESCR5] 0x3e1,
180	[MSR_IQ_ESCR1] 0x3bb,
181	[MSR_RAT_ESCR1] 0x3bd,
182	[MSR_AFL_ESCR1] 0x3cb,
183	},
184	[MSR_IQ_COUNTER4] {
185	[MSR_CRU_ESCR0] 0x3b8,
186	[MSR_CRU_ESCR2] 0x3cc,
187	[MSR_CRU_ESCR4] 0x3e0,
188	[MSR_IQ_ESCR0] 0x3ba,
189	[MSR_RAT_ESCR0] 0x3bc,
190	[MSR_SSU_ESCR0] 0x3be,
191	[MSR_AFL_ESCR0] 0x3ca,
192	},
193	[MSR_IQ_COUNTER5] {
194	[MSR_CRU_ESCR1] 0x3b9,
195	[MSR_CRU_ESCR3] 0x3cd,
196	[MSR_CRU_ESCR5] 0x3e1,
197	[MSR_IQ_ESCR1] 0x3bb,
198	[MSR_RAT_ESCR1] 0x3bd,
199	[MSR_AFL_ESCR1] 0x3cb,
200	},
201	};
202	#define PMC_ESCR_ADDR(id,esid) pmc_escr_addr_table[id][esid]
203
204	typedef struct {
205	pmc_id_t id_max; /* Maximum counter id */
206	pmc_machine_t machine_type; /* P6 or P4/Xeon */
207	uint32_t msr_counter_base; /* First counter MSR */
208	uint32_t msr_control_base; /* First control MSR */
209	boolean_t reserved[18]; /* Max-sized arrays... */
210	pmc_ovf_func_t *ovf_func[18];
211	#ifdef DEBUG
212	pmc_cccr_t cccr_shadow[18]; /* Last cccr values set */
213	pmc_counter_t counter_shadow[18]; /* Last counter values set */
214	uint32_t ovfs_unexpected[18]; /* Count of unexpected intrs */
215	#endif
216	} pmc_table_t;
217
218	static pmc_machine_t
219	_pmc_machine_type(void)
220	{
221	i386_cpu_info_t *infop = cpuid_info();
222
223	if (strncmp(infop->cpuid_vendor, CPUID_VID_INTEL, sizeof(CPUID_VID_INTEL)) != 0)
224	return pmc_none;
225
226	if (!pmc_is_available())
227	return pmc_none;
228
229	switch (infop->cpuid_family) {
230	case 0x6:
231	return pmc_P6;
232	case 0xf:
233	return pmc_P4_Xeon;
234	default:
235	return pmc_unknown;
236	}
237	}
238
239	static void
240	pmc_p4_intr(void *state)
241	{
242	pmc_table_t pmc_table = (pmc_table_t ) cpu_core()->pmc;
243	uint32_t cccr_addr;
244	pmc_cccr_t cccr;
245	pmc_id_t id;
246	int my_logical_cpu = cpu_to_logical_cpu(cpu_number());
247
248	/*
249	* Scan through table for reserved counters with overflow and
250	* with a registered overflow function.
251	*/
252	for (id = 0; id <= pmc_table->id_max; id++) {
253	if (!pmc_table->reserved[id])
254	continue;
255	cccr_addr = pmc_table->msr_control_base + id;
256	cccr.u_u64 = rdmsr64(cccr_addr);
257	#ifdef DEBUG
258	pmc_table->cccr_shadow[id] = cccr;
259	((uint64_t ) &pmc_table->counter_shadow[id]) =
260	rdmsr64(pmc_table->msr_counter_base + id);
261	#endif
262	if (cccr.u_htt.ovf == 0)
263	continue;
264	if ((cccr.u_htt.ovf_pmi_t0 == 1 && my_logical_cpu == 0) \|\|
265	(cccr.u_htt.ovf_pmi_t1 == 1 && my_logical_cpu == 1)) {
266	if (pmc_table->ovf_func[id]) {
267	(*pmc_table->ovf_func[id])(id, state);
268	/* func expected to clear overflow */
269	continue;
270	}
271	}
272	/* Clear overflow for unexpected interrupt */
273	#ifdef DEBUG
274	pmc_table->ovfs_unexpected[id]++;
275	#endif
276	}
277	}
278
279	static void
280	pmc_p6_intr(void *state)
281	{
282	pmc_table_t pmc_table = (pmc_table_t ) cpu_core()->pmc;
283	pmc_id_t id;
284
285	/*
286	* Can't determine which counter has overflow
287	* so call all registered functions.
288	*/
289	for (id = 0; id <= pmc_table->id_max; id++)
290	if (pmc_table->reserved[id] && pmc_table->ovf_func[id])
291	(*pmc_table->ovf_func[id])(id, state);
292	}
293
4452a7af A	294	void *
4452a7af A	295	pmc_alloc(void)
91447636 A	296	{
91447636 A	297	int ret;
91447636 A	298	pmc_table_t *pmc_table;
	299	pmc_machine_t pmc_type;
	300
91447636 A	301	pmc_type = _pmc_machine_type();
91447636 A	302	if (pmc_type == pmc_none) {
4452a7af	303	return NULL;
91447636 A	304	}
91447636 A	305
91447636 A	306	ret = kmem_alloc(kernel_map,
	307	(void *) &pmc_table, sizeof(pmc_table_t));
	308	if (ret != KERN_SUCCESS)
	309	panic("pmc_init() kmem_alloc returned %d\n", ret);
	310	bzero((void *)pmc_table, sizeof(pmc_table_t));
	311
	312	pmc_table->machine_type = pmc_type;
	313	switch (pmc_type) {
	314	case pmc_P4_Xeon:
	315	pmc_table->id_max = 17;
	316	pmc_table->msr_counter_base = MSR_COUNTER_ADDR(0);
	317	pmc_table->msr_control_base = MSR_CCCR_ADDR(0);
	318	lapic_set_pmi_func(&pmc_p4_intr);
	319	break;
	320	case pmc_P6:
	321	pmc_table->id_max = 1;
	322	pmc_table->msr_counter_base = MSR_P6_COUNTER_ADDR(0);
	323	pmc_table->msr_control_base = MSR_P6_PES_ADDR(0);
	324	lapic_set_pmi_func(&pmc_p6_intr);
	325	break;
	326	default:
	327	break;
	328	}
4452a7af	329	return (void *) pmc_table;
21362eb3	330	}
89b3af67	331
4452a7af	332
91447636 A	333	static inline pmc_table_t *
	334	pmc_table_valid(pmc_id_t id)
	335	{
	336	cpu_core_t *my_core = cpu_core();
	337	pmc_table_t *pmc_table;
	338
	339	assert(my_core);
	340
	341	pmc_table = (pmc_table_t *) my_core->pmc;
	342	return (pmc_table == NULL \|\|
	343	id > pmc_table->id_max \|\|
	344	!pmc_table->reserved[id]) ? NULL : pmc_table;
	345	}
	346
	347	int
	348	pmc_machine_type(pmc_machine_t *type)
	349	{
	350	cpu_core_t *my_core = cpu_core();
	351	pmc_table_t *pmc_table;
	352
	353	assert(my_core);
	354
	355	pmc_table = (pmc_table_t *) my_core->pmc;
	356	if (pmc_table == NULL)
	357	return KERN_FAILURE;
	358
	359	*type = pmc_table->machine_type;
	360
	361	return KERN_SUCCESS;
	362	}
	363
	364	int
	365	pmc_reserve(pmc_id_t id)
	366	{
	367	cpu_core_t *my_core = cpu_core();
	368	pmc_table_t *pmc_table;
	369
	370	assert(my_core);
	371
	372	pmc_table = (pmc_table_t *) my_core->pmc;
	373	if (pmc_table == NULL)
	374	return KERN_FAILURE;
	375	if (id > pmc_table->id_max)
	376	return KERN_INVALID_ARGUMENT;
	377	if (pmc_table->reserved[id])
	378	return KERN_FAILURE;
	379
	380	pmc_table->reserved[id] = TRUE;
	381
	382	return KERN_SUCCESS;
	383	}
	384
	385	boolean_t
	386	pmc_is_reserved(pmc_id_t id)
	387	{
	388	return pmc_table_valid(id) != NULL;
	389	}
	390
	391	int
	392	pmc_free(pmc_id_t id)
	393	{
	394	pmc_table_t *pmc_table = pmc_table_valid(id);
	395
	396	if (pmc_table == NULL)
397	return KERN_INVALID_ARGUMENT;
398
399	pmc_cccr_write(id, 0x0ULL);
400	pmc_table->reserved[id] = FALSE;
401	pmc_table->ovf_func[id] = NULL;
402
403	return KERN_SUCCESS;
404	}
405
406	int
407	pmc_counter_read(pmc_id_t id, pmc_counter_t *val)
408	{
409	pmc_table_t *pmc_table = pmc_table_valid(id);
410
411	if (pmc_table == NULL)
412	return KERN_INVALID_ARGUMENT;
413
414	(uint64_t )val = rdmsr64(pmc_table->msr_counter_base + id);
415
416	return KERN_SUCCESS;
417	}
418
419	int
420	pmc_counter_write(pmc_id_t id, pmc_counter_t *val)
421	{
422	pmc_table_t *pmc_table = pmc_table_valid(id);
423
424	if (pmc_table == NULL)
425	return KERN_INVALID_ARGUMENT;
426
427	wrmsr64(pmc_table->msr_counter_base + id, (uint64_t )val);
428
429	return KERN_SUCCESS;
430	}
431
432	int
433	pmc_cccr_read(pmc_id_t id, pmc_cccr_t *cccr)
434	{
435	pmc_table_t *pmc_table = pmc_table_valid(id);
436
437	if (pmc_table == NULL)
438	return KERN_INVALID_ARGUMENT;
439
440	if (pmc_table->machine_type != pmc_P4_Xeon)
441	return KERN_FAILURE;
442
443	(uint64_t )cccr = rdmsr64(pmc_table->msr_control_base + id);
444
445	return KERN_SUCCESS;
446	}
447
448	int
449	pmc_cccr_write(pmc_id_t id, pmc_cccr_t *cccr)
450	{
451	pmc_table_t *pmc_table = pmc_table_valid(id);
452
453	if (pmc_table == NULL)
454	return KERN_INVALID_ARGUMENT;
455
456	if (pmc_table->machine_type != pmc_P4_Xeon)
457	return KERN_FAILURE;
458
459	wrmsr64(pmc_table->msr_control_base + id, (uint64_t )cccr);
460
461	return KERN_SUCCESS;
462	}
463
464	int
465	pmc_evtsel_read(pmc_id_t id, pmc_evtsel_t *evtsel)
466	{
467	pmc_table_t *pmc_table = pmc_table_valid(id);
468
469	if (pmc_table == NULL)
470	return KERN_INVALID_ARGUMENT;
471
472	if (pmc_table->machine_type != pmc_P6)
473	return KERN_FAILURE;
474
475	(uint64_t )evtsel = rdmsr64(pmc_table->msr_control_base + id);
476
477	return KERN_SUCCESS;
478	}
479
480	int
481	pmc_evtsel_write(pmc_id_t id, pmc_evtsel_t *evtsel)
482	{
483	pmc_table_t *pmc_table = pmc_table_valid(id);
484
485	if (pmc_table == NULL)
486	return KERN_INVALID_ARGUMENT;
487
488	if (pmc_table->machine_type != pmc_P4_Xeon)
489	return KERN_FAILURE;
490
491	wrmsr64(pmc_table->msr_control_base + id, (uint64_t )evtsel);
492
493	return KERN_SUCCESS;
494	}
495
496	int
497	pmc_escr_read(pmc_id_t id, pmc_escr_id_t esid, pmc_escr_t *escr)
498	{
499	uint32_t addr;
500	pmc_table_t *pmc_table = pmc_table_valid(id);
501
502	if (pmc_table == NULL)
503	return KERN_INVALID_ARGUMENT;
504
505	if (pmc_table->machine_type != pmc_P4_Xeon)
506	return KERN_FAILURE;
507
508	if (esid > PMC_ESID_MAX)
509	return KERN_INVALID_ARGUMENT;
510
511	addr = PMC_ESCR_ADDR(id, esid);
512	if (addr == 0)
513	return KERN_INVALID_ARGUMENT;
514
515	(uint64_t )escr = rdmsr64(addr);
516
517	return KERN_SUCCESS;
518	}
519
520	int
521	pmc_escr_write(pmc_id_t id, pmc_escr_id_t esid, pmc_escr_t *escr)
522	{
523	uint32_t addr;
524	pmc_table_t *pmc_table = pmc_table_valid(id);
525
526	if (pmc_table == NULL)
527	return KERN_FAILURE;
528
529	if (pmc_table->machine_type != pmc_P4_Xeon)
530	return KERN_FAILURE;
531
532	if (esid > PMC_ESID_MAX)
533	return KERN_INVALID_ARGUMENT;
534
535	addr = PMC_ESCR_ADDR(id, esid);
536	if (addr == 0)
537	return KERN_INVALID_ARGUMENT;
538
539	wrmsr64(addr, (uint64_t )escr);
540
541	return KERN_SUCCESS;
542	}
543
544	int
545	pmc_set_ovf_func(pmc_id_t id, pmc_ovf_func_t func)
546	{
547	pmc_table_t *pmc_table = pmc_table_valid(id);
548
549	if (pmc_table == NULL)
550	return KERN_INVALID_ARGUMENT;
551
552	pmc_table->ovf_func[id] = func;
553
554	return KERN_SUCCESS;
555	}
4452a7af A	556
	557	int
	558	pmc_acquire(task_t task)
	559	{
	560	kern_return_t retval = KERN_SUCCESS;
	561
	562	simple_lock(&pmc_lock);
	563
	564	if(pmc_owner == task) {
	565	DBG("pmc_acquire - "
	566	"ACQUIRED: already owner\n");
	567	retval = KERN_SUCCESS;
	568	/* already own it */
	569	} else if(pmc_owner == TASK_NULL) { /* no one owns it */
	570	pmc_owner = task;
	571	pmc_thread_count = 0;
	572	DBG("pmc_acquire - "
	573	"ACQUIRED: no current owner - made new owner\n");
	574	retval = KERN_SUCCESS;
	575	} else { /* someone already owns it */
	576	if(pmc_owner == kernel_task) {
	577	if(pmc_thread_count == 0) {
	578	/* kernel owns it but no threads using it */
	579	pmc_owner = task;
	580	pmc_thread_count = 0;
	581	DBG("pmc_acquire - "
	582	"ACQUIRED: owned by kernel, no threads\n");
	583	retval = KERN_SUCCESS;
	584	} else {
	585	DBG("pmc_acquire - "
	586	"DENIED: owned by kernel, in use\n");
	587	retval = KERN_RESOURCE_SHORTAGE;
	588	}
	589	} else { /* non-kernel owner */
	590	DBG("pmc_acquire - "
	591	"DENIED: owned by another task\n");
	592	retval = KERN_RESOURCE_SHORTAGE;
	593	}
	594	}
	595
	596	simple_unlock(&pmc_lock);
	597	return retval;
	598	}
	599
	600	int
	601	pmc_release(task_t task)
	602	{
	603	kern_return_t retval = KERN_SUCCESS;
	604	task_t old_pmc_owner = pmc_owner;
	605
	606	simple_lock(&pmc_lock);
	607
	608	if(task != pmc_owner) {
	609	retval = KERN_NO_ACCESS;
	610	} else {
	611	if(old_pmc_owner == kernel_task) {
	612	if(pmc_thread_count>0) {
	613	DBG("pmc_release - "
	614	"NOT RELEASED: owned by kernel, in use\n");
	615	retval = KERN_NO_ACCESS;
	616	} else {
	617	DBG("pmc_release - "
	618	"RELEASED: was owned by kernel\n");
	619	pmc_owner = TASK_NULL;
620	retval = KERN_SUCCESS;
621	}
622	} else {
623	DBG("pmc_release - "
624	"RELEASED: was owned by user\n");
625	pmc_owner = TASK_NULL;
626	retval = KERN_SUCCESS;
627	}
628	}
629
630	simple_unlock(&pmc_lock);
631	return retval;
632	}
633