[apple/xnu.git] / osfmk / i386 / perfmon.c

/*
 * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */

#include <mach/std_types.h>
#include <i386/cpu_data.h>
#include <i386/cpu_number.h>
#include <i386/perfmon.h>
#include <i386/proc_reg.h>
#include <i386/cpu_threads.h>
#include <i386/mp.h>
#include <i386/cpuid.h>
#include <i386/lock.h>
#include <vm/vm_kern.h>

#ifdef DEBUG
#define	DBG(x...)	kprintf(x)
#else
#define	DBG(x...)
#endif

/*
 * Table of ESCRs and addresses associated with performance counters/CCCRs.
 * See Intel SDM Vol 3, Table 15-4 (section 15.9):
 */
static uint16_t pmc_escr_addr_table[18][8] = {
	[MSR_BPU_COUNTER0] {
		[MSR_BSU_ESCR0]		0x3a0,
		[MSR_FSB_ESCR0]		0x3a2,
		[MSR_MOB_ESCR0]		0x3aa,
		[MSR_PMH_ESCR0]		0x3ac,
		[MSR_BPU_ESCR0]		0x3b2,
		[MSR_IS_ESCR0]		0x3b4,
		[MSR_ITLB_ESCR0]	0x3b6,
		[MSR_IX_ESCR0]		0x3c8,
	},
	[MSR_BPU_COUNTER1] {
		[MSR_BSU_ESCR0]		0x3a0,
		[MSR_FSB_ESCR0]		0x3a2,
		[MSR_MOB_ESCR0]		0x3aa,
		[MSR_PMH_ESCR0]		0x3ac,
		[MSR_BPU_ESCR0]		0x3b2,
		[MSR_IS_ESCR0]		0x3b4,
		[MSR_ITLB_ESCR0]	0x3b6,
		[MSR_IX_ESCR0]		0x3c8,
	},
	[MSR_BPU_COUNTER2] {
		[MSR_BSU_ESCR1]		0x3a1,
		[MSR_FSB_ESCR1]		0x3a3,
		[MSR_MOB_ESCR1]		0x3ab,
		[MSR_PMH_ESCR1]		0x3ad,
		[MSR_BPU_ESCR1]		0x3b3,
		[MSR_IS_ESCR1]		0x3b5,
		[MSR_ITLB_ESCR1]	0x3b7,
		[MSR_IX_ESCR1]		0x3c9,
	},
	[MSR_BPU_COUNTER3] {
		[MSR_BSU_ESCR1]		0x3a1,
		[MSR_FSB_ESCR1]		0x3a3,
		[MSR_MOB_ESCR1]		0x3ab,
		[MSR_PMH_ESCR1]		0x3ad,
		[MSR_BPU_ESCR1]		0x3b3,
		[MSR_IS_ESCR1]		0x3b5,
		[MSR_ITLB_ESCR1]	0x3b7,
		[MSR_IX_ESCR1]		0x3c9,
	},
	[MSR_MS_COUNTER0] {
		[MSR_MS_ESCR1]		0x3c1,
		[MSR_TBPU_ESCR1]	0x3c3,
		[MSR_TC_ESCR1]		0x3c5,
	},
	[MSR_MS_COUNTER1] {
		[MSR_MS_ESCR1]		0x3c1,
		[MSR_TBPU_ESCR1]	0x3c3,
		[MSR_TC_ESCR1]		0x3c5,
	},
	[MSR_MS_COUNTER2] {
		[MSR_MS_ESCR1]		0x3c1,
		[MSR_TBPU_ESCR1]	0x3c3,
		[MSR_TC_ESCR1]		0x3c5,
	},
	[MSR_MS_COUNTER3] {
		[MSR_MS_ESCR1]		0x3c1,
		[MSR_TBPU_ESCR1]	0x3c3,
		[MSR_TC_ESCR1]		0x3c5,
	},
	[MSR_FLAME_COUNTER0] {
		[MSR_FIRM_ESCR0]	0x3a4,
		[MSR_FLAME_ESCR0]	0x3a6,
		[MSR_DAC_ESCR0]		0x3a8,
		[MSR_SAT_ESCR0]		0x3ae,
		[MSR_U2L_ESCR0]		0x3b0,
	},
	[MSR_FLAME_COUNTER1] {
		[MSR_FIRM_ESCR0]	0x3a4,
		[MSR_FLAME_ESCR0]	0x3a6,
		[MSR_DAC_ESCR0]		0x3a8,
		[MSR_SAT_ESCR0]		0x3ae,
		[MSR_U2L_ESCR0]		0x3b0,
	},
	[MSR_FLAME_COUNTER2] {
		[MSR_FIRM_ESCR1]	0x3a5,
		[MSR_FLAME_ESCR1]	0x3a7,
		[MSR_DAC_ESCR1]		0x3a9,
		[MSR_SAT_ESCR1]		0x3af,
		[MSR_U2L_ESCR1]		0x3b1,
	},
	[MSR_FLAME_COUNTER3] {
		[MSR_FIRM_ESCR1]	0x3a5,
		[MSR_FLAME_ESCR1]	0x3a7,
		[MSR_DAC_ESCR1]		0x3a9,
		[MSR_SAT_ESCR1]		0x3af,
		[MSR_U2L_ESCR1]		0x3b1,
	},
	[MSR_IQ_COUNTER0] {
		[MSR_CRU_ESCR0]		0x3b8,
		[MSR_CRU_ESCR2]		0x3cc,
		[MSR_CRU_ESCR4]		0x3e0,
		[MSR_IQ_ESCR0]		0x3ba,
		[MSR_RAT_ESCR0]		0x3bc,
		[MSR_SSU_ESCR0]		0x3be,
		[MSR_AFL_ESCR0]		0x3ca,
	},
	[MSR_IQ_COUNTER1] {
		[MSR_CRU_ESCR0]		0x3b8,
		[MSR_CRU_ESCR2]		0x3cc,
		[MSR_CRU_ESCR4]		0x3e0,
		[MSR_IQ_ESCR0]		0x3ba,
		[MSR_RAT_ESCR0]		0x3bc,
		[MSR_SSU_ESCR0]		0x3be,
		[MSR_AFL_ESCR0]		0x3ca,
	},
	[MSR_IQ_COUNTER2] {
		[MSR_CRU_ESCR1]		0x3b9,
		[MSR_CRU_ESCR3]		0x3cd,
		[MSR_CRU_ESCR5]		0x3e1,
		[MSR_IQ_ESCR1]		0x3bb,
		[MSR_RAT_ESCR1]		0x3bd,
		[MSR_AFL_ESCR1]		0x3cb,
	},
	[MSR_IQ_COUNTER3] {
		[MSR_CRU_ESCR1]		0x3b9,
		[MSR_CRU_ESCR3]		0x3cd,
		[MSR_CRU_ESCR5]		0x3e1,
		[MSR_IQ_ESCR1]		0x3bb,
		[MSR_RAT_ESCR1]		0x3bd,
		[MSR_AFL_ESCR1]		0x3cb,
	},
	[MSR_IQ_COUNTER4] {
		[MSR_CRU_ESCR0]		0x3b8,
		[MSR_CRU_ESCR2]		0x3cc,
		[MSR_CRU_ESCR4]		0x3e0,
		[MSR_IQ_ESCR0]		0x3ba,
		[MSR_RAT_ESCR0]		0x3bc,
		[MSR_SSU_ESCR0]		0x3be,
		[MSR_AFL_ESCR0]		0x3ca,
	},
	[MSR_IQ_COUNTER5] {
		[MSR_CRU_ESCR1]		0x3b9,
		[MSR_CRU_ESCR3]		0x3cd,
		[MSR_CRU_ESCR5]		0x3e1,
		[MSR_IQ_ESCR1]		0x3bb,
		[MSR_RAT_ESCR1]		0x3bd,
		[MSR_AFL_ESCR1]		0x3cb,
	},
};
#define PMC_ESCR_ADDR(id,esid)	pmc_escr_addr_table[id][esid]

typedef struct {
	pmc_id_t	id_max;			/* Maximum counter id */
	pmc_machine_t	machine_type;		/* P6 or P4/Xeon */
	uint32_t	msr_counter_base;	/* First counter MSR */
	uint32_t	msr_control_base;	/* First control MSR */
	boolean_t	reserved[18];		/* Max-sized arrays... */
	pmc_ovf_func_t	*ovf_func[18];
#ifdef DEBUG
	pmc_cccr_t	cccr_shadow[18];	/* Last cccr values set */
	pmc_counter_t	counter_shadow[18];	/* Last counter values set */
	uint32_t	ovfs_unexpected[18];	/* Count of unexpected intrs */
#endif
} pmc_table_t;

static pmc_machine_t
_pmc_machine_type(void)
{
	i386_cpu_info_t	*infop = cpuid_info();
        
	if (strncmp(infop->cpuid_vendor, CPUID_VID_INTEL, sizeof(CPUID_VID_INTEL)) != 0)
		return pmc_none;
	
	if (!pmc_is_available())
		return pmc_none;

	switch (infop->cpuid_family) {
	case 0x6:
		return pmc_P6;
	case 0xf:
		return pmc_P4_Xeon;
	default:
		return pmc_unknown;
	}
}

static void
pmc_p4_intr(void *state)
{
	pmc_table_t	*pmc_table = (pmc_table_t *) cpu_core()->pmc;
	uint32_t	cccr_addr;
	pmc_cccr_t	cccr;
	pmc_id_t	id;
	int		my_logical_cpu = cpu_to_logical_cpu(cpu_number());

	/*
	 * Scan through table for reserved counters with overflow and
	 * with a registered overflow function.
	 */
	for (id = 0; id <= pmc_table->id_max; id++) {
		if (!pmc_table->reserved[id])
			continue;
		cccr_addr = pmc_table->msr_control_base + id;
		cccr.u_u64 = rdmsr64(cccr_addr);
#ifdef DEBUG
		pmc_table->cccr_shadow[id] = cccr;
		*((uint64_t *) &pmc_table->counter_shadow[id]) =
			rdmsr64(pmc_table->msr_counter_base + id);
#endif
		if (cccr.u_htt.ovf == 0)
			continue;
		if ((cccr.u_htt.ovf_pmi_t0 == 1 && my_logical_cpu == 0) ||
		    (cccr.u_htt.ovf_pmi_t1 == 1 && my_logical_cpu == 1)) {
			if (pmc_table->ovf_func[id]) {
				(*pmc_table->ovf_func[id])(id, state);
				/* func expected to clear overflow */
				continue;
			}
		}
		/* Clear overflow for unexpected interrupt */
#ifdef DEBUG
		pmc_table->ovfs_unexpected[id]++;
#endif
	}
}

static void
pmc_p6_intr(void *state)
{
	pmc_table_t	*pmc_table = (pmc_table_t *) cpu_core()->pmc;
	pmc_id_t	id;

	/*
	 * Can't determine which counter has overflow
	 * so call all registered functions.
	 */
	for (id = 0; id <= pmc_table->id_max; id++)
		if (pmc_table->reserved[id] && pmc_table->ovf_func[id])
			(*pmc_table->ovf_func[id])(id, state);
}

int
pmc_init(void)
{
	int		ret;
	cpu_core_t	*my_core;
	pmc_table_t	*pmc_table;
	pmc_machine_t	pmc_type;

	my_core = cpu_core();
	assert(my_core);

	pmc_type = _pmc_machine_type();
	if (pmc_type == pmc_none) {
		return KERN_FAILURE;
	}
	
	pmc_table = (pmc_table_t *) my_core->pmc;
	if (pmc_table == NULL) {
		ret = kmem_alloc(kernel_map,
			(void *) &pmc_table, sizeof(pmc_table_t));
		if (ret != KERN_SUCCESS)
			panic("pmc_init() kmem_alloc returned %d\n", ret);
		bzero((void *)pmc_table, sizeof(pmc_table_t));

		pmc_table->machine_type = pmc_type;
		switch (pmc_type) {
		case pmc_P4_Xeon:
			pmc_table->id_max = 17;
			pmc_table->msr_counter_base = MSR_COUNTER_ADDR(0);
			pmc_table->msr_control_base = MSR_CCCR_ADDR(0);
			lapic_set_pmi_func(&pmc_p4_intr);
			break;
		case pmc_P6:
			pmc_table->id_max = 1;
			pmc_table->msr_counter_base = MSR_P6_COUNTER_ADDR(0);
			pmc_table->msr_control_base = MSR_P6_PES_ADDR(0);
			lapic_set_pmi_func(&pmc_p6_intr);
			break;
		default:
			break;
		}
		if (!atomic_cmpxchg((uint32_t *) &my_core->pmc,
				    0, (uint32_t) pmc_table)) {
			kmem_free(kernel_map,
				  (vm_offset_t) pmc_table, sizeof(pmc_table_t));
		}
	}
	DBG("pmc_init() done for cpu %d my_core->pmc=0x%x type=%d\n",
		cpu_number(), my_core->pmc, pmc_type);

	return KERN_SUCCESS;
}

static inline pmc_table_t *
pmc_table_valid(pmc_id_t id)
{
	cpu_core_t	*my_core = cpu_core();
	pmc_table_t	*pmc_table;

	assert(my_core);
	
	pmc_table = (pmc_table_t *) my_core->pmc;
	return (pmc_table == NULL ||
		id > pmc_table->id_max ||
		!pmc_table->reserved[id]) ? NULL : pmc_table;
}

int
pmc_machine_type(pmc_machine_t *type)
{
	cpu_core_t	*my_core = cpu_core();
	pmc_table_t	*pmc_table;

	assert(my_core);

	pmc_table = (pmc_table_t *) my_core->pmc;
	if (pmc_table == NULL)
		return KERN_FAILURE;

	*type = pmc_table->machine_type;
	
	return KERN_SUCCESS;
}

int
pmc_reserve(pmc_id_t id)
{
	cpu_core_t	*my_core = cpu_core();
	pmc_table_t	*pmc_table;

	assert(my_core);

	pmc_table = (pmc_table_t *) my_core->pmc;
	if (pmc_table == NULL)
		return KERN_FAILURE;
	if (id > pmc_table->id_max)
		return KERN_INVALID_ARGUMENT;
	if (pmc_table->reserved[id])
		return KERN_FAILURE;

	pmc_table->reserved[id] = TRUE;

	return KERN_SUCCESS;
}

boolean_t
pmc_is_reserved(pmc_id_t id)
{
	return pmc_table_valid(id) != NULL;
}

int
pmc_free(pmc_id_t id)
{
	pmc_table_t	*pmc_table = pmc_table_valid(id);

	if (pmc_table == NULL)
		return KERN_INVALID_ARGUMENT;

	pmc_cccr_write(id, 0x0ULL);
	pmc_table->reserved[id] = FALSE;
	pmc_table->ovf_func[id] = NULL;

	return KERN_SUCCESS;
}

int
pmc_counter_read(pmc_id_t id, pmc_counter_t *val)
{
	pmc_table_t	*pmc_table = pmc_table_valid(id);

	if (pmc_table == NULL)
		return KERN_INVALID_ARGUMENT;
	
	*(uint64_t *)val = rdmsr64(pmc_table->msr_counter_base + id);

	return KERN_SUCCESS;
}

int
pmc_counter_write(pmc_id_t id, pmc_counter_t *val)
{
	pmc_table_t	*pmc_table = pmc_table_valid(id);

	if (pmc_table == NULL)
		return KERN_INVALID_ARGUMENT;
	
	wrmsr64(pmc_table->msr_counter_base + id, *(uint64_t *)val);

	return KERN_SUCCESS;
}

int
pmc_cccr_read(pmc_id_t id, pmc_cccr_t *cccr)
{
	pmc_table_t	*pmc_table = pmc_table_valid(id);

	if (pmc_table == NULL)
		return KERN_INVALID_ARGUMENT;
	
	if (pmc_table->machine_type != pmc_P4_Xeon)
		return KERN_FAILURE;
	
	*(uint64_t *)cccr = rdmsr64(pmc_table->msr_control_base + id);

	return KERN_SUCCESS;
}

int
pmc_cccr_write(pmc_id_t id, pmc_cccr_t *cccr)
{
	pmc_table_t	*pmc_table = pmc_table_valid(id);

	if (pmc_table == NULL)
		return KERN_INVALID_ARGUMENT;

	if (pmc_table->machine_type != pmc_P4_Xeon)
		return KERN_FAILURE;
	
	wrmsr64(pmc_table->msr_control_base + id, *(uint64_t *)cccr);

	return KERN_SUCCESS;
}

int
pmc_evtsel_read(pmc_id_t id, pmc_evtsel_t *evtsel)
{
	pmc_table_t	*pmc_table = pmc_table_valid(id);

	if (pmc_table == NULL)
		return KERN_INVALID_ARGUMENT;
	
	if (pmc_table->machine_type != pmc_P6)
		return KERN_FAILURE;
	
	*(uint64_t *)evtsel = rdmsr64(pmc_table->msr_control_base + id);

	return KERN_SUCCESS;
}

int
pmc_evtsel_write(pmc_id_t id, pmc_evtsel_t *evtsel)
{
	pmc_table_t	*pmc_table = pmc_table_valid(id);

	if (pmc_table == NULL)
		return KERN_INVALID_ARGUMENT;

	if (pmc_table->machine_type != pmc_P4_Xeon)
		return KERN_FAILURE;
	
	wrmsr64(pmc_table->msr_control_base + id, *(uint64_t *)evtsel);

	return KERN_SUCCESS;
}

int
pmc_escr_read(pmc_id_t id, pmc_escr_id_t esid, pmc_escr_t *escr)
{
	uint32_t	addr;
	pmc_table_t	*pmc_table = pmc_table_valid(id);

	if (pmc_table == NULL)
		return KERN_INVALID_ARGUMENT;

	if (pmc_table->machine_type != pmc_P4_Xeon)
		return KERN_FAILURE;
	
	if (esid > PMC_ESID_MAX)
		return KERN_INVALID_ARGUMENT;

	addr = PMC_ESCR_ADDR(id, esid);
	if (addr == 0)
		return KERN_INVALID_ARGUMENT;

	*(uint64_t *)escr = rdmsr64(addr);

	return KERN_SUCCESS;
}

int
pmc_escr_write(pmc_id_t id, pmc_escr_id_t esid, pmc_escr_t *escr)
{
	uint32_t	addr;
	pmc_table_t	*pmc_table = pmc_table_valid(id);

	if (pmc_table == NULL)
		return KERN_FAILURE;

	if (pmc_table->machine_type != pmc_P4_Xeon)
		return KERN_FAILURE;
	
	if (esid > PMC_ESID_MAX)
		return KERN_INVALID_ARGUMENT;

	addr = PMC_ESCR_ADDR(id, esid);
	if (addr == 0)
		return KERN_INVALID_ARGUMENT;

	wrmsr64(addr, *(uint64_t *)escr);

	return KERN_SUCCESS;
}

int
pmc_set_ovf_func(pmc_id_t id, pmc_ovf_func_t func)
{
	pmc_table_t	*pmc_table = pmc_table_valid(id);

	if (pmc_table == NULL)
		return KERN_INVALID_ARGUMENT;

	pmc_table->ovf_func[id] = func;

	return KERN_SUCCESS;
}
Commit	Line	Data
91447636 A	1	/*
	2	* Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
ff6e181a A	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. Please obtain a copy of the License at
	10	* http://www.opensource.apple.com/apsl/ and read it before using this
	11	* file.
91447636	12	*
ff6e181a A	13	* The Original Code and all software distributed under the License are
ff6e181a A	14	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
91447636 A	15	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
91447636 A	16	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
ff6e181a A	17	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	18	* Please see the License for the specific language governing rights and
	19	* limitations under the License.
91447636 A	20	*
	21	* @APPLE_LICENSE_HEADER_END@
	22	*/
	23
	24	#include <mach/std_types.h>
	25	#include <i386/cpu_data.h>
	26	#include <i386/cpu_number.h>
	27	#include <i386/perfmon.h>
	28	#include <i386/proc_reg.h>
	29	#include <i386/cpu_threads.h>
	30	#include <i386/mp.h>
	31	#include <i386/cpuid.h>
	32	#include <i386/lock.h>
	33	#include <vm/vm_kern.h>
	34
	35	#ifdef DEBUG
	36	#define DBG(x...) kprintf(x)
	37	#else
	38	#define DBG(x...)
	39	#endif
	40
	41	/*
	42	* Table of ESCRs and addresses associated with performance counters/CCCRs.
	43	* See Intel SDM Vol 3, Table 15-4 (section 15.9):
	44	*/
	45	static uint16_t pmc_escr_addr_table[18][8] = {
	46	[MSR_BPU_COUNTER0] {
	47	[MSR_BSU_ESCR0] 0x3a0,
	48	[MSR_FSB_ESCR0] 0x3a2,
	49	[MSR_MOB_ESCR0] 0x3aa,
	50	[MSR_PMH_ESCR0] 0x3ac,
	51	[MSR_BPU_ESCR0] 0x3b2,
	52	[MSR_IS_ESCR0] 0x3b4,
	53	[MSR_ITLB_ESCR0] 0x3b6,
	54	[MSR_IX_ESCR0] 0x3c8,
	55	},
	56	[MSR_BPU_COUNTER1] {
	57	[MSR_BSU_ESCR0] 0x3a0,
	58	[MSR_FSB_ESCR0] 0x3a2,
	59	[MSR_MOB_ESCR0] 0x3aa,
	60	[MSR_PMH_ESCR0] 0x3ac,
	61	[MSR_BPU_ESCR0] 0x3b2,
	62	[MSR_IS_ESCR0] 0x3b4,
	63	[MSR_ITLB_ESCR0] 0x3b6,
	64	[MSR_IX_ESCR0] 0x3c8,
	65	},
	66	[MSR_BPU_COUNTER2] {
	67	[MSR_BSU_ESCR1] 0x3a1,
	68	[MSR_FSB_ESCR1] 0x3a3,
	69	[MSR_MOB_ESCR1] 0x3ab,
	70	[MSR_PMH_ESCR1] 0x3ad,
	71	[MSR_BPU_ESCR1] 0x3b3,
	72	[MSR_IS_ESCR1] 0x3b5,
	73	[MSR_ITLB_ESCR1] 0x3b7,
	74	[MSR_IX_ESCR1] 0x3c9,
	75	},
	76	[MSR_BPU_COUNTER3] {
	77	[MSR_BSU_ESCR1] 0x3a1,
	78	[MSR_FSB_ESCR1] 0x3a3,
	79	[MSR_MOB_ESCR1] 0x3ab,
	80	[MSR_PMH_ESCR1] 0x3ad,
	81	[MSR_BPU_ESCR1] 0x3b3,
	82	[MSR_IS_ESCR1] 0x3b5,
	83	[MSR_ITLB_ESCR1] 0x3b7,
84	[MSR_IX_ESCR1] 0x3c9,
85	},
86	[MSR_MS_COUNTER0] {
87	[MSR_MS_ESCR1] 0x3c1,
88	[MSR_TBPU_ESCR1] 0x3c3,
89	[MSR_TC_ESCR1] 0x3c5,
90	},
91	[MSR_MS_COUNTER1] {
92	[MSR_MS_ESCR1] 0x3c1,
93	[MSR_TBPU_ESCR1] 0x3c3,
94	[MSR_TC_ESCR1] 0x3c5,
95	},
96	[MSR_MS_COUNTER2] {
97	[MSR_MS_ESCR1] 0x3c1,
98	[MSR_TBPU_ESCR1] 0x3c3,
99	[MSR_TC_ESCR1] 0x3c5,
100	},
101	[MSR_MS_COUNTER3] {
102	[MSR_MS_ESCR1] 0x3c1,
103	[MSR_TBPU_ESCR1] 0x3c3,
104	[MSR_TC_ESCR1] 0x3c5,
105	},
106	[MSR_FLAME_COUNTER0] {
107	[MSR_FIRM_ESCR0] 0x3a4,
108	[MSR_FLAME_ESCR0] 0x3a6,
109	[MSR_DAC_ESCR0] 0x3a8,
110	[MSR_SAT_ESCR0] 0x3ae,
111	[MSR_U2L_ESCR0] 0x3b0,
112	},
113	[MSR_FLAME_COUNTER1] {
114	[MSR_FIRM_ESCR0] 0x3a4,
115	[MSR_FLAME_ESCR0] 0x3a6,
116	[MSR_DAC_ESCR0] 0x3a8,
117	[MSR_SAT_ESCR0] 0x3ae,
118	[MSR_U2L_ESCR0] 0x3b0,
119	},
120	[MSR_FLAME_COUNTER2] {
121	[MSR_FIRM_ESCR1] 0x3a5,
122	[MSR_FLAME_ESCR1] 0x3a7,
123	[MSR_DAC_ESCR1] 0x3a9,
124	[MSR_SAT_ESCR1] 0x3af,
125	[MSR_U2L_ESCR1] 0x3b1,
126	},
127	[MSR_FLAME_COUNTER3] {
128	[MSR_FIRM_ESCR1] 0x3a5,
129	[MSR_FLAME_ESCR1] 0x3a7,
130	[MSR_DAC_ESCR1] 0x3a9,
131	[MSR_SAT_ESCR1] 0x3af,
132	[MSR_U2L_ESCR1] 0x3b1,
133	},
134	[MSR_IQ_COUNTER0] {
135	[MSR_CRU_ESCR0] 0x3b8,
136	[MSR_CRU_ESCR2] 0x3cc,
137	[MSR_CRU_ESCR4] 0x3e0,
138	[MSR_IQ_ESCR0] 0x3ba,
139	[MSR_RAT_ESCR0] 0x3bc,
140	[MSR_SSU_ESCR0] 0x3be,
141	[MSR_AFL_ESCR0] 0x3ca,
142	},
143	[MSR_IQ_COUNTER1] {
144	[MSR_CRU_ESCR0] 0x3b8,
145	[MSR_CRU_ESCR2] 0x3cc,
146	[MSR_CRU_ESCR4] 0x3e0,
147	[MSR_IQ_ESCR0] 0x3ba,
148	[MSR_RAT_ESCR0] 0x3bc,
149	[MSR_SSU_ESCR0] 0x3be,
150	[MSR_AFL_ESCR0] 0x3ca,
151	},
152	[MSR_IQ_COUNTER2] {
153	[MSR_CRU_ESCR1] 0x3b9,
154	[MSR_CRU_ESCR3] 0x3cd,
155	[MSR_CRU_ESCR5] 0x3e1,
156	[MSR_IQ_ESCR1] 0x3bb,
157	[MSR_RAT_ESCR1] 0x3bd,
158	[MSR_AFL_ESCR1] 0x3cb,
159	},
160	[MSR_IQ_COUNTER3] {
161	[MSR_CRU_ESCR1] 0x3b9,
162	[MSR_CRU_ESCR3] 0x3cd,
163	[MSR_CRU_ESCR5] 0x3e1,
164	[MSR_IQ_ESCR1] 0x3bb,
165	[MSR_RAT_ESCR1] 0x3bd,
166	[MSR_AFL_ESCR1] 0x3cb,
167	},
168	[MSR_IQ_COUNTER4] {
169	[MSR_CRU_ESCR0] 0x3b8,
170	[MSR_CRU_ESCR2] 0x3cc,
171	[MSR_CRU_ESCR4] 0x3e0,
172	[MSR_IQ_ESCR0] 0x3ba,
173	[MSR_RAT_ESCR0] 0x3bc,
174	[MSR_SSU_ESCR0] 0x3be,
175	[MSR_AFL_ESCR0] 0x3ca,
176	},
177	[MSR_IQ_COUNTER5] {
178	[MSR_CRU_ESCR1] 0x3b9,
179	[MSR_CRU_ESCR3] 0x3cd,
180	[MSR_CRU_ESCR5] 0x3e1,
181	[MSR_IQ_ESCR1] 0x3bb,
182	[MSR_RAT_ESCR1] 0x3bd,
183	[MSR_AFL_ESCR1] 0x3cb,
184	},
185	};
186	#define PMC_ESCR_ADDR(id,esid) pmc_escr_addr_table[id][esid]
187
188	typedef struct {
189	pmc_id_t id_max; /* Maximum counter id */
190	pmc_machine_t machine_type; /* P6 or P4/Xeon */
191	uint32_t msr_counter_base; /* First counter MSR */
192	uint32_t msr_control_base; /* First control MSR */
193	boolean_t reserved[18]; /* Max-sized arrays... */
194	pmc_ovf_func_t *ovf_func[18];
195	#ifdef DEBUG
196	pmc_cccr_t cccr_shadow[18]; /* Last cccr values set */
197	pmc_counter_t counter_shadow[18]; /* Last counter values set */
198	uint32_t ovfs_unexpected[18]; /* Count of unexpected intrs */
199	#endif
200	} pmc_table_t;
201
202	static pmc_machine_t
203	_pmc_machine_type(void)
204	{
205	i386_cpu_info_t *infop = cpuid_info();
206
207	if (strncmp(infop->cpuid_vendor, CPUID_VID_INTEL, sizeof(CPUID_VID_INTEL)) != 0)
208	return pmc_none;
209
210	if (!pmc_is_available())
211	return pmc_none;
212
213	switch (infop->cpuid_family) {
214	case 0x6:
215	return pmc_P6;
216	case 0xf:
217	return pmc_P4_Xeon;
218	default:
219	return pmc_unknown;
220	}
221	}
222
223	static void
224	pmc_p4_intr(void *state)
225	{
226	pmc_table_t pmc_table = (pmc_table_t ) cpu_core()->pmc;
227	uint32_t cccr_addr;
228	pmc_cccr_t cccr;
229	pmc_id_t id;
230	int my_logical_cpu = cpu_to_logical_cpu(cpu_number());
231
232	/*
233	* Scan through table for reserved counters with overflow and
234	* with a registered overflow function.
235	*/
236	for (id = 0; id <= pmc_table->id_max; id++) {
237	if (!pmc_table->reserved[id])
238	continue;
239	cccr_addr = pmc_table->msr_control_base + id;
240	cccr.u_u64 = rdmsr64(cccr_addr);
241	#ifdef DEBUG
242	pmc_table->cccr_shadow[id] = cccr;
243	((uint64_t ) &pmc_table->counter_shadow[id]) =
244	rdmsr64(pmc_table->msr_counter_base + id);
245	#endif
246	if (cccr.u_htt.ovf == 0)
247	continue;
248	if ((cccr.u_htt.ovf_pmi_t0 == 1 && my_logical_cpu == 0) \|\|
249	(cccr.u_htt.ovf_pmi_t1 == 1 && my_logical_cpu == 1)) {
250	if (pmc_table->ovf_func[id]) {
251	(*pmc_table->ovf_func[id])(id, state);
252	/* func expected to clear overflow */
253	continue;
254	}
255	}
256	/* Clear overflow for unexpected interrupt */
257	#ifdef DEBUG
258	pmc_table->ovfs_unexpected[id]++;
259	#endif
260	}
261	}
262
263	static void
264	pmc_p6_intr(void *state)
265	{
266	pmc_table_t pmc_table = (pmc_table_t ) cpu_core()->pmc;
267	pmc_id_t id;
268
269	/*
270	* Can't determine which counter has overflow
271	* so call all registered functions.
272	*/
273	for (id = 0; id <= pmc_table->id_max; id++)
274	if (pmc_table->reserved[id] && pmc_table->ovf_func[id])
275	(*pmc_table->ovf_func[id])(id, state);
276	}
277
278	int
279	pmc_init(void)
280	{
281	int ret;
282	cpu_core_t *my_core;
283	pmc_table_t *pmc_table;
284	pmc_machine_t pmc_type;
285
286	my_core = cpu_core();
287	assert(my_core);
288
289	pmc_type = _pmc_machine_type();
290	if (pmc_type == pmc_none) {
291	return KERN_FAILURE;
292	}
293
294	pmc_table = (pmc_table_t *) my_core->pmc;
295	if (pmc_table == NULL) {
296	ret = kmem_alloc(kernel_map,
297	(void *) &pmc_table, sizeof(pmc_table_t));
298	if (ret != KERN_SUCCESS)
299	panic("pmc_init() kmem_alloc returned %d\n", ret);
300	bzero((void *)pmc_table, sizeof(pmc_table_t));
301
302	pmc_table->machine_type = pmc_type;
303	switch (pmc_type) {
304	case pmc_P4_Xeon:
305	pmc_table->id_max = 17;
306	pmc_table->msr_counter_base = MSR_COUNTER_ADDR(0);
307	pmc_table->msr_control_base = MSR_CCCR_ADDR(0);
308	lapic_set_pmi_func(&pmc_p4_intr);
309	break;
310	case pmc_P6:
311	pmc_table->id_max = 1;
312	pmc_table->msr_counter_base = MSR_P6_COUNTER_ADDR(0);
313	pmc_table->msr_control_base = MSR_P6_PES_ADDR(0);
314	lapic_set_pmi_func(&pmc_p6_intr);
315	break;
316	default:
317	break;
318	}
319	if (!atomic_cmpxchg((uint32_t *) &my_core->pmc,
320	0, (uint32_t) pmc_table)) {
321	kmem_free(kernel_map,
322	(vm_offset_t) pmc_table, sizeof(pmc_table_t));
323	}
324	}
325	DBG("pmc_init() done for cpu %d my_core->pmc=0x%x type=%d\n",
326	cpu_number(), my_core->pmc, pmc_type);
327
328	return KERN_SUCCESS;
329	}
330
331	static inline pmc_table_t *
332	pmc_table_valid(pmc_id_t id)
333	{
334	cpu_core_t *my_core = cpu_core();
335	pmc_table_t *pmc_table;
336
337	assert(my_core);
338
339	pmc_table = (pmc_table_t *) my_core->pmc;
340	return (pmc_table == NULL \|\|
341	id > pmc_table->id_max \|\|
342	!pmc_table->reserved[id]) ? NULL : pmc_table;
343	}
344
345	int
346	pmc_machine_type(pmc_machine_t *type)
347	{
348	cpu_core_t *my_core = cpu_core();
349	pmc_table_t *pmc_table;
350
351	assert(my_core);
352
353	pmc_table = (pmc_table_t *) my_core->pmc;
354	if (pmc_table == NULL)
355	return KERN_FAILURE;
356
357	*type = pmc_table->machine_type;
358
359	return KERN_SUCCESS;
360	}
361
362	int
363	pmc_reserve(pmc_id_t id)
364	{
365	cpu_core_t *my_core = cpu_core();
366	pmc_table_t *pmc_table;
367
368	assert(my_core);
369
370	pmc_table = (pmc_table_t *) my_core->pmc;
371	if (pmc_table == NULL)
372	return KERN_FAILURE;
373	if (id > pmc_table->id_max)
374	return KERN_INVALID_ARGUMENT;
375	if (pmc_table->reserved[id])
376	return KERN_FAILURE;
377
378	pmc_table->reserved[id] = TRUE;
379
380	return KERN_SUCCESS;
381	}
382
383	boolean_t
384	pmc_is_reserved(pmc_id_t id)
385	{
386	return pmc_table_valid(id) != NULL;
387	}
388
389	int
390	pmc_free(pmc_id_t id)
391	{
392	pmc_table_t *pmc_table = pmc_table_valid(id);
393
394	if (pmc_table == NULL)
395	return KERN_INVALID_ARGUMENT;
396
397	pmc_cccr_write(id, 0x0ULL);
398	pmc_table->reserved[id] = FALSE;
399	pmc_table->ovf_func[id] = NULL;
400
401	return KERN_SUCCESS;
402	}
403
404	int
405	pmc_counter_read(pmc_id_t id, pmc_counter_t *val)
406	{
407	pmc_table_t *pmc_table = pmc_table_valid(id);
408
409	if (pmc_table == NULL)
410	return KERN_INVALID_ARGUMENT;
411
412	(uint64_t )val = rdmsr64(pmc_table->msr_counter_base + id);
413
414	return KERN_SUCCESS;
415	}
416
417	int
418	pmc_counter_write(pmc_id_t id, pmc_counter_t *val)
419	{
420	pmc_table_t *pmc_table = pmc_table_valid(id);
421
422	if (pmc_table == NULL)
423	return KERN_INVALID_ARGUMENT;
424
425	wrmsr64(pmc_table->msr_counter_base + id, (uint64_t )val);
426
427	return KERN_SUCCESS;
428	}
429
430	int
431	pmc_cccr_read(pmc_id_t id, pmc_cccr_t *cccr)
432	{
433	pmc_table_t *pmc_table = pmc_table_valid(id);
434
435	if (pmc_table == NULL)
436	return KERN_INVALID_ARGUMENT;
437
438	if (pmc_table->machine_type != pmc_P4_Xeon)
439	return KERN_FAILURE;
440
441	(uint64_t )cccr = rdmsr64(pmc_table->msr_control_base + id);
442
443	return KERN_SUCCESS;
444	}
445
446	int
447	pmc_cccr_write(pmc_id_t id, pmc_cccr_t *cccr)
448	{
449	pmc_table_t *pmc_table = pmc_table_valid(id);
450
451	if (pmc_table == NULL)
452	return KERN_INVALID_ARGUMENT;
453
454	if (pmc_table->machine_type != pmc_P4_Xeon)
455	return KERN_FAILURE;
456
457	wrmsr64(pmc_table->msr_control_base + id, (uint64_t )cccr);
458
459	return KERN_SUCCESS;
460	}
461
462	int
463	pmc_evtsel_read(pmc_id_t id, pmc_evtsel_t *evtsel)
464	{
465	pmc_table_t *pmc_table = pmc_table_valid(id);
466
467	if (pmc_table == NULL)
468	return KERN_INVALID_ARGUMENT;
469
470	if (pmc_table->machine_type != pmc_P6)
471	return KERN_FAILURE;
472
473	(uint64_t )evtsel = rdmsr64(pmc_table->msr_control_base + id);
474
475	return KERN_SUCCESS;
476	}
477
478	int
479	pmc_evtsel_write(pmc_id_t id, pmc_evtsel_t *evtsel)
480	{
481	pmc_table_t *pmc_table = pmc_table_valid(id);
482
483	if (pmc_table == NULL)
484	return KERN_INVALID_ARGUMENT;
485
486	if (pmc_table->machine_type != pmc_P4_Xeon)
487	return KERN_FAILURE;
488
489	wrmsr64(pmc_table->msr_control_base + id, (uint64_t )evtsel);
490
491	return KERN_SUCCESS;
492	}
493
494	int
495	pmc_escr_read(pmc_id_t id, pmc_escr_id_t esid, pmc_escr_t *escr)
496	{
497	uint32_t addr;
498	pmc_table_t *pmc_table = pmc_table_valid(id);
499
500	if (pmc_table == NULL)
501	return KERN_INVALID_ARGUMENT;
502
503	if (pmc_table->machine_type != pmc_P4_Xeon)
504	return KERN_FAILURE;
505
506	if (esid > PMC_ESID_MAX)
507	return KERN_INVALID_ARGUMENT;
508
509	addr = PMC_ESCR_ADDR(id, esid);
510	if (addr == 0)
511	return KERN_INVALID_ARGUMENT;
512
513	(uint64_t )escr = rdmsr64(addr);
514
515	return KERN_SUCCESS;
516	}
517
518	int
519	pmc_escr_write(pmc_id_t id, pmc_escr_id_t esid, pmc_escr_t *escr)
520	{
521	uint32_t addr;
522	pmc_table_t *pmc_table = pmc_table_valid(id);
523
524	if (pmc_table == NULL)
525	return KERN_FAILURE;
526
527	if (pmc_table->machine_type != pmc_P4_Xeon)
528	return KERN_FAILURE;
529
530	if (esid > PMC_ESID_MAX)
531	return KERN_INVALID_ARGUMENT;
532
533	addr = PMC_ESCR_ADDR(id, esid);
534	if (addr == 0)
535	return KERN_INVALID_ARGUMENT;
536
537	wrmsr64(addr, (uint64_t )escr);
538
539	return KERN_SUCCESS;
540	}
541
542	int
543	pmc_set_ovf_func(pmc_id_t id, pmc_ovf_func_t func)
544	{
545	pmc_table_t *pmc_table = pmc_table_valid(id);
546
547	if (pmc_table == NULL)
548	return KERN_INVALID_ARGUMENT;
549
550	pmc_table->ovf_func[id] = func;
551
552	return KERN_SUCCESS;
553	}