[apple/xnu.git] / osfmk / i386 / Diagnostics.c

/*
 * Copyright (c) 2005-2008 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
/*
 * @OSF_FREE_COPYRIGHT@
 */
/*
 * @APPLE_FREE_COPYRIGHT@
 */

/*
 *	Author: Bill Angell, Apple
 *	Date:	10/auht-five
 *
 *	Random diagnostics, augmented Derek Kumar 2011
 *
 *
 */


#include <kern/machine.h>
#include <kern/processor.h>
#include <mach/machine.h>
#include <mach/processor_info.h>
#include <mach/mach_types.h>
#include <mach/boolean.h>
#include <kern/thread.h>
#include <kern/task.h>
#include <kern/ipc_kobject.h>
#include <mach/vm_param.h>
#include <ipc/port.h>
#include <ipc/ipc_entry.h>
#include <ipc/ipc_space.h>
#include <ipc/ipc_object.h>
#include <ipc/ipc_port.h>
#include <vm/vm_kern.h>
#include <vm/vm_map.h>
#include <vm/vm_page.h>
#include <vm/pmap.h>
#include <pexpert/pexpert.h>
#include <console/video_console.h>
#include <i386/cpu_data.h>
#include <i386/Diagnostics.h>
#include <i386/mp.h>
#include <i386/pmCPU.h>
#include <i386/tsc.h>
#include <mach/i386/syscall_sw.h>
#include <kern/kalloc.h>
#include <sys/kdebug.h>
#include <i386/machine_cpu.h>
#include <i386/misc_protos.h>
#include <i386/cpuid.h>

#if MONOTONIC
#include <kern/monotonic.h>
#endif /* MONOTONIC */

#define PERMIT_PERMCHECK (0)

diagWork        dgWork;
uint64_t        lastRuptClear = 0ULL;
boolean_t       diag_pmc_enabled = FALSE;
void cpu_powerstats(void *);

typedef struct {
	uint64_t caperf;
	uint64_t cmperf;
	uint64_t ccres[6];
	uint64_t crtimes[CPU_RTIME_BINS];
	uint64_t citimes[CPU_ITIME_BINS];
	uint64_t crtime_total;
	uint64_t citime_total;
	uint64_t cpu_idle_exits;
	uint64_t cpu_insns;
	uint64_t cpu_ucc;
	uint64_t cpu_urc;
#if     DIAG_ALL_PMCS
	uint64_t gpmcs[4];
#endif /* DIAG_ALL_PMCS */
} core_energy_stat_t;

typedef struct {
	uint64_t pkes_version;
	uint64_t pkg_cres[2][7];
	uint64_t pkg_power_unit;
	uint64_t pkg_energy;
	uint64_t pp0_energy;
	uint64_t pp1_energy;
	uint64_t ddr_energy;
	uint64_t llc_flushed_cycles;
	uint64_t ring_ratio_instantaneous;
	uint64_t IA_frequency_clipping_cause;
	uint64_t GT_frequency_clipping_cause;
	uint64_t pkg_idle_exits;
	uint64_t pkg_rtimes[CPU_RTIME_BINS];
	uint64_t pkg_itimes[CPU_ITIME_BINS];
	uint64_t mbus_delay_time;
	uint64_t mint_delay_time;
	uint32_t ncpus;
	core_energy_stat_t cest[];
} pkg_energy_statistics_t;


int
diagCall64(x86_saved_state_t * state)
{
	uint64_t        curpos, i, j;
	uint64_t        selector, data;
	uint64_t        currNap, durNap;
	x86_saved_state64_t     *regs;
	boolean_t       diagflag;
	uint32_t        rval = 0;

	assert(is_saved_state64(state));
	regs = saved_state64(state);

	diagflag = ((dgWork.dgFlags & enaDiagSCs) != 0);
	selector = regs->rdi;

	switch (selector) {     /* Select the routine */
	case dgRuptStat:        /* Suck Interruption statistics */
		(void) ml_set_interrupts_enabled(TRUE);
		data = regs->rsi; /* Get the number of processors */

		if (data == 0) { /* If no location is specified for data, clear all
			          * counts
			          */
			for (i = 0; i < real_ncpus; i++) {      /* Cycle through
				                                 * processors */
				for (j = 0; j < 256; j++) {
					cpu_data_ptr[i]->cpu_hwIntCnt[j] = 0;
				}
			}

			lastRuptClear = mach_absolute_time();   /* Get the time of clear */
			rval = 1;       /* Normal return */
			(void) ml_set_interrupts_enabled(FALSE);
			break;
		}

		(void) copyout((char *) &real_ncpus, data, sizeof(real_ncpus)); /* Copy out number of
		                                                                 * processors */
		currNap = mach_absolute_time(); /* Get the time now */
		durNap = currNap - lastRuptClear;       /* Get the last interval
		                                         * duration */
		if (durNap == 0) {
			durNap = 1;     /* This is a very short time, make it
			                 * bigger */
		}
		curpos = data + sizeof(real_ncpus);     /* Point to the next
		                                         * available spot */

		for (i = 0; i < real_ncpus; i++) {      /* Move 'em all out */
			(void) copyout((char *) &durNap, curpos, 8);    /* Copy out the time
			                                                 * since last clear */
			(void) copyout((char *) &cpu_data_ptr[i]->cpu_hwIntCnt, curpos + 8, 256 * sizeof(uint32_t));    /* Copy out interrupt
			                                                                                                 * data for this
			                                                                                                 * processor */
			curpos = curpos + (256 * sizeof(uint32_t) + 8); /* Point to next out put
			                                                 * slot */
		}
		rval = 1;
		(void) ml_set_interrupts_enabled(FALSE);
		break;

	case dgPowerStat:
	{
		uint32_t c2l = 0, c2h = 0, c3l = 0, c3h = 0, c6l = 0, c6h = 0, c7l = 0, c7h = 0;
		uint32_t pkg_unit_l = 0, pkg_unit_h = 0, pkg_ecl = 0, pkg_ech = 0;

		pkg_energy_statistics_t pkes;
		core_energy_stat_t cest;

		bzero(&pkes, sizeof(pkes));
		bzero(&cest, sizeof(cest));

		pkes.pkes_version = 1ULL;
		rdmsr_carefully(MSR_IA32_PKG_C2_RESIDENCY, &c2l, &c2h);
		rdmsr_carefully(MSR_IA32_PKG_C3_RESIDENCY, &c3l, &c3h);
		rdmsr_carefully(MSR_IA32_PKG_C6_RESIDENCY, &c6l, &c6h);
		rdmsr_carefully(MSR_IA32_PKG_C7_RESIDENCY, &c7l, &c7h);

		pkes.pkg_cres[0][0] = ((uint64_t)c2h << 32) | c2l;
		pkes.pkg_cres[0][1] = ((uint64_t)c3h << 32) | c3l;
		pkes.pkg_cres[0][2] = ((uint64_t)c6h << 32) | c6l;
		pkes.pkg_cres[0][3] = ((uint64_t)c7h << 32) | c7l;

		uint64_t c8r = ~0ULL, c9r = ~0ULL, c10r = ~0ULL;

		rdmsr64_carefully(MSR_IA32_PKG_C8_RESIDENCY, &c8r);
		rdmsr64_carefully(MSR_IA32_PKG_C9_RESIDENCY, &c9r);
		rdmsr64_carefully(MSR_IA32_PKG_C10_RESIDENCY, &c10r);

		pkes.pkg_cres[0][4] = c8r;
		pkes.pkg_cres[0][5] = c9r;
		pkes.pkg_cres[0][6] = c10r;

		pkes.ddr_energy = ~0ULL;
		rdmsr64_carefully(MSR_IA32_DDR_ENERGY_STATUS, &pkes.ddr_energy);
		pkes.llc_flushed_cycles = ~0ULL;
		rdmsr64_carefully(MSR_IA32_LLC_FLUSHED_RESIDENCY_TIMER, &pkes.llc_flushed_cycles);

		pkes.ring_ratio_instantaneous = ~0ULL;
		rdmsr64_carefully(MSR_IA32_RING_PERF_STATUS, &pkes.ring_ratio_instantaneous);

		pkes.IA_frequency_clipping_cause = ~0ULL;

		uint32_t ia_perf_limits = MSR_IA32_IA_PERF_LIMIT_REASONS;
		/* Should perhaps be a generic register map module for these
		 * registers with identical functionality that were renumbered.
		 */
		switch (cpuid_cpufamily()) {
		case CPUFAMILY_INTEL_SKYLAKE:
		case CPUFAMILY_INTEL_KABYLAKE:
		case CPUFAMILY_INTEL_ICELAKE:
			ia_perf_limits = MSR_IA32_IA_PERF_LIMIT_REASONS_SKL;
			break;
		default:
			break;
		}

		rdmsr64_carefully(ia_perf_limits, &pkes.IA_frequency_clipping_cause);

		pkes.GT_frequency_clipping_cause = ~0ULL;
		rdmsr64_carefully(MSR_IA32_GT_PERF_LIMIT_REASONS, &pkes.GT_frequency_clipping_cause);

		rdmsr_carefully(MSR_IA32_PKG_POWER_SKU_UNIT, &pkg_unit_l, &pkg_unit_h);
		rdmsr_carefully(MSR_IA32_PKG_ENERGY_STATUS, &pkg_ecl, &pkg_ech);
		pkes.pkg_power_unit = ((uint64_t)pkg_unit_h << 32) | pkg_unit_l;
		pkes.pkg_energy = ((uint64_t)pkg_ech << 32) | pkg_ecl;

		rdmsr_carefully(MSR_IA32_PP0_ENERGY_STATUS, &pkg_ecl, &pkg_ech);
		pkes.pp0_energy = ((uint64_t)pkg_ech << 32) | pkg_ecl;

		rdmsr_carefully(MSR_IA32_PP1_ENERGY_STATUS, &pkg_ecl, &pkg_ech);
		pkes.pp1_energy = ((uint64_t)pkg_ech << 32) | pkg_ecl;

		pkes.pkg_idle_exits = current_cpu_datap()->lcpu.package->package_idle_exits;
		pkes.ncpus = real_ncpus;

		(void) ml_set_interrupts_enabled(TRUE);

		copyout(&pkes, regs->rsi, sizeof(pkes));
		curpos = regs->rsi + sizeof(pkes);

		mp_cpus_call(CPUMASK_ALL, ASYNC, cpu_powerstats, NULL);

		for (i = 0; i < real_ncpus; i++) {
			(void) ml_set_interrupts_enabled(FALSE);

			cest.caperf = cpu_data_ptr[i]->cpu_aperf;
			cest.cmperf = cpu_data_ptr[i]->cpu_mperf;
			cest.ccres[0] = cpu_data_ptr[i]->cpu_c3res;
			cest.ccres[1] = cpu_data_ptr[i]->cpu_c6res;
			cest.ccres[2] = cpu_data_ptr[i]->cpu_c7res;

			bcopy(&cpu_data_ptr[i]->cpu_rtimes[0], &cest.crtimes[0], sizeof(cest.crtimes));
			bcopy(&cpu_data_ptr[i]->cpu_itimes[0], &cest.citimes[0], sizeof(cest.citimes));

			cest.citime_total = cpu_data_ptr[i]->cpu_itime_total;
			cest.crtime_total = cpu_data_ptr[i]->cpu_rtime_total;
			cest.cpu_idle_exits = cpu_data_ptr[i]->cpu_idle_exits;
#if MONOTONIC
			cest.cpu_insns = cpu_data_ptr[i]->cpu_monotonic.mtc_counts[MT_CORE_INSTRS];
			cest.cpu_ucc = cpu_data_ptr[i]->cpu_monotonic.mtc_counts[MT_CORE_CYCLES];
			cest.cpu_urc = cpu_data_ptr[i]->cpu_monotonic.mtc_counts[MT_CORE_REFCYCLES];
#endif /* MONOTONIC */
#if DIAG_ALL_PMCS
			bcopy(&cpu_data_ptr[i]->cpu_gpmcs[0], &cest.gpmcs[0], sizeof(cest.gpmcs));
#endif /* DIAG_ALL_PMCS */
			(void) ml_set_interrupts_enabled(TRUE);

			copyout(&cest, curpos, sizeof(cest));
			curpos += sizeof(cest);
		}
		rval = 1;
		(void) ml_set_interrupts_enabled(FALSE);
	}
	break;
	case dgEnaPMC:
	{
		boolean_t enable = TRUE;
		uint32_t cpuinfo[4];
		/* Require architectural PMC v2 or higher, corresponding to
		 * Merom+, or equivalent virtualised facility.
		 */
		do_cpuid(0xA, &cpuinfo[0]);
		if ((cpuinfo[0] & 0xFF) >= 2) {
			mp_cpus_call(CPUMASK_ALL, ASYNC, cpu_pmc_control, &enable);
			diag_pmc_enabled = TRUE;
		}
		rval = 1;
	}
	break;
#if     DEVELOPMENT || DEBUG
	case dgGzallocTest:
	{
		(void) ml_set_interrupts_enabled(TRUE);
		if (diagflag) {
			unsigned *ptr = (unsigned *)kalloc(1024);
			kfree(ptr, 1024);
			*ptr = 0x42;
		}
		(void) ml_set_interrupts_enabled(FALSE);
	}
	break;
#endif

#if DEVELOPMENT || DEBUG
	case    dgPermCheck:
	{
		(void) ml_set_interrupts_enabled(TRUE);
		if (diagflag) {
			rval = pmap_permissions_verify(kernel_pmap, kernel_map, 0, ~0ULL);
		}
		(void) ml_set_interrupts_enabled(FALSE);
	}
	break;
#endif /* DEVELOPMENT || DEBUG */
	default:                /* Handle invalid ones */
		rval = 0;       /* Return an exception */
	}

	regs->rax = rval;

	assert(ml_get_interrupts_enabled() == FALSE);
	return rval;
}

void
cpu_powerstats(__unused void *arg)
{
	cpu_data_t *cdp = current_cpu_datap();
	__unused int cnum = cdp->cpu_number;
	uint32_t cl = 0, ch = 0, mpl = 0, mph = 0, apl = 0, aph = 0;

	rdmsr_carefully(MSR_IA32_MPERF, &mpl, &mph);
	rdmsr_carefully(MSR_IA32_APERF, &apl, &aph);

	cdp->cpu_mperf = ((uint64_t)mph << 32) | mpl;
	cdp->cpu_aperf = ((uint64_t)aph << 32) | apl;

	uint64_t ctime = mach_absolute_time();
	cdp->cpu_rtime_total += ctime - cdp->cpu_ixtime;
	cdp->cpu_ixtime = ctime;

	rdmsr_carefully(MSR_IA32_CORE_C3_RESIDENCY, &cl, &ch);
	cdp->cpu_c3res = ((uint64_t)ch << 32) | cl;

	rdmsr_carefully(MSR_IA32_CORE_C6_RESIDENCY, &cl, &ch);
	cdp->cpu_c6res = ((uint64_t)ch << 32) | cl;

	rdmsr_carefully(MSR_IA32_CORE_C7_RESIDENCY, &cl, &ch);
	cdp->cpu_c7res = ((uint64_t)ch << 32) | cl;

	if (diag_pmc_enabled) {
#if MONOTONIC
		mt_update_fixed_counts();
#else /* MONOTONIC */
		uint64_t insns = read_pmc(FIXED_PMC0);
		uint64_t ucc = read_pmc(FIXED_PMC1);
		uint64_t urc = read_pmc(FIXED_PMC2);
#endif /* !MONOTONIC */
#if DIAG_ALL_PMCS
		int i;

		for (i = 0; i < 4; i++) {
			cdp->cpu_gpmcs[i] = read_pmc(i);
		}
#endif /* DIAG_ALL_PMCS */
#if !MONOTONIC
		cdp->cpu_cur_insns = insns;
		cdp->cpu_cur_ucc = ucc;
		cdp->cpu_cur_urc = urc;
#endif /* !MONOTONIC */
	}
}

void
cpu_pmc_control(void *enablep)
{
#if !MONOTONIC
	boolean_t enable = *(boolean_t *)enablep;
	cpu_data_t      *cdp = current_cpu_datap();

	if (enable) {
		wrmsr64(0x38F, 0x70000000FULL);
		wrmsr64(0x38D, 0x333);
		set_cr4(get_cr4() | CR4_PCE);
	} else {
		wrmsr64(0x38F, 0);
		wrmsr64(0x38D, 0);
		set_cr4((get_cr4() & ~CR4_PCE));
	}
	cdp->cpu_fixed_pmcs_enabled = enable;
#else /* !MONOTONIC */
#pragma unused(enablep)
#endif /* MONOTONIC */
}
Commit	Line	Data
0c530ab8	1	/*
593a1d5f	2	* Copyright (c) 2005-2008 Apple Inc. All rights reserved.
0c530ab8	3	*
2d21ac55	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
0a7de745	5	*
2d21ac55 A	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
0a7de745	14	*
2d21ac55 A	15	* Please obtain a copy of the License at
2d21ac55 A	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
0a7de745	17	*
2d21ac55 A	18	* The Original Code and all software distributed under the License are
2d21ac55 A	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
0c530ab8 A	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
0c530ab8 A	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55 A	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
0a7de745	25	*
2d21ac55	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
0c530ab8 A	27	*/
	28	/*
	29	* @OSF_FREE_COPYRIGHT@
	30	*/
	31	/*
	32	* @APPLE_FREE_COPYRIGHT@
	33	*/
	34
	35	/*
	36	* Author: Bill Angell, Apple
	37	* Date: 10/auht-five
	38	*
316670eb	39	* Random diagnostics, augmented Derek Kumar 2011
0c530ab8	40	*
0c530ab8 A	41	*
	42	*/
	43
	44
	45	#include <kern/machine.h>
	46	#include <kern/processor.h>
	47	#include <mach/machine.h>
	48	#include <mach/processor_info.h>
	49	#include <mach/mach_types.h>
	50	#include <mach/boolean.h>
	51	#include <kern/thread.h>
	52	#include <kern/task.h>
	53	#include <kern/ipc_kobject.h>
	54	#include <mach/vm_param.h>
	55	#include <ipc/port.h>
	56	#include <ipc/ipc_entry.h>
	57	#include <ipc/ipc_space.h>
	58	#include <ipc/ipc_object.h>
	59	#include <ipc/ipc_port.h>
	60	#include <vm/vm_kern.h>
	61	#include <vm/vm_map.h>
	62	#include <vm/vm_page.h>
	63	#include <vm/pmap.h>
	64	#include <pexpert/pexpert.h>
	65	#include <console/video_console.h>
	66	#include <i386/cpu_data.h>
	67	#include <i386/Diagnostics.h>
	68	#include <i386/mp.h>
	69	#include <i386/pmCPU.h>
	70	#include <i386/tsc.h>
0c530ab8	71	#include <mach/i386/syscall_sw.h>
316670eb	72	#include <kern/kalloc.h>
db609669	73	#include <sys/kdebug.h>
bd504ef0 A	74	#include <i386/machine_cpu.h>
	75	#include <i386/misc_protos.h>
	76	#include <i386/cpuid.h>
	77
5ba3f43e A	78	#if MONOTONIC
	79	#include <kern/monotonic.h>
	80	#endif /* MONOTONIC */
	81
bd504ef0 A	82	#define PERMIT_PERMCHECK (0)
bd504ef0 A	83
0c530ab8	84	diagWork dgWork;
0c530ab8	85	uint64_t lastRuptClear = 0ULL;
0a7de745	86	boolean_t diag_pmc_enabled = FALSE;
4b17d6b6 A	87	void cpu_powerstats(void *);
	88
	89	typedef struct {
	90	uint64_t caperf;
	91	uint64_t cmperf;
bd504ef0 A	92	uint64_t ccres[6];
	93	uint64_t crtimes[CPU_RTIME_BINS];
	94	uint64_t citimes[CPU_ITIME_BINS];
4b17d6b6 A	95	uint64_t crtime_total;
4b17d6b6 A	96	uint64_t citime_total;
bd504ef0 A	97	uint64_t cpu_idle_exits;
	98	uint64_t cpu_insns;
	99	uint64_t cpu_ucc;
	100	uint64_t cpu_urc;
0a7de745	101	#if DIAG_ALL_PMCS
a1c7dba1 A	102	uint64_t gpmcs[4];
a1c7dba1 A	103	#endif /* DIAG_ALL_PMCS */
4b17d6b6 A	104	} core_energy_stat_t;
	105
	106	typedef struct {
39236c6e	107	uint64_t pkes_version;
bd504ef0	108	uint64_t pkg_cres[2][7];
4b17d6b6 A	109	uint64_t pkg_power_unit;
4b17d6b6 A	110	uint64_t pkg_energy;
bd504ef0 A	111	uint64_t pp0_energy;
	112	uint64_t pp1_energy;
	113	uint64_t ddr_energy;
	114	uint64_t llc_flushed_cycles;
	115	uint64_t ring_ratio_instantaneous;
	116	uint64_t IA_frequency_clipping_cause;
	117	uint64_t GT_frequency_clipping_cause;
	118	uint64_t pkg_idle_exits;
	119	uint64_t pkg_rtimes[CPU_RTIME_BINS];
	120	uint64_t pkg_itimes[CPU_ITIME_BINS];
	121	uint64_t mbus_delay_time;
	122	uint64_t mint_delay_time;
4b17d6b6 A	123	uint32_t ncpus;
	124	core_energy_stat_t cest[];
	125	} pkg_energy_statistics_t;
	126
db609669	127
0a7de745	128	int
2d21ac55	129	diagCall64(x86_saved_state_t * state)
0c530ab8	130	{
0a7de745 A	131	uint64_t curpos, i, j;
	132	uint64_t selector, data;
	133	uint64_t currNap, durNap;
	134	x86_saved_state64_t *regs;
	135	boolean_t diagflag;
	136	uint32_t rval = 0;
2d21ac55 A	137
	138	assert(is_saved_state64(state));
	139	regs = saved_state64(state);
39236c6e	140
db609669	141	diagflag = ((dgWork.dgFlags & enaDiagSCs) != 0);
2d21ac55 A	142	selector = regs->rdi;
2d21ac55 A	143
0a7de745 A	144	switch (selector) { /* Select the routine */
0a7de745 A	145	case dgRuptStat: /* Suck Interruption statistics */
316670eb	146	(void) ml_set_interrupts_enabled(TRUE);
2d21ac55 A	147	data = regs->rsi; /* Get the number of processors */
	148
	149	if (data == 0) { /* If no location is specified for data, clear all
0a7de745 A	150	* counts
	151	*/
	152	for (i = 0; i < real_ncpus; i++) { /* Cycle through
	153	* processors */
	154	for (j = 0; j < 256; j++) {
2d21ac55	155	cpu_data_ptr[i]->cpu_hwIntCnt[j] = 0;
0a7de745	156	}
2d21ac55 A	157	}
2d21ac55 A	158
0a7de745 A	159	lastRuptClear = mach_absolute_time(); /* Get the time of clear */
0a7de745 A	160	rval = 1; /* Normal return */
3e170ce0	161	(void) ml_set_interrupts_enabled(FALSE);
4b17d6b6	162	break;
2d21ac55 A	163	}
2d21ac55 A	164
0a7de745 A	165	(void) copyout((char ) &real_ncpus, data, sizeof(real_ncpus)); / Copy out number of
	166	* processors */
	167	currNap = mach_absolute_time(); /* Get the time now */
	168	durNap = currNap - lastRuptClear; /* Get the last interval
	169	* duration */
	170	if (durNap == 0) {
	171	durNap = 1; /* This is a very short time, make it
	172	* bigger */
	173	}
	174	curpos = data + sizeof(real_ncpus); /* Point to the next
	175	* available spot */
	176
	177	for (i = 0; i < real_ncpus; i++) { /* Move 'em all out */
	178	(void) copyout((char ) &durNap, curpos, 8); / Copy out the time
	179	* since last clear */
	180	(void) copyout((char ) &cpu_data_ptr[i]->cpu_hwIntCnt, curpos + 8, 256 sizeof(uint32_t)); /* Copy out interrupt
	181	* data for this
	182	* processor */
	183	curpos = curpos + (256 * sizeof(uint32_t) + 8); /* Point to next out put
	184	* slot */
2d21ac55	185	}
4b17d6b6	186	rval = 1;
3e170ce0	187	(void) ml_set_interrupts_enabled(FALSE);
2d21ac55	188	break;
39236c6e	189
4b17d6b6 A	190	case dgPowerStat:
	191	{
	192	uint32_t c2l = 0, c2h = 0, c3l = 0, c3h = 0, c6l = 0, c6h = 0, c7l = 0, c7h = 0;
	193	uint32_t pkg_unit_l = 0, pkg_unit_h = 0, pkg_ecl = 0, pkg_ech = 0;
	194
	195	pkg_energy_statistics_t pkes;
	196	core_energy_stat_t cest;
	197
	198	bzero(&pkes, sizeof(pkes));
	199	bzero(&cest, sizeof(cest));
	200
39236c6e	201	pkes.pkes_version = 1ULL;
4b17d6b6 A	202	rdmsr_carefully(MSR_IA32_PKG_C2_RESIDENCY, &c2l, &c2h);
	203	rdmsr_carefully(MSR_IA32_PKG_C3_RESIDENCY, &c3l, &c3h);
	204	rdmsr_carefully(MSR_IA32_PKG_C6_RESIDENCY, &c6l, &c6h);
	205	rdmsr_carefully(MSR_IA32_PKG_C7_RESIDENCY, &c7l, &c7h);
	206
	207	pkes.pkg_cres[0][0] = ((uint64_t)c2h << 32) \| c2l;
	208	pkes.pkg_cres[0][1] = ((uint64_t)c3h << 32) \| c3l;
	209	pkes.pkg_cres[0][2] = ((uint64_t)c6h << 32) \| c6l;
	210	pkes.pkg_cres[0][3] = ((uint64_t)c7h << 32) \| c7l;
	211
bd504ef0 A	212	uint64_t c8r = ~0ULL, c9r = ~0ULL, c10r = ~0ULL;
bd504ef0 A	213
fe8ab488 A	214	rdmsr64_carefully(MSR_IA32_PKG_C8_RESIDENCY, &c8r);
	215	rdmsr64_carefully(MSR_IA32_PKG_C9_RESIDENCY, &c9r);
	216	rdmsr64_carefully(MSR_IA32_PKG_C10_RESIDENCY, &c10r);
bd504ef0 A	217
	218	pkes.pkg_cres[0][4] = c8r;
	219	pkes.pkg_cres[0][5] = c9r;
	220	pkes.pkg_cres[0][6] = c10r;
	221
	222	pkes.ddr_energy = ~0ULL;
	223	rdmsr64_carefully(MSR_IA32_DDR_ENERGY_STATUS, &pkes.ddr_energy);
	224	pkes.llc_flushed_cycles = ~0ULL;
	225	rdmsr64_carefully(MSR_IA32_LLC_FLUSHED_RESIDENCY_TIMER, &pkes.llc_flushed_cycles);
	226
	227	pkes.ring_ratio_instantaneous = ~0ULL;
	228	rdmsr64_carefully(MSR_IA32_RING_PERF_STATUS, &pkes.ring_ratio_instantaneous);
	229
	230	pkes.IA_frequency_clipping_cause = ~0ULL;
3e170ce0 A	231
3e170ce0 A	232	uint32_t ia_perf_limits = MSR_IA32_IA_PERF_LIMIT_REASONS;
2dced7af A	233	/* Should perhaps be a generic register map module for these
	234	* registers with identical functionality that were renumbered.
	235	*/
	236	switch (cpuid_cpufamily()) {
	237	case CPUFAMILY_INTEL_SKYLAKE:
5ba3f43e	238	case CPUFAMILY_INTEL_KABYLAKE:
f427ee49	239	case CPUFAMILY_INTEL_ICELAKE:
2dced7af A	240	ia_perf_limits = MSR_IA32_IA_PERF_LIMIT_REASONS_SKL;
	241	break;
	242	default:
	243	break;
	244	}
3e170ce0 A	245
3e170ce0 A	246	rdmsr64_carefully(ia_perf_limits, &pkes.IA_frequency_clipping_cause);
bd504ef0 A	247
	248	pkes.GT_frequency_clipping_cause = ~0ULL;
	249	rdmsr64_carefully(MSR_IA32_GT_PERF_LIMIT_REASONS, &pkes.GT_frequency_clipping_cause);
	250
4b17d6b6 A	251	rdmsr_carefully(MSR_IA32_PKG_POWER_SKU_UNIT, &pkg_unit_l, &pkg_unit_h);
4b17d6b6 A	252	rdmsr_carefully(MSR_IA32_PKG_ENERGY_STATUS, &pkg_ecl, &pkg_ech);
4b17d6b6 A	253	pkes.pkg_power_unit = ((uint64_t)pkg_unit_h << 32) \| pkg_unit_l;
	254	pkes.pkg_energy = ((uint64_t)pkg_ech << 32) \| pkg_ecl;
	255
bd504ef0 A	256	rdmsr_carefully(MSR_IA32_PP0_ENERGY_STATUS, &pkg_ecl, &pkg_ech);
	257	pkes.pp0_energy = ((uint64_t)pkg_ech << 32) \| pkg_ecl;
	258
	259	rdmsr_carefully(MSR_IA32_PP1_ENERGY_STATUS, &pkg_ecl, &pkg_ech);
	260	pkes.pp1_energy = ((uint64_t)pkg_ech << 32) \| pkg_ecl;
	261
	262	pkes.pkg_idle_exits = current_cpu_datap()->lcpu.package->package_idle_exits;
4b17d6b6 A	263	pkes.ncpus = real_ncpus;
	264
	265	(void) ml_set_interrupts_enabled(TRUE);
	266
	267	copyout(&pkes, regs->rsi, sizeof(pkes));
	268	curpos = regs->rsi + sizeof(pkes);
	269
	270	mp_cpus_call(CPUMASK_ALL, ASYNC, cpu_powerstats, NULL);
0a7de745	271
4b17d6b6	272	for (i = 0; i < real_ncpus; i++) {
bd504ef0 A	273	(void) ml_set_interrupts_enabled(FALSE);
bd504ef0 A	274
4b17d6b6 A	275	cest.caperf = cpu_data_ptr[i]->cpu_aperf;
	276	cest.cmperf = cpu_data_ptr[i]->cpu_mperf;
	277	cest.ccres[0] = cpu_data_ptr[i]->cpu_c3res;
	278	cest.ccres[1] = cpu_data_ptr[i]->cpu_c6res;
	279	cest.ccres[2] = cpu_data_ptr[i]->cpu_c7res;
	280
	281	bcopy(&cpu_data_ptr[i]->cpu_rtimes[0], &cest.crtimes[0], sizeof(cest.crtimes));
	282	bcopy(&cpu_data_ptr[i]->cpu_itimes[0], &cest.citimes[0], sizeof(cest.citimes));
bd504ef0	283
4b17d6b6 A	284	cest.citime_total = cpu_data_ptr[i]->cpu_itime_total;
4b17d6b6 A	285	cest.crtime_total = cpu_data_ptr[i]->cpu_rtime_total;
5ba3f43e A	286	cest.cpu_idle_exits = cpu_data_ptr[i]->cpu_idle_exits;
	287	#if MONOTONIC
	288	cest.cpu_insns = cpu_data_ptr[i]->cpu_monotonic.mtc_counts[MT_CORE_INSTRS];
	289	cest.cpu_ucc = cpu_data_ptr[i]->cpu_monotonic.mtc_counts[MT_CORE_CYCLES];
	290	cest.cpu_urc = cpu_data_ptr[i]->cpu_monotonic.mtc_counts[MT_CORE_REFCYCLES];
	291	#endif /* MONOTONIC */
a1c7dba1 A	292	#if DIAG_ALL_PMCS
a1c7dba1 A	293	bcopy(&cpu_data_ptr[i]->cpu_gpmcs[0], &cest.gpmcs[0], sizeof(cest.gpmcs));
3e170ce0	294	#endif /* DIAG_ALL_PMCS */
5ba3f43e	295	(void) ml_set_interrupts_enabled(TRUE);
4b17d6b6 A	296
	297	copyout(&cest, curpos, sizeof(cest));
	298	curpos += sizeof(cest);
	299	}
	300	rval = 1;
3e170ce0	301	(void) ml_set_interrupts_enabled(FALSE);
4b17d6b6	302	}
0a7de745 A	303	break;
	304	case dgEnaPMC:
	305	{
	306	boolean_t enable = TRUE;
39236c6e A	307	uint32_t cpuinfo[4];
	308	/* Require architectural PMC v2 or higher, corresponding to
	309	* Merom+, or equivalent virtualised facility.
	310	*/
	311	do_cpuid(0xA, &cpuinfo[0]);
	312	if ((cpuinfo[0] & 0xFF) >= 2) {
	313	mp_cpus_call(CPUMASK_ALL, ASYNC, cpu_pmc_control, &enable);
	314	diag_pmc_enabled = TRUE;
	315	}
0a7de745 A	316	rval = 1;
	317	}
	318	break;
	319	#if DEVELOPMENT \|\| DEBUG
316670eb A	320	case dgGzallocTest:
	321	{
	322	(void) ml_set_interrupts_enabled(TRUE);
39236c6e A	323	if (diagflag) {
	324	unsigned ptr = (unsigned )kalloc(1024);
	325	kfree(ptr, 1024);
	326	*ptr = 0x42;
	327	}
3e170ce0	328	(void) ml_set_interrupts_enabled(FALSE);
2d21ac55	329	}
bd504ef0	330	break;
316670eb	331	#endif
2d21ac55	332
39037602	333	#if DEVELOPMENT \|\| DEBUG
0a7de745	334	case dgPermCheck:
316670eb A	335	{
316670eb A	336	(void) ml_set_interrupts_enabled(TRUE);
0a7de745	337	if (diagflag) {
39236c6e	338	rval = pmap_permissions_verify(kernel_pmap, kernel_map, 0, ~0ULL);
0a7de745	339	}
3e170ce0	340	(void) ml_set_interrupts_enabled(FALSE);
0c530ab8	341	}
0a7de745	342	break;
39037602	343	#endif /* DEVELOPMENT \|\| DEBUG */
0a7de745 A	344	default: /* Handle invalid ones */
0a7de745 A	345	rval = 0; /* Return an exception */
0c530ab8 A	346	}
0c530ab8 A	347
4b17d6b6 A	348	regs->rax = rval;
4b17d6b6 A	349
3e170ce0	350	assert(ml_get_interrupts_enabled() == FALSE);
39236c6e	351	return rval;
4b17d6b6 A	352	}
4b17d6b6 A	353
0a7de745 A	354	void
	355	cpu_powerstats(__unused void *arg)
	356	{
4b17d6b6	357	cpu_data_t *cdp = current_cpu_datap();
bd504ef0	358	__unused int cnum = cdp->cpu_number;
4b17d6b6 A	359	uint32_t cl = 0, ch = 0, mpl = 0, mph = 0, apl = 0, aph = 0;
	360
	361	rdmsr_carefully(MSR_IA32_MPERF, &mpl, &mph);
	362	rdmsr_carefully(MSR_IA32_APERF, &apl, &aph);
	363
	364	cdp->cpu_mperf = ((uint64_t)mph << 32) \| mpl;
	365	cdp->cpu_aperf = ((uint64_t)aph << 32) \| apl;
	366
bd504ef0 A	367	uint64_t ctime = mach_absolute_time();
	368	cdp->cpu_rtime_total += ctime - cdp->cpu_ixtime;
	369	cdp->cpu_ixtime = ctime;
4b17d6b6 A	370
	371	rdmsr_carefully(MSR_IA32_CORE_C3_RESIDENCY, &cl, &ch);
	372	cdp->cpu_c3res = ((uint64_t)ch << 32) \| cl;
	373
	374	rdmsr_carefully(MSR_IA32_CORE_C6_RESIDENCY, &cl, &ch);
	375	cdp->cpu_c6res = ((uint64_t)ch << 32) \| cl;
	376
	377	rdmsr_carefully(MSR_IA32_CORE_C7_RESIDENCY, &cl, &ch);
	378	cdp->cpu_c7res = ((uint64_t)ch << 32) \| cl;
39236c6e A	379
39236c6e A	380	if (diag_pmc_enabled) {
5ba3f43e A	381	#if MONOTONIC
	382	mt_update_fixed_counts();
	383	#else /* MONOTONIC */
39236c6e A	384	uint64_t insns = read_pmc(FIXED_PMC0);
	385	uint64_t ucc = read_pmc(FIXED_PMC1);
	386	uint64_t urc = read_pmc(FIXED_PMC2);
5ba3f43e	387	#endif /* !MONOTONIC */
a1c7dba1 A	388	#if DIAG_ALL_PMCS
	389	int i;
	390
	391	for (i = 0; i < 4; i++) {
	392	cdp->cpu_gpmcs[i] = read_pmc(i);
	393	}
	394	#endif /* DIAG_ALL_PMCS */
5ba3f43e	395	#if !MONOTONIC
39236c6e A	396	cdp->cpu_cur_insns = insns;
	397	cdp->cpu_cur_ucc = ucc;
	398	cdp->cpu_cur_urc = urc;
5ba3f43e	399	#endif /* !MONOTONIC */
39236c6e	400	}
bd504ef0 A	401	}
bd504ef0 A	402
0a7de745 A	403	void
	404	cpu_pmc_control(void *enablep)
	405	{
5ba3f43e	406	#if !MONOTONIC
bd504ef0	407	boolean_t enable = (boolean_t )enablep;
0a7de745	408	cpu_data_t *cdp = current_cpu_datap();
bd504ef0 A	409
	410	if (enable) {
	411	wrmsr64(0x38F, 0x70000000FULL);
	412	wrmsr64(0x38D, 0x333);
	413	set_cr4(get_cr4() \| CR4_PCE);
bd504ef0 A	414	} else {
	415	wrmsr64(0x38F, 0);
	416	wrmsr64(0x38D, 0);
	417	set_cr4((get_cr4() & ~CR4_PCE));
	418	}
	419	cdp->cpu_fixed_pmcs_enabled = enable;
5ba3f43e A	420	#else /* !MONOTONIC */
	421	#pragma unused(enablep)
	422	#endif /* MONOTONIC */
0c530ab8	423	}