[apple/xnu.git] / bsd / dev / monotonic.c

/*
 * Copyright (c) 2017 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <kern/monotonic.h>
#include <machine/machine_routines.h>
#include <machine/monotonic.h>
#include <pexpert/pexpert.h>
#include <sys/param.h> /* NULL */
#include <sys/stat.h> /* dev_t */
#include <miscfs/devfs/devfs.h> /* must come after sys/stat.h */
#include <sys/conf.h> /* must come after sys/stat.h */
#include <sys/sysctl.h>
#include <sys/sysproto.h>
#include <sys/systm.h>
#include <sys/types.h>
#include <sys/monotonic.h>

static int mt_cdev_open(dev_t dev, int flags, int devtype, proc_t p);
static int mt_cdev_close(dev_t dev, int flags, int devtype, proc_t p);
static int mt_cdev_ioctl(dev_t dev, unsigned long cmd, char *uptr, int fflag,
    proc_t p);

#define MT_NODE "monotonic"

static const struct cdevsw mt_cdevsw = {
	.d_open = mt_cdev_open,
	.d_close = mt_cdev_close,
	.d_ioctl = mt_cdev_ioctl,

	.d_read = eno_rdwrt, .d_write = eno_rdwrt, .d_stop = eno_stop,
	.d_reset = eno_reset, .d_ttys = NULL, .d_select = eno_select,
	.d_mmap = eno_mmap, .d_strategy = eno_strat, .d_type = 0
};

/*
 * Written at initialization, read-only thereafter.
 */
lck_grp_t *mt_lock_grp = NULL;
static int mt_dev_major;

static mt_device_t
mt_get_device(dev_t devnum)
{
	return &mt_devices[minor(devnum)];
}

static void
mt_device_lock(mt_device_t dev)
{
	lck_mtx_lock(&dev->mtd_lock);
}

static void
mt_device_unlock(mt_device_t dev)
{
	lck_mtx_unlock(&dev->mtd_lock);
}

static void
mt_device_assert_lock_held(__assert_only mt_device_t dev)
{
	LCK_MTX_ASSERT(&dev->mtd_lock, LCK_MTX_ASSERT_OWNED);
}

static void
mt_device_assert_inuse(__assert_only mt_device_t dev)
{
	assert(dev->mtd_inuse == true);
}

int
mt_dev_init(void)
{
	mt_lock_grp = lck_grp_alloc_init(MT_NODE, LCK_GRP_ATTR_NULL);
	assert(mt_lock_grp != NULL);

	mt_dev_major = cdevsw_add(-1 /* allocate a major number */, &mt_cdevsw);
	if (mt_dev_major < 0) {
		panic("monotonic: cdevsw_add failed: %d", mt_dev_major);
		__builtin_unreachable();
	}

	for (int i = 0; i < MT_NDEVS; i++) {
		if (mt_devices[i].mtd_init(&mt_devices[i])) {
			continue;
		}

		assert(mt_devices[i].mtd_ncounters > 0);

		dev_t dev = makedev(mt_dev_major, i);
		char name[128];
		snprintf(name, sizeof(name), MT_NODE "/%s", mt_devices[i].mtd_name);
		void *node = devfs_make_node(dev, DEVFS_CHAR, UID_ROOT,
		    GID_WINDOWSERVER, 0666, name);
		if (!node) {
			panic("monotonic: devfs_make_node failed for '%s'",
			    mt_devices[i].mtd_name);
			__builtin_unreachable();
		}

		lck_mtx_init(&mt_devices[i].mtd_lock, mt_lock_grp, LCK_ATTR_NULL);
	}

	return 0;
}

static int
mt_cdev_open(dev_t devnum, __unused int flags, __unused int devtype,
    __unused proc_t p)
{
	int error = 0;

	mt_device_t dev = mt_get_device(devnum);
	mt_device_lock(dev);
	if (dev->mtd_inuse) {
		error = EBUSY;
	} else {
		dev->mtd_inuse = true;
	}
	mt_device_unlock(dev);

	return error;
}

static int
mt_cdev_close(dev_t devnum, __unused int flags, __unused int devtype,
    __unused struct proc *p)
{
	mt_device_t dev = mt_get_device(devnum);

	mt_device_lock(dev);
	mt_device_assert_inuse(dev);
	dev->mtd_inuse = false;
	dev->mtd_reset();
	mt_device_unlock(dev);

	return 0;
}

static int
mt_ctl_add(mt_device_t dev, user_addr_t uptr)
{
	int error;
	uint32_t ctr;
	union monotonic_ctl_add ctl;

	mt_device_assert_lock_held(dev);

	error = copyin(uptr, &ctl, sizeof(ctl.in));
	if (error) {
		return error;
	}

	error = dev->mtd_add(&ctl.in.config, &ctr);
	if (error) {
		return error;
	}

	ctl.out.ctr = ctr;

	error = copyout(&ctl, uptr, sizeof(ctl.out));
	if (error) {
		return error;
	}

	return 0;
}

static int
mt_ctl_counts(mt_device_t dev, user_addr_t uptr)
{
	int error;
	union monotonic_ctl_counts ctl;

	mt_device_assert_lock_held(dev);

	error = copyin(uptr, &ctl, sizeof(ctl.in));
	if (error) {
		return error;
	}

	if (ctl.in.ctr_mask == 0) {
		return EINVAL;
	}

	{
		uint64_t counts[dev->mtd_nmonitors][dev->mtd_ncounters];
		memset(counts, 0,
		    dev->mtd_ncounters * dev->mtd_nmonitors * sizeof(counts[0][0]));
		error = dev->mtd_read(ctl.in.ctr_mask, (uint64_t *)counts);
		if (error) {
			return error;
		}

		error = copyout(&counts, uptr, sizeof(counts));
		if (error) {
			return error;
		}
	}

	return 0;
}

static int
mt_ctl_enable(mt_device_t dev, user_addr_t uptr)
{
	int error;
	union monotonic_ctl_enable ctl;

	mt_device_assert_lock_held(dev);

	error = copyin(uptr, &ctl, sizeof(ctl));
	if (error) {
		return error;
	}

	dev->mtd_enable(ctl.in.enable);

	return 0;
}

static int
mt_ctl_reset(mt_device_t dev)
{
	mt_device_assert_lock_held(dev);
	dev->mtd_reset();
	return 0;
}

static int
mt_cdev_ioctl(dev_t devnum, unsigned long cmd, char *arg, __unused int flags,
    __unused proc_t p)
{
	int error = ENODEV;
	user_addr_t uptr = *(user_addr_t *)(void *)arg;

	mt_device_t dev = mt_get_device(devnum);
	mt_device_lock(dev);

	switch (cmd) {
	case MT_IOC_RESET:
		error = mt_ctl_reset(dev);
		break;

	case MT_IOC_ADD:
		error = mt_ctl_add(dev, uptr);
		break;

	case MT_IOC_ENABLE:
		error = mt_ctl_enable(dev, uptr);
		break;

	case MT_IOC_COUNTS:
		error = mt_ctl_counts(dev, uptr);
		break;

	case MT_IOC_GET_INFO: {
		union monotonic_ctl_info info = {
			.out = {
				.nmonitors = dev->mtd_nmonitors,
				.ncounters = dev->mtd_ncounters,
			},
		};
		error = copyout(&info, uptr, sizeof(info));
		break;
	}

	default:
		error = ENODEV;
		break;
	}

	mt_device_unlock(dev);

	return error;
}

int
thread_selfcounts(__unused struct proc *p,
    struct thread_selfcounts_args *uap, __unused int *ret_out)
{
	switch (uap->type) {
	case 1: {
		uint64_t counts[2] = { 0 };
		uint64_t thread_counts[MT_CORE_NFIXED] = { 0 };

		mt_cur_thread_fixed_counts(thread_counts);

#ifdef MT_CORE_INSTRS
		counts[0] = thread_counts[MT_CORE_INSTRS];
#endif /* defined(MT_CORE_INSTRS) */
		counts[1] = thread_counts[MT_CORE_CYCLES];

		return copyout(counts, uap->buf, MIN(sizeof(counts), uap->nbytes));
	}
	default:
		return EINVAL;
	}
}

enum mt_sysctl {
	MT_SUPPORTED,
	MT_PMIS,
	MT_RETROGRADE,
	MT_TASK_THREAD,
	MT_DEBUG,
	MT_KDBG_TEST,
	MT_FIX_CPU_PERF,
	MT_FIX_THREAD_PERF,
	MT_FIX_TASK_PERF,
};

static int
mt_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg2)
	uint64_t start[MT_CORE_NFIXED] = { 0 }, end[MT_CORE_NFIXED] = { 0 };
	uint64_t counts[2] = { 0 };

	switch ((enum mt_sysctl)arg1) {
	case MT_SUPPORTED:
		return sysctl_io_number(req, (int)mt_core_supported, sizeof(int), NULL, NULL);
	case MT_PMIS:
		return sysctl_io_number(req, mt_count_pmis(), sizeof(uint64_t), NULL, NULL);
	case MT_RETROGRADE: {
		uint64_t value = os_atomic_load_wide(&mt_retrograde, relaxed);
		return sysctl_io_number(req, value, sizeof(mt_retrograde), NULL, NULL);
	}
	case MT_TASK_THREAD:
		return sysctl_io_number(req, (int)mt_core_supported, sizeof(int), NULL, NULL);
	case MT_DEBUG: {
		int value = mt_debug;

		int r = sysctl_io_number(req, value, sizeof(value), &value, NULL);
		if (r) {
			return r;
		}
		mt_debug = value;

		return 0;
	}
	case MT_KDBG_TEST: {
		if (req->newptr == USER_ADDR_NULL) {
			return EINVAL;
		}

		int intrs_en = ml_set_interrupts_enabled(FALSE);
		MT_KDBG_TMPCPU_START(0x3fff);
		MT_KDBG_TMPCPU_END(0x3fff);

		MT_KDBG_TMPTH_START(0x3fff);
		MT_KDBG_TMPTH_END(0x3fff);
		ml_set_interrupts_enabled(intrs_en);

		return 0;
	}
	case MT_FIX_CPU_PERF: {
		int intrs_en = ml_set_interrupts_enabled(FALSE);
		mt_fixed_counts(start);
		mt_fixed_counts(end);
		ml_set_interrupts_enabled(intrs_en);

		goto copyout_counts;
	}
	case MT_FIX_THREAD_PERF: {
		int intrs_en = ml_set_interrupts_enabled(FALSE);
		mt_cur_thread_fixed_counts(start);
		mt_cur_thread_fixed_counts(end);
		ml_set_interrupts_enabled(intrs_en);

		goto copyout_counts;
	}
	case MT_FIX_TASK_PERF: {
		int intrs_en = ml_set_interrupts_enabled(FALSE);
		mt_cur_task_fixed_counts(start);
		mt_cur_task_fixed_counts(end);
		ml_set_interrupts_enabled(intrs_en);

		goto copyout_counts;
	}
	default:
		return ENOENT;
	}

copyout_counts:

#ifdef MT_CORE_INSTRS
	counts[0] = end[MT_CORE_INSTRS] - start[MT_CORE_INSTRS];
#endif /* defined(MT_CORE_INSTRS) */
	counts[1] = end[MT_CORE_CYCLES] - start[MT_CORE_CYCLES];

	return copyout(counts, req->oldptr, MIN(req->oldlen, sizeof(counts)));
}

SYSCTL_DECL(_kern_monotonic);
SYSCTL_NODE(_kern, OID_AUTO, monotonic, CTLFLAG_RW | CTLFLAG_LOCKED, 0,
    "monotonic");

#define MT_SYSCTL(NAME, ARG, FLAGS, SIZE, SIZESTR, DESC) \
    SYSCTL_PROC(_kern_monotonic, OID_AUTO, NAME, \
    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED | (FLAGS), \
    (void *)(ARG), SIZE, mt_sysctl, SIZESTR, DESC)

MT_SYSCTL(supported, MT_SUPPORTED, 0, sizeof(int), "I",
    "whether monotonic is supported");
MT_SYSCTL(debug, MT_DEBUG, CTLFLAG_MASKED, sizeof(int), "I",
    "whether monotonic is printing debug messages");
MT_SYSCTL(pmis, MT_PMIS, 0, sizeof(uint64_t), "Q",
    "number of PMIs seen");
MT_SYSCTL(retrograde_updates, MT_RETROGRADE, 0, sizeof(uint64_t), "Q",
    "number of times a counter appeared to go backwards");
MT_SYSCTL(task_thread_counting, MT_TASK_THREAD, 0, sizeof(int), "I",
    "whether task and thread counting is enabled");
MT_SYSCTL(kdebug_test, MT_KDBG_TEST, CTLFLAG_MASKED, sizeof(int), "O",
    "whether task and thread counting is enabled");
MT_SYSCTL(fixed_cpu_perf, MT_FIX_CPU_PERF, CTLFLAG_MASKED,
    sizeof(uint64_t) * 2, "O",
    "overhead of accessing the current CPU's counters");
MT_SYSCTL(fixed_thread_perf, MT_FIX_THREAD_PERF, CTLFLAG_MASKED,
    sizeof(uint64_t) * 2, "O",
    "overhead of accessing the current thread's counters");
MT_SYSCTL(fixed_task_perf, MT_FIX_TASK_PERF, CTLFLAG_MASKED,
    sizeof(uint64_t) * 2, "O",
    "overhead of accessing the current task's counters");
Commit	Line	Data
5ba3f43e A	1	/*
	2	* Copyright (c) 2017 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	#include <kern/monotonic.h>
	30	#include <machine/machine_routines.h>
	31	#include <machine/monotonic.h>
	32	#include <pexpert/pexpert.h>
	33	#include <sys/param.h> /* NULL */
	34	#include <sys/stat.h> /* dev_t */
	35	#include <miscfs/devfs/devfs.h> /* must come after sys/stat.h */
	36	#include <sys/conf.h> /* must come after sys/stat.h */
	37	#include <sys/sysctl.h>
	38	#include <sys/sysproto.h>
	39	#include <sys/systm.h>
	40	#include <sys/types.h>
	41	#include <sys/monotonic.h>
	42
d9a64523 A	43	static int mt_cdev_open(dev_t dev, int flags, int devtype, proc_t p);
	44	static int mt_cdev_close(dev_t dev, int flags, int devtype, proc_t p);
	45	static int mt_cdev_ioctl(dev_t dev, unsigned long cmd, char *uptr, int fflag,
0a7de745	46	proc_t p);
d9a64523 A	47
d9a64523 A	48	#define MT_NODE "monotonic"
5ba3f43e	49
f427ee49	50	static const struct cdevsw mt_cdevsw = {
d9a64523 A	51	.d_open = mt_cdev_open,
	52	.d_close = mt_cdev_close,
	53	.d_ioctl = mt_cdev_ioctl,
	54
	55	.d_read = eno_rdwrt, .d_write = eno_rdwrt, .d_stop = eno_stop,
	56	.d_reset = eno_reset, .d_ttys = NULL, .d_select = eno_select,
	57	.d_mmap = eno_mmap, .d_strategy = eno_strat, .d_type = 0
5ba3f43e A	58	};
	59
	60	/*
	61	* Written at initialization, read-only thereafter.
	62	*/
	63	lck_grp_t *mt_lock_grp = NULL;
5ba3f43e	64	static int mt_dev_major;
d9a64523 A	65
	66	static mt_device_t
	67	mt_get_device(dev_t devnum)
	68	{
	69	return &mt_devices[minor(devnum)];
	70	}
	71
	72	static void
	73	mt_device_lock(mt_device_t dev)
	74	{
	75	lck_mtx_lock(&dev->mtd_lock);
	76	}
5ba3f43e A	77
5ba3f43e A	78	static void
d9a64523	79	mt_device_unlock(mt_device_t dev)
5ba3f43e	80	{
d9a64523	81	lck_mtx_unlock(&dev->mtd_lock);
5ba3f43e A	82	}
	83
	84	static void
d9a64523	85	mt_device_assert_lock_held(__assert_only mt_device_t dev)
5ba3f43e	86	{
d9a64523	87	LCK_MTX_ASSERT(&dev->mtd_lock, LCK_MTX_ASSERT_OWNED);
5ba3f43e A	88	}
	89
	90	static void
d9a64523	91	mt_device_assert_inuse(__assert_only mt_device_t dev)
5ba3f43e	92	{
d9a64523	93	assert(dev->mtd_inuse == true);
5ba3f43e A	94	}
	95
	96	int
	97	mt_dev_init(void)
	98	{
d9a64523 A	99	mt_lock_grp = lck_grp_alloc_init(MT_NODE, LCK_GRP_ATTR_NULL);
d9a64523 A	100	assert(mt_lock_grp != NULL);
5ba3f43e A	101
	102	mt_dev_major = cdevsw_add(-1 /* allocate a major number */, &mt_cdevsw);
	103	if (mt_dev_major < 0) {
	104	panic("monotonic: cdevsw_add failed: %d", mt_dev_major);
d9a64523	105	__builtin_unreachable();
5ba3f43e A	106	}
	107
	108	for (int i = 0; i < MT_NDEVS; i++) {
d9a64523	109	if (mt_devices[i].mtd_init(&mt_devices[i])) {
5ba3f43e A	110	continue;
	111	}
	112
d9a64523 A	113	assert(mt_devices[i].mtd_ncounters > 0);
	114
	115	dev_t dev = makedev(mt_dev_major, i);
	116	char name[128];
	117	snprintf(name, sizeof(name), MT_NODE "/%s", mt_devices[i].mtd_name);
	118	void *node = devfs_make_node(dev, DEVFS_CHAR, UID_ROOT,
0a7de745	119	GID_WINDOWSERVER, 0666, name);
d9a64523	120	if (!node) {
5ba3f43e	121	panic("monotonic: devfs_make_node failed for '%s'",
0a7de745	122	mt_devices[i].mtd_name);
d9a64523	123	__builtin_unreachable();
5ba3f43e A	124	}
5ba3f43e A	125
d9a64523	126	lck_mtx_init(&mt_devices[i].mtd_lock, mt_lock_grp, LCK_ATTR_NULL);
5ba3f43e A	127	}
	128
	129	return 0;
	130	}
	131
	132	static int
d9a64523	133	mt_cdev_open(dev_t devnum, __unused int flags, __unused int devtype,
0a7de745	134	__unused proc_t p)
5ba3f43e A	135	{
	136	int error = 0;
	137
d9a64523 A	138	mt_device_t dev = mt_get_device(devnum);
	139	mt_device_lock(dev);
	140	if (dev->mtd_inuse) {
5ba3f43e	141	error = EBUSY;
d9a64523 A	142	} else {
d9a64523 A	143	dev->mtd_inuse = true;
5ba3f43e	144	}
d9a64523	145	mt_device_unlock(dev);
5ba3f43e	146
5ba3f43e A	147	return error;
	148	}
	149
	150	static int
d9a64523	151	mt_cdev_close(dev_t devnum, __unused int flags, __unused int devtype,
0a7de745	152	__unused struct proc *p)
5ba3f43e	153	{
d9a64523	154	mt_device_t dev = mt_get_device(devnum);
5ba3f43e	155
d9a64523 A	156	mt_device_lock(dev);
	157	mt_device_assert_inuse(dev);
	158	dev->mtd_inuse = false;
	159	dev->mtd_reset();
	160	mt_device_unlock(dev);
5ba3f43e A	161
	162	return 0;
	163	}
	164
	165	static int
d9a64523	166	mt_ctl_add(mt_device_t dev, user_addr_t uptr)
5ba3f43e A	167	{
	168	int error;
	169	uint32_t ctr;
	170	union monotonic_ctl_add ctl;
	171
d9a64523	172	mt_device_assert_lock_held(dev);
5ba3f43e A	173
	174	error = copyin(uptr, &ctl, sizeof(ctl.in));
	175	if (error) {
	176	return error;
	177	}
	178
d9a64523	179	error = dev->mtd_add(&ctl.in.config, &ctr);
5ba3f43e A	180	if (error) {
	181	return error;
	182	}
	183
	184	ctl.out.ctr = ctr;
	185
	186	error = copyout(&ctl, uptr, sizeof(ctl.out));
	187	if (error) {
	188	return error;
	189	}
	190
	191	return 0;
	192	}
	193
	194	static int
d9a64523	195	mt_ctl_counts(mt_device_t dev, user_addr_t uptr)
5ba3f43e A	196	{
5ba3f43e A	197	int error;
5ba3f43e A	198	union monotonic_ctl_counts ctl;
5ba3f43e A	199
d9a64523	200	mt_device_assert_lock_held(dev);
5ba3f43e A	201
	202	error = copyin(uptr, &ctl, sizeof(ctl.in));
	203	if (error) {
	204	return error;
	205	}
	206
	207	if (ctl.in.ctr_mask == 0) {
	208	return EINVAL;
	209	}
5ba3f43e A	210
5ba3f43e A	211	{
d9a64523 A	212	uint64_t counts[dev->mtd_nmonitors][dev->mtd_ncounters];
d9a64523 A	213	memset(counts, 0,
0a7de745	214	dev->mtd_ncounters * dev->mtd_nmonitors * sizeof(counts[0][0]));
d9a64523	215	error = dev->mtd_read(ctl.in.ctr_mask, (uint64_t *)counts);
5ba3f43e A	216	if (error) {
	217	return error;
	218	}
	219
	220	error = copyout(&counts, uptr, sizeof(counts));
	221	if (error) {
	222	return error;
	223	}
	224	}
	225
	226	return 0;
	227	}
	228
	229	static int
d9a64523	230	mt_ctl_enable(mt_device_t dev, user_addr_t uptr)
5ba3f43e A	231	{
	232	int error;
	233	union monotonic_ctl_enable ctl;
	234
d9a64523	235	mt_device_assert_lock_held(dev);
5ba3f43e A	236
	237	error = copyin(uptr, &ctl, sizeof(ctl));
	238	if (error) {
	239	return error;
	240	}
	241
d9a64523	242	dev->mtd_enable(ctl.in.enable);
5ba3f43e A	243
	244	return 0;
	245	}
	246
	247	static int
d9a64523	248	mt_ctl_reset(mt_device_t dev)
5ba3f43e	249	{
d9a64523 A	250	mt_device_assert_lock_held(dev);
d9a64523 A	251	dev->mtd_reset();
5ba3f43e A	252	return 0;
	253	}
	254
	255	static int
d9a64523	256	mt_cdev_ioctl(dev_t devnum, unsigned long cmd, char *arg, __unused int flags,
0a7de745	257	__unused proc_t p)
5ba3f43e	258	{
d9a64523	259	int error = ENODEV;
5ba3f43e A	260	user_addr_t uptr = (user_addr_t )(void *)arg;
5ba3f43e A	261
d9a64523 A	262	mt_device_t dev = mt_get_device(devnum);
d9a64523 A	263	mt_device_lock(dev);
5ba3f43e A	264
	265	switch (cmd) {
	266	case MT_IOC_RESET:
	267	error = mt_ctl_reset(dev);
	268	break;
	269
	270	case MT_IOC_ADD:
d9a64523	271	error = mt_ctl_add(dev, uptr);
5ba3f43e A	272	break;
	273
	274	case MT_IOC_ENABLE:
	275	error = mt_ctl_enable(dev, uptr);
	276	break;
	277
	278	case MT_IOC_COUNTS:
d9a64523	279	error = mt_ctl_counts(dev, uptr);
5ba3f43e A	280	break;
5ba3f43e A	281
d9a64523 A	282	case MT_IOC_GET_INFO: {
	283	union monotonic_ctl_info info = {
	284	.out = {
	285	.nmonitors = dev->mtd_nmonitors,
	286	.ncounters = dev->mtd_ncounters,
	287	},
	288	};
	289	error = copyout(&info, uptr, sizeof(info));
	290	break;
	291	}
	292
5ba3f43e A	293	default:
	294	error = ENODEV;
	295	break;
	296	}
	297
d9a64523	298	mt_device_unlock(dev);
5ba3f43e A	299
	300	return error;
	301	}
	302
0a7de745 A	303	int
	304	thread_selfcounts(__unused struct proc *p,
	305	struct thread_selfcounts_args uap, __unused int ret_out)
5ba3f43e A	306	{
	307	switch (uap->type) {
	308	case 1: {
e8c3f781 A	309	uint64_t counts[2] = { 0 };
e8c3f781 A	310	uint64_t thread_counts[MT_CORE_NFIXED] = { 0 };
5ba3f43e A	311
	312	mt_cur_thread_fixed_counts(thread_counts);
	313
	314	#ifdef MT_CORE_INSTRS
	315	counts[0] = thread_counts[MT_CORE_INSTRS];
	316	#endif /* defined(MT_CORE_INSTRS) */
	317	counts[1] = thread_counts[MT_CORE_CYCLES];
	318
	319	return copyout(counts, uap->buf, MIN(sizeof(counts), uap->nbytes));
	320	}
	321	default:
	322	return EINVAL;
	323	}
	324	}
	325
	326	enum mt_sysctl {
	327	MT_SUPPORTED,
	328	MT_PMIS,
	329	MT_RETROGRADE,
	330	MT_TASK_THREAD,
	331	MT_DEBUG,
	332	MT_KDBG_TEST,
	333	MT_FIX_CPU_PERF,
	334	MT_FIX_THREAD_PERF,
	335	MT_FIX_TASK_PERF,
	336	};
	337
	338	static int
	339	mt_sysctl SYSCTL_HANDLER_ARGS
	340	{
	341	#pragma unused(oidp, arg2)
e8c3f781 A	342	uint64_t start[MT_CORE_NFIXED] = { 0 }, end[MT_CORE_NFIXED] = { 0 };
e8c3f781 A	343	uint64_t counts[2] = { 0 };
5ba3f43e A	344
	345	switch ((enum mt_sysctl)arg1) {
	346	case MT_SUPPORTED:
a39ff7e2	347	return sysctl_io_number(req, (int)mt_core_supported, sizeof(int), NULL, NULL);
5ba3f43e	348	case MT_PMIS:
cb323159 A	349	return sysctl_io_number(req, mt_count_pmis(), sizeof(uint64_t), NULL, NULL);
	350	case MT_RETROGRADE: {
	351	uint64_t value = os_atomic_load_wide(&mt_retrograde, relaxed);
	352	return sysctl_io_number(req, value, sizeof(mt_retrograde), NULL, NULL);
	353	}
5ba3f43e	354	case MT_TASK_THREAD:
a39ff7e2	355	return sysctl_io_number(req, (int)mt_core_supported, sizeof(int), NULL, NULL);
5ba3f43e A	356	case MT_DEBUG: {
	357	int value = mt_debug;
	358
	359	int r = sysctl_io_number(req, value, sizeof(value), &value, NULL);
	360	if (r) {
	361	return r;
	362	}
	363	mt_debug = value;
	364
	365	return 0;
	366	}
	367	case MT_KDBG_TEST: {
	368	if (req->newptr == USER_ADDR_NULL) {
	369	return EINVAL;
	370	}
	371
	372	int intrs_en = ml_set_interrupts_enabled(FALSE);
	373	MT_KDBG_TMPCPU_START(0x3fff);
	374	MT_KDBG_TMPCPU_END(0x3fff);
	375
	376	MT_KDBG_TMPTH_START(0x3fff);
	377	MT_KDBG_TMPTH_END(0x3fff);
	378	ml_set_interrupts_enabled(intrs_en);
	379
	380	return 0;
	381	}
	382	case MT_FIX_CPU_PERF: {
	383	int intrs_en = ml_set_interrupts_enabled(FALSE);
	384	mt_fixed_counts(start);
	385	mt_fixed_counts(end);
	386	ml_set_interrupts_enabled(intrs_en);
	387
	388	goto copyout_counts;
	389	}
	390	case MT_FIX_THREAD_PERF: {
	391	int intrs_en = ml_set_interrupts_enabled(FALSE);
	392	mt_cur_thread_fixed_counts(start);
	393	mt_cur_thread_fixed_counts(end);
	394	ml_set_interrupts_enabled(intrs_en);
	395
	396	goto copyout_counts;
	397	}
	398	case MT_FIX_TASK_PERF: {
	399	int intrs_en = ml_set_interrupts_enabled(FALSE);
	400	mt_cur_task_fixed_counts(start);
	401	mt_cur_task_fixed_counts(end);
	402	ml_set_interrupts_enabled(intrs_en);
	403
	404	goto copyout_counts;
	405	}
	406	default:
	407	return ENOENT;
	408	}
	409
	410	copyout_counts:
	411
	412	#ifdef MT_CORE_INSTRS
	413	counts[0] = end[MT_CORE_INSTRS] - start[MT_CORE_INSTRS];
	414	#endif /* defined(MT_CORE_INSTRS) */
	415	counts[1] = end[MT_CORE_CYCLES] - start[MT_CORE_CYCLES];
	416
	417	return copyout(counts, req->oldptr, MIN(req->oldlen, sizeof(counts)));
	418	}
	419
420	SYSCTL_DECL(_kern_monotonic);
421	SYSCTL_NODE(_kern, OID_AUTO, monotonic, CTLFLAG_RW \| CTLFLAG_LOCKED, 0,
0a7de745 A	422	"monotonic");
	423
	424	#define MT_SYSCTL(NAME, ARG, FLAGS, SIZE, SIZESTR, DESC) \
	425	SYSCTL_PROC(_kern_monotonic, OID_AUTO, NAME, \
	426	CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_LOCKED \| (FLAGS), \
	427	(void *)(ARG), SIZE, mt_sysctl, SIZESTR, DESC)
	428
	429	MT_SYSCTL(supported, MT_SUPPORTED, 0, sizeof(int), "I",
	430	"whether monotonic is supported");
	431	MT_SYSCTL(debug, MT_DEBUG, CTLFLAG_MASKED, sizeof(int), "I",
	432	"whether monotonic is printing debug messages");
	433	MT_SYSCTL(pmis, MT_PMIS, 0, sizeof(uint64_t), "Q",
	434	"number of PMIs seen");
	435	MT_SYSCTL(retrograde_updates, MT_RETROGRADE, 0, sizeof(uint64_t), "Q",
	436	"number of times a counter appeared to go backwards");
	437	MT_SYSCTL(task_thread_counting, MT_TASK_THREAD, 0, sizeof(int), "I",
	438	"whether task and thread counting is enabled");
	439	MT_SYSCTL(kdebug_test, MT_KDBG_TEST, CTLFLAG_MASKED, sizeof(int), "O",
	440	"whether task and thread counting is enabled");
	441	MT_SYSCTL(fixed_cpu_perf, MT_FIX_CPU_PERF, CTLFLAG_MASKED,
	442	sizeof(uint64_t) * 2, "O",
	443	"overhead of accessing the current CPU's counters");
	444	MT_SYSCTL(fixed_thread_perf, MT_FIX_THREAD_PERF, CTLFLAG_MASKED,
	445	sizeof(uint64_t) * 2, "O",
	446	"overhead of accessing the current thread's counters");
	447	MT_SYSCTL(fixed_task_perf, MT_FIX_TASK_PERF, CTLFLAG_MASKED,
	448	sizeof(uint64_t) * 2, "O",
	449	"overhead of accessing the current task's counters");