[apple/xnu.git] / bsd / dev / monotonic.c

/*
 * Copyright (c) 2017 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <kern/monotonic.h>
#include <machine/machine_routines.h>
#include <machine/monotonic.h>
#include <pexpert/pexpert.h>
#include <sys/param.h> /* NULL */
#include <sys/stat.h> /* dev_t */
#include <miscfs/devfs/devfs.h> /* must come after sys/stat.h */
#include <sys/conf.h> /* must come after sys/stat.h */
#include <sys/sysctl.h>
#include <sys/sysproto.h>
#include <sys/systm.h>
#include <sys/types.h>
#include <sys/monotonic.h>

static int mt_dev_open(dev_t dev, int flags, int devtype, struct proc *p);
static int mt_dev_close(dev_t dev, int flags, int devtype, struct proc *p);
static int mt_dev_ioctl(dev_t dev, unsigned long cmd, char *uptr, int fflag,
		struct proc *p);

static struct cdevsw mt_cdevsw = {
	.d_open = mt_dev_open,
	.d_close = mt_dev_close,
	.d_read = eno_rdwrt,
	.d_write = eno_rdwrt,
	.d_ioctl = mt_dev_ioctl,
	.d_stop = eno_stop,
	.d_reset = eno_reset,
	.d_ttys = NULL,
	.d_select = eno_select,
	.d_mmap = eno_mmap,
	.d_strategy = eno_strat,
	.d_type = 0
};

/*
 * Written at initialization, read-only thereafter.
 */
lck_grp_t *mt_lock_grp = NULL;

static int mt_dev_major;
decl_lck_mtx_data(static, mt_dev_mtxs[MT_NDEVS]);
static bool mt_dev_owned[MT_NDEVS];

static void
mt_dev_lock(dev_t dev)
{
	lck_mtx_lock(&mt_dev_mtxs[minor(dev)]);
}

static void
mt_dev_unlock(dev_t dev)
{
	lck_mtx_unlock(&mt_dev_mtxs[minor(dev)]);
}

static void
mt_dev_assert_lock_held(__assert_only dev_t dev)
{
	LCK_MTX_ASSERT(&mt_dev_mtxs[minor(dev)], LCK_MTX_ASSERT_OWNED);
}

int
mt_dev_init(void)
{
	lck_grp_attr_t *lock_grp_attr = NULL;
	int devices = 0;

	lock_grp_attr = lck_grp_attr_alloc_init();
	mt_lock_grp = lck_grp_alloc_init("monotonic", lock_grp_attr);
	lck_grp_attr_free(lock_grp_attr);

	mt_dev_major = cdevsw_add(-1 /* allocate a major number */, &mt_cdevsw);
	if (mt_dev_major < 0) {
		panic("monotonic: cdevsw_add failed: %d", mt_dev_major);
		__builtin_trap();
	}

	for (int i = 0; i < MT_NDEVS; i++) {
		dev_t dev;
		void *dn;
		int error;

		error = monotonic_devs[i].mtd_init();
		if (error) {
			continue;
		}

		dev = makedev(mt_dev_major, i);
		dn = devfs_make_node(dev,
				DEVFS_CHAR, UID_ROOT, GID_WINDOWSERVER, 0666,
				monotonic_devs[i].mtd_name);
		if (dn == NULL) {
			panic("monotonic: devfs_make_node failed for '%s'",
					monotonic_devs[i].mtd_name);
			__builtin_trap();
		}

		lck_mtx_init(&mt_dev_mtxs[i], mt_lock_grp, LCK_ATTR_NULL);

		devices++;
	}

	return 0;
}

static int
mt_dev_open(dev_t dev, __unused int flags, __unused int devtype,
		__unused struct proc *p)
{
	int error = 0;

	mt_dev_lock(dev);

	if (mt_dev_owned[minor(dev)]) {
		error = EBUSY;
		goto out;
	}

	mt_dev_owned[minor(dev)] = true;

out:
	mt_dev_unlock(dev);
	return error;
}

static int
mt_dev_close(dev_t dev, __unused int flags, __unused int devtype,
		__unused struct proc *p)
{
	mt_dev_lock(dev);

	assert(mt_dev_owned[minor(dev)]);
	mt_dev_owned[minor(dev)] = false;

	monotonic_devs[minor(dev)].mtd_reset();

	mt_dev_unlock(dev);

	return 0;
}

static int
mt_ctl_add(dev_t dev, user_addr_t uptr, __unused int flags,
		__unused struct proc *p)
{
	int error;
	uint32_t ctr;
	union monotonic_ctl_add ctl;

	mt_dev_assert_lock_held(dev);

	error = copyin(uptr, &ctl, sizeof(ctl.in));
	if (error) {
		return error;
	}

	error = monotonic_devs[minor(dev)].mtd_add(&ctl.in.config, &ctr);
	if (error) {
		return error;
	}

	ctl.out.ctr = ctr;

	error = copyout(&ctl, uptr, sizeof(ctl.out));
	if (error) {
		return error;
	}

	return 0;
}

static int
mt_ctl_counts(dev_t dev, user_addr_t uptr, __unused int flags,
		__unused struct proc *p)
{
	int error;
	uint64_t ctrs;
	union monotonic_ctl_counts ctl;

	mt_dev_assert_lock_held(dev);

	error = copyin(uptr, &ctl, sizeof(ctl.in));
	if (error) {
		return error;
	}

	if (ctl.in.ctr_mask == 0) {
		return EINVAL;
	}
	ctrs = __builtin_popcountll(ctl.in.ctr_mask);

	{
		uint64_t counts[ctrs];
		error = monotonic_devs[minor(dev)].mtd_read(ctl.in.ctr_mask, counts);
		if (error) {
			return error;
		}

		error = copyout(&counts, uptr, sizeof(counts));
		if (error) {
			return error;
		}
	}

	return 0;
}

static int
mt_ctl_enable(dev_t dev, user_addr_t uptr)
{
	int error;
	union monotonic_ctl_enable ctl;

	mt_dev_assert_lock_held(dev);

	error = copyin(uptr, &ctl, sizeof(ctl));
	if (error) {
		return error;
	}

	monotonic_devs[minor(dev)].mtd_enable(ctl.in.enable);

	return 0;
}

static int
mt_ctl_reset(dev_t dev)
{
	mt_dev_assert_lock_held(dev);
	monotonic_devs[minor(dev)].mtd_reset();
	return 0;
}

static int
mt_dev_ioctl(dev_t dev, unsigned long cmd, char *arg, int flags,
		struct proc *p)
{
	int error;
	user_addr_t uptr = *(user_addr_t *)(void *)arg;

	mt_dev_lock(dev);

	switch (cmd) {
	case MT_IOC_RESET:
		error = mt_ctl_reset(dev);
		break;

	case MT_IOC_ADD:
		error = mt_ctl_add(dev, uptr, flags, p);
		break;

	case MT_IOC_ENABLE:
		error = mt_ctl_enable(dev, uptr);
		break;

	case MT_IOC_COUNTS:
		error = mt_ctl_counts(dev, uptr, flags, p);
		break;

	default:
		error = ENODEV;
		break;
	}

	mt_dev_unlock(dev);

	return error;
}

int thread_selfcounts(__unused struct proc *p,
		struct thread_selfcounts_args *uap, __unused int *ret_out)
{
	switch (uap->type) {
	case 1: {
		uint64_t counts[2] = {};
		uint64_t thread_counts[MT_CORE_NFIXED];

		mt_cur_thread_fixed_counts(thread_counts);

#ifdef MT_CORE_INSTRS
		counts[0] = thread_counts[MT_CORE_INSTRS];
#endif /* defined(MT_CORE_INSTRS) */
		counts[1] = thread_counts[MT_CORE_CYCLES];

		return copyout(counts, uap->buf, MIN(sizeof(counts), uap->nbytes));
	}
	default:
		return EINVAL;
	}
}

enum mt_sysctl {
	MT_SUPPORTED,
	MT_PMIS,
	MT_RETROGRADE,
	MT_TASK_THREAD,
	MT_DEBUG,
	MT_KDBG_TEST,
	MT_FIX_CPU_PERF,
	MT_FIX_THREAD_PERF,
	MT_FIX_TASK_PERF,
};

static int
mt_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg2)
	uint64_t start[MT_CORE_NFIXED], end[MT_CORE_NFIXED];
	uint64_t counts[2] = {};

	switch ((enum mt_sysctl)arg1) {
	case MT_SUPPORTED:
		return sysctl_io_number(req, (int)mt_core_supported, sizeof(int), NULL, NULL);
	case MT_PMIS:
		return sysctl_io_number(req, mt_pmis, sizeof(mt_pmis), NULL, NULL);
	case MT_RETROGRADE:
		return sysctl_io_number(req, mt_retrograde, sizeof(mt_retrograde), NULL, NULL);
	case MT_TASK_THREAD:
		return sysctl_io_number(req, (int)mt_core_supported, sizeof(int), NULL, NULL);
	case MT_DEBUG: {
		int value = mt_debug;

		int r = sysctl_io_number(req, value, sizeof(value), &value, NULL);
		if (r) {
			return r;
		}
		mt_debug = value;

		return 0;
	}
	case MT_KDBG_TEST: {
		if (req->newptr == USER_ADDR_NULL) {
			return EINVAL;
		}

		int intrs_en = ml_set_interrupts_enabled(FALSE);
		MT_KDBG_TMPCPU_START(0x3fff);
		MT_KDBG_TMPCPU_END(0x3fff);

		MT_KDBG_TMPTH_START(0x3fff);
		MT_KDBG_TMPTH_END(0x3fff);
		ml_set_interrupts_enabled(intrs_en);

		return 0;
	}
	case MT_FIX_CPU_PERF: {
		int intrs_en = ml_set_interrupts_enabled(FALSE);
		mt_fixed_counts(start);
		mt_fixed_counts(end);
		ml_set_interrupts_enabled(intrs_en);

		goto copyout_counts;
	}
	case MT_FIX_THREAD_PERF: {
		int intrs_en = ml_set_interrupts_enabled(FALSE);
		mt_cur_thread_fixed_counts(start);
		mt_cur_thread_fixed_counts(end);
		ml_set_interrupts_enabled(intrs_en);

		goto copyout_counts;
	}
	case MT_FIX_TASK_PERF: {
		int intrs_en = ml_set_interrupts_enabled(FALSE);
		mt_cur_task_fixed_counts(start);
		mt_cur_task_fixed_counts(end);
		ml_set_interrupts_enabled(intrs_en);

		goto copyout_counts;
	}
	default:
		return ENOENT;
	}

copyout_counts:

#ifdef MT_CORE_INSTRS
	counts[0] = end[MT_CORE_INSTRS] - start[MT_CORE_INSTRS];
#endif /* defined(MT_CORE_INSTRS) */
	counts[1] = end[MT_CORE_CYCLES] - start[MT_CORE_CYCLES];

	return copyout(counts, req->oldptr, MIN(req->oldlen, sizeof(counts)));
}

SYSCTL_DECL(_kern_monotonic);
SYSCTL_NODE(_kern, OID_AUTO, monotonic, CTLFLAG_RW | CTLFLAG_LOCKED, 0,
		"monotonic");

SYSCTL_PROC(_kern_monotonic, OID_AUTO, supported,
		CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED,
		(void *)MT_SUPPORTED, sizeof(int), mt_sysctl, "I",
		"whether monotonic is supported");

SYSCTL_PROC(_kern_monotonic, OID_AUTO, debug,
		CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MASKED,
		(void *)MT_DEBUG, sizeof(int), mt_sysctl, "I",
		"whether monotonic is printing debug messages");

SYSCTL_PROC(_kern_monotonic, OID_AUTO, pmis,
		CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED,
		(void *)MT_PMIS, sizeof(uint64_t), mt_sysctl, "Q",
		"how many PMIs have been seen");

SYSCTL_PROC(_kern_monotonic, OID_AUTO, retrograde_updates,
		CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED,
		(void *)MT_RETROGRADE, sizeof(uint64_t), mt_sysctl, "Q",
		"how many times a counter appeared to go backwards");

SYSCTL_PROC(_kern_monotonic, OID_AUTO, task_thread_counting,
		CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASKED,
		(void *)MT_TASK_THREAD, sizeof(int), mt_sysctl, "I",
		"task and thread counting enabled");

SYSCTL_PROC(_kern_monotonic, OID_AUTO, kdebug_test,
		CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MASKED | CTLFLAG_LOCKED,
		(void *)MT_KDBG_TEST, sizeof(int), mt_sysctl, "O",
		"test that kdebug integration works");

SYSCTL_PROC(_kern_monotonic, OID_AUTO, fixed_cpu_perf,
		CTLFLAG_RW | CTLFLAG_MASKED | CTLFLAG_LOCKED,
		(void *)MT_FIX_CPU_PERF, sizeof(uint64_t) * 2, mt_sysctl, "O",
		"overhead of accessing the current CPU's counters");

SYSCTL_PROC(_kern_monotonic, OID_AUTO, fixed_thread_perf,
		CTLFLAG_RW | CTLFLAG_MASKED | CTLFLAG_LOCKED,
		(void *)MT_FIX_THREAD_PERF, sizeof(uint64_t) * 2, mt_sysctl, "O",
		"overhead of accessing the current thread's counters");

SYSCTL_PROC(_kern_monotonic, OID_AUTO, fixed_task_perf,
		CTLFLAG_RW | CTLFLAG_MASKED | CTLFLAG_LOCKED,
		(void *)MT_FIX_TASK_PERF, sizeof(uint64_t) * 2, mt_sysctl, "O",
		"overhead of accessing the current task's counters");
Commit	Line	Data
5ba3f43e A	1	/*
	2	* Copyright (c) 2017 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	#include <kern/monotonic.h>
	30	#include <machine/machine_routines.h>
	31	#include <machine/monotonic.h>
	32	#include <pexpert/pexpert.h>
	33	#include <sys/param.h> /* NULL */
	34	#include <sys/stat.h> /* dev_t */
	35	#include <miscfs/devfs/devfs.h> /* must come after sys/stat.h */
	36	#include <sys/conf.h> /* must come after sys/stat.h */
	37	#include <sys/sysctl.h>
	38	#include <sys/sysproto.h>
	39	#include <sys/systm.h>
	40	#include <sys/types.h>
	41	#include <sys/monotonic.h>
	42
	43	static int mt_dev_open(dev_t dev, int flags, int devtype, struct proc *p);
	44	static int mt_dev_close(dev_t dev, int flags, int devtype, struct proc *p);
	45	static int mt_dev_ioctl(dev_t dev, unsigned long cmd, char *uptr, int fflag,
	46	struct proc *p);
	47
	48	static struct cdevsw mt_cdevsw = {
	49	.d_open = mt_dev_open,
	50	.d_close = mt_dev_close,
	51	.d_read = eno_rdwrt,
	52	.d_write = eno_rdwrt,
	53	.d_ioctl = mt_dev_ioctl,
	54	.d_stop = eno_stop,
	55	.d_reset = eno_reset,
	56	.d_ttys = NULL,
	57	.d_select = eno_select,
	58	.d_mmap = eno_mmap,
	59	.d_strategy = eno_strat,
	60	.d_type = 0
	61	};
	62
	63	/*
	64	* Written at initialization, read-only thereafter.
65	*/
66	lck_grp_t *mt_lock_grp = NULL;
67
68	static int mt_dev_major;
69	decl_lck_mtx_data(static, mt_dev_mtxs[MT_NDEVS]);
70	static bool mt_dev_owned[MT_NDEVS];
71
72	static void
73	mt_dev_lock(dev_t dev)
74	{
75	lck_mtx_lock(&mt_dev_mtxs[minor(dev)]);
76	}
77
78	static void
79	mt_dev_unlock(dev_t dev)
80	{
81	lck_mtx_unlock(&mt_dev_mtxs[minor(dev)]);
82	}
83
84	static void
85	mt_dev_assert_lock_held(__assert_only dev_t dev)
86	{
87	LCK_MTX_ASSERT(&mt_dev_mtxs[minor(dev)], LCK_MTX_ASSERT_OWNED);
88	}
89
90	int
91	mt_dev_init(void)
92	{
93	lck_grp_attr_t *lock_grp_attr = NULL;
94	int devices = 0;
95
96	lock_grp_attr = lck_grp_attr_alloc_init();
97	mt_lock_grp = lck_grp_alloc_init("monotonic", lock_grp_attr);
98	lck_grp_attr_free(lock_grp_attr);
99
100	mt_dev_major = cdevsw_add(-1 /* allocate a major number */, &mt_cdevsw);
101	if (mt_dev_major < 0) {
102	panic("monotonic: cdevsw_add failed: %d", mt_dev_major);
103	__builtin_trap();
104	}
105
106	for (int i = 0; i < MT_NDEVS; i++) {
107	dev_t dev;
108	void *dn;
109	int error;
110
111	error = monotonic_devs[i].mtd_init();
112	if (error) {
113	continue;
114	}
115
116	dev = makedev(mt_dev_major, i);
117	dn = devfs_make_node(dev,
118	DEVFS_CHAR, UID_ROOT, GID_WINDOWSERVER, 0666,
119	monotonic_devs[i].mtd_name);
120	if (dn == NULL) {
121	panic("monotonic: devfs_make_node failed for '%s'",
122	monotonic_devs[i].mtd_name);
123	__builtin_trap();
124	}
125
126	lck_mtx_init(&mt_dev_mtxs[i], mt_lock_grp, LCK_ATTR_NULL);
127
128	devices++;
129	}
130
131	return 0;
132	}
133
134	static int
135	mt_dev_open(dev_t dev, __unused int flags, __unused int devtype,
136	__unused struct proc *p)
137	{
138	int error = 0;
139
140	mt_dev_lock(dev);
141
142	if (mt_dev_owned[minor(dev)]) {
143	error = EBUSY;
144	goto out;
145	}
146
147	mt_dev_owned[minor(dev)] = true;
148
149	out:
150	mt_dev_unlock(dev);
151	return error;
152	}
153
154	static int
155	mt_dev_close(dev_t dev, __unused int flags, __unused int devtype,
156	__unused struct proc *p)
157	{
158	mt_dev_lock(dev);
159
160	assert(mt_dev_owned[minor(dev)]);
161	mt_dev_owned[minor(dev)] = false;
162
163	monotonic_devs[minor(dev)].mtd_reset();
164
165	mt_dev_unlock(dev);
166
167	return 0;
168	}
169
170	static int
171	mt_ctl_add(dev_t dev, user_addr_t uptr, __unused int flags,
172	__unused struct proc *p)
173	{
174	int error;
175	uint32_t ctr;
176	union monotonic_ctl_add ctl;
177
178	mt_dev_assert_lock_held(dev);
179
180	error = copyin(uptr, &ctl, sizeof(ctl.in));
181	if (error) {
182	return error;
183	}
184
185	error = monotonic_devs[minor(dev)].mtd_add(&ctl.in.config, &ctr);
186	if (error) {
187	return error;
188	}
189
190	ctl.out.ctr = ctr;
191
192	error = copyout(&ctl, uptr, sizeof(ctl.out));
193	if (error) {
194	return error;
195	}
196
197	return 0;
198	}
199
200	static int
201	mt_ctl_counts(dev_t dev, user_addr_t uptr, __unused int flags,
202	__unused struct proc *p)
203	{
204	int error;
205	uint64_t ctrs;
206	union monotonic_ctl_counts ctl;
207
208	mt_dev_assert_lock_held(dev);
209
210	error = copyin(uptr, &ctl, sizeof(ctl.in));
211	if (error) {
212	return error;
213	}
214
215	if (ctl.in.ctr_mask == 0) {
216	return EINVAL;
217	}
218	ctrs = __builtin_popcountll(ctl.in.ctr_mask);
219
220	{
221	uint64_t counts[ctrs];
222	error = monotonic_devs[minor(dev)].mtd_read(ctl.in.ctr_mask, counts);
223	if (error) {
224	return error;
225	}
226
227	error = copyout(&counts, uptr, sizeof(counts));
228	if (error) {
229	return error;
230	}
231	}
232
233	return 0;
234	}
235
236	static int
237	mt_ctl_enable(dev_t dev, user_addr_t uptr)
238	{
239	int error;
240	union monotonic_ctl_enable ctl;
241
242	mt_dev_assert_lock_held(dev);
243
244	error = copyin(uptr, &ctl, sizeof(ctl));
245	if (error) {
246	return error;
247	}
248
249	monotonic_devs[minor(dev)].mtd_enable(ctl.in.enable);
250
251	return 0;
252	}
253
254	static int
255	mt_ctl_reset(dev_t dev)
256	{
257	mt_dev_assert_lock_held(dev);
258	monotonic_devs[minor(dev)].mtd_reset();
259	return 0;
260	}
261
262	static int
263	mt_dev_ioctl(dev_t dev, unsigned long cmd, char *arg, int flags,
264	struct proc *p)
265	{
266	int error;
267	user_addr_t uptr = (user_addr_t )(void *)arg;
268
269	mt_dev_lock(dev);
270
271	switch (cmd) {
272	case MT_IOC_RESET:
273	error = mt_ctl_reset(dev);
274	break;
275
276	case MT_IOC_ADD:
277	error = mt_ctl_add(dev, uptr, flags, p);
278	break;
279
280	case MT_IOC_ENABLE:
281	error = mt_ctl_enable(dev, uptr);
282	break;
283
284	case MT_IOC_COUNTS:
285	error = mt_ctl_counts(dev, uptr, flags, p);
286	break;
287
288	default:
289	error = ENODEV;
290	break;
291	}
292
293	mt_dev_unlock(dev);
294
295	return error;
296	}
297
298	int thread_selfcounts(__unused struct proc *p,
299	struct thread_selfcounts_args uap, __unused int ret_out)
300	{
301	switch (uap->type) {
302	case 1: {
303	uint64_t counts[2] = {};
304	uint64_t thread_counts[MT_CORE_NFIXED];
305
306	mt_cur_thread_fixed_counts(thread_counts);
307
308	#ifdef MT_CORE_INSTRS
309	counts[0] = thread_counts[MT_CORE_INSTRS];
310	#endif /* defined(MT_CORE_INSTRS) */
311	counts[1] = thread_counts[MT_CORE_CYCLES];
312
313	return copyout(counts, uap->buf, MIN(sizeof(counts), uap->nbytes));
314	}
315	default:
316	return EINVAL;
317	}
318	}
319
320	enum mt_sysctl {
321	MT_SUPPORTED,
322	MT_PMIS,
323	MT_RETROGRADE,
324	MT_TASK_THREAD,
325	MT_DEBUG,
326	MT_KDBG_TEST,
327	MT_FIX_CPU_PERF,
328	MT_FIX_THREAD_PERF,
329	MT_FIX_TASK_PERF,
330	};
331
332	static int
333	mt_sysctl SYSCTL_HANDLER_ARGS
334	{
335	#pragma unused(oidp, arg2)
336	uint64_t start[MT_CORE_NFIXED], end[MT_CORE_NFIXED];
337	uint64_t counts[2] = {};
338
339	switch ((enum mt_sysctl)arg1) {
340	case MT_SUPPORTED:
a39ff7e2	341	return sysctl_io_number(req, (int)mt_core_supported, sizeof(int), NULL, NULL);
5ba3f43e A	342	case MT_PMIS:
	343	return sysctl_io_number(req, mt_pmis, sizeof(mt_pmis), NULL, NULL);
	344	case MT_RETROGRADE:
	345	return sysctl_io_number(req, mt_retrograde, sizeof(mt_retrograde), NULL, NULL);
	346	case MT_TASK_THREAD:
a39ff7e2	347	return sysctl_io_number(req, (int)mt_core_supported, sizeof(int), NULL, NULL);
5ba3f43e A	348	case MT_DEBUG: {
	349	int value = mt_debug;
	350
	351	int r = sysctl_io_number(req, value, sizeof(value), &value, NULL);
	352	if (r) {
	353	return r;
	354	}
	355	mt_debug = value;
	356
	357	return 0;
	358	}
	359	case MT_KDBG_TEST: {
	360	if (req->newptr == USER_ADDR_NULL) {
	361	return EINVAL;
	362	}
	363
	364	int intrs_en = ml_set_interrupts_enabled(FALSE);
	365	MT_KDBG_TMPCPU_START(0x3fff);
	366	MT_KDBG_TMPCPU_END(0x3fff);
	367
	368	MT_KDBG_TMPTH_START(0x3fff);
	369	MT_KDBG_TMPTH_END(0x3fff);
	370	ml_set_interrupts_enabled(intrs_en);
	371
	372	return 0;
	373	}
	374	case MT_FIX_CPU_PERF: {
	375	int intrs_en = ml_set_interrupts_enabled(FALSE);
	376	mt_fixed_counts(start);
	377	mt_fixed_counts(end);
	378	ml_set_interrupts_enabled(intrs_en);
	379
	380	goto copyout_counts;
	381	}
	382	case MT_FIX_THREAD_PERF: {
	383	int intrs_en = ml_set_interrupts_enabled(FALSE);
	384	mt_cur_thread_fixed_counts(start);
	385	mt_cur_thread_fixed_counts(end);
	386	ml_set_interrupts_enabled(intrs_en);
	387
	388	goto copyout_counts;
	389	}
	390	case MT_FIX_TASK_PERF: {
	391	int intrs_en = ml_set_interrupts_enabled(FALSE);
	392	mt_cur_task_fixed_counts(start);
	393	mt_cur_task_fixed_counts(end);
	394	ml_set_interrupts_enabled(intrs_en);
	395
	396	goto copyout_counts;
	397	}
	398	default:
	399	return ENOENT;
	400	}
	401
	402	copyout_counts:
	403
	404	#ifdef MT_CORE_INSTRS
	405	counts[0] = end[MT_CORE_INSTRS] - start[MT_CORE_INSTRS];
	406	#endif /* defined(MT_CORE_INSTRS) */
	407	counts[1] = end[MT_CORE_CYCLES] - start[MT_CORE_CYCLES];
	408
	409	return copyout(counts, req->oldptr, MIN(req->oldlen, sizeof(counts)));
	410	}
	411
412	SYSCTL_DECL(_kern_monotonic);
413	SYSCTL_NODE(_kern, OID_AUTO, monotonic, CTLFLAG_RW \| CTLFLAG_LOCKED, 0,
414	"monotonic");
415
416	SYSCTL_PROC(_kern_monotonic, OID_AUTO, supported,
417	CTLTYPE_INT \| CTLFLAG_RD \| CTLFLAG_MASKED \| CTLFLAG_LOCKED,
418	(void *)MT_SUPPORTED, sizeof(int), mt_sysctl, "I",
419	"whether monotonic is supported");
420
421	SYSCTL_PROC(_kern_monotonic, OID_AUTO, debug,
422	CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_MASKED,
423	(void *)MT_DEBUG, sizeof(int), mt_sysctl, "I",
424	"whether monotonic is printing debug messages");
425
426	SYSCTL_PROC(_kern_monotonic, OID_AUTO, pmis,
427	CTLTYPE_INT \| CTLFLAG_RD \| CTLFLAG_MASKED \| CTLFLAG_LOCKED,
428	(void *)MT_PMIS, sizeof(uint64_t), mt_sysctl, "Q",
429	"how many PMIs have been seen");
430
431	SYSCTL_PROC(_kern_monotonic, OID_AUTO, retrograde_updates,
432	CTLTYPE_INT \| CTLFLAG_RD \| CTLFLAG_MASKED \| CTLFLAG_LOCKED,
433	(void *)MT_RETROGRADE, sizeof(uint64_t), mt_sysctl, "Q",
434	"how many times a counter appeared to go backwards");
435
436	SYSCTL_PROC(_kern_monotonic, OID_AUTO, task_thread_counting,
437	CTLTYPE_INT \| CTLFLAG_RD \| CTLFLAG_MASKED,
438	(void *)MT_TASK_THREAD, sizeof(int), mt_sysctl, "I",
439	"task and thread counting enabled");
440
441	SYSCTL_PROC(_kern_monotonic, OID_AUTO, kdebug_test,
442	CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_MASKED \| CTLFLAG_LOCKED,
443	(void *)MT_KDBG_TEST, sizeof(int), mt_sysctl, "O",
444	"test that kdebug integration works");
445
446	SYSCTL_PROC(_kern_monotonic, OID_AUTO, fixed_cpu_perf,
447	CTLFLAG_RW \| CTLFLAG_MASKED \| CTLFLAG_LOCKED,
448	(void )MT_FIX_CPU_PERF, sizeof(uint64_t) 2, mt_sysctl, "O",
449	"overhead of accessing the current CPU's counters");
450
451	SYSCTL_PROC(_kern_monotonic, OID_AUTO, fixed_thread_perf,
452	CTLFLAG_RW \| CTLFLAG_MASKED \| CTLFLAG_LOCKED,
453	(void )MT_FIX_THREAD_PERF, sizeof(uint64_t) 2, mt_sysctl, "O",
454	"overhead of accessing the current thread's counters");
455
456	SYSCTL_PROC(_kern_monotonic, OID_AUTO, fixed_task_perf,
457	CTLFLAG_RW \| CTLFLAG_MASKED \| CTLFLAG_LOCKED,
458	(void )MT_FIX_TASK_PERF, sizeof(uint64_t) 2, mt_sysctl, "O",
459	"overhead of accessing the current task's counters");