[apple/xnu.git] / bsd / dev / dtrace / lockprof.c

/*
 * Copyright (c) 2019 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
#include <sys/ioctl.h>

#include <sys/stat.h>
#include <miscfs/devfs/devfs.h>
#include <sys/conf.h>
#include <sys/systm.h>
#include <sys/dtrace.h>
#include <sys/dtrace_impl.h>
#include <kern/lock_group.h>
#include <kern/lock_stat.h>

#if LOCK_STATS

#define LP_NODE "lockprof"

#define LOCKPROF_AFRAMES 3
#define LOCKPROF_LEN 64

static dtrace_provider_id_t lockprof_id;

decl_lck_mtx_data(extern, lck_grp_lock);
extern queue_head_t lck_grp_queue;
extern unsigned int lck_grp_cnt;

extern void lck_grp_reference(lck_grp_t *grp);
extern void lck_grp_deallocate(lck_grp_t *grp);

#define LOCKPROF_MAX 10000 /* maximum number of lockprof probes */
static uint32_t lockprof_count; /* current number of lockprof probes */

enum probe_flags {
	/*
	 * Counts time spent spinning/blocking
	 */
	TIME_EVENT = 0x01,
	/*
	 * Requires LCK_GRP_ATTR_STAT to be set on the lock
	 * group, either via lck_grp_attr_setsta on the lock group,
	 * or globally via the lcks=3 boot-arg
	 */
	STAT_NEEDED = 0x02
};

static const struct {
	const char *prefix;
	int flags;
	size_t count_offset;
	size_t stat_offset;
} probes[] = {
	{"spin-held-", 0, offsetof(lck_grp_t, lck_grp_spincnt), offsetof(lck_grp_stats_t, lgss_spin_held)},
	{"spin-miss-", 0, offsetof(lck_grp_t, lck_grp_spincnt), offsetof(lck_grp_stats_t, lgss_spin_miss)},
	{"spin-spin-", TIME_EVENT, offsetof(lck_grp_t, lck_grp_spincnt), offsetof(lck_grp_stats_t, lgss_spin_spin)},
	{"ticket-held-", 0, offsetof(lck_grp_t, lck_grp_ticketcnt), offsetof(lck_grp_stats_t, lgss_ticket_held)},
	{"ticket-miss-", 0, offsetof(lck_grp_t, lck_grp_ticketcnt), offsetof(lck_grp_stats_t, lgss_ticket_miss)},
	{"ticket-spin-", TIME_EVENT, offsetof(lck_grp_t, lck_grp_ticketcnt), offsetof(lck_grp_stats_t, lgss_ticket_spin)},
#if HAS_EXT_MUTEXES
	{"adaptive-held-", STAT_NEEDED, offsetof(lck_grp_t, lck_grp_mtxcnt), offsetof(lck_grp_stats_t, lgss_mtx_held)},
	{"adaptive-miss-", STAT_NEEDED, offsetof(lck_grp_t, lck_grp_mtxcnt), offsetof(lck_grp_stats_t, lgss_mtx_miss)},
	{"adaptive-wait-", STAT_NEEDED, offsetof(lck_grp_t, lck_grp_mtxcnt), offsetof(lck_grp_stats_t, lgss_mtx_wait)},
	{"adaptive-direct-wait-", STAT_NEEDED, offsetof(lck_grp_t, lck_grp_mtxcnt), offsetof(lck_grp_stats_t, lgss_mtx_direct_wait)},
#endif /* HAS_EXT_MUTEXES */
	{NULL, false, 0, 0}
};

/*
 * Default defined probes for counting events
 */
const static int hold_defaults[] = {
	10000 /* 10000 events */
};

/*
 * Default defined probes for time events
 */
const static struct {
	unsigned int time;
	const char *suffix;
	uint64_t mult;
} cont_defaults[] = {
	{100, "ms", NANOSEC / MILLISEC} /* 100 ms */
};

typedef struct lockprof_probe {
	int lockprof_kind;
	dtrace_id_t lockprof_id;
	uint64_t lockprof_limit;
	lck_grp_t *lockprof_grp;
} lockprof_probe_t;

void
lockprof_invoke(lck_grp_t *grp, lck_grp_stat_t *stat, uint64_t val)
{
	dtrace_probe(stat->lgs_probeid, (uintptr_t)grp, val, 0, 0, 0);
}

static int
lockprof_lock_count(lck_grp_t *grp, int kind)
{
	return *(int*)((void*)(grp) + probes[kind].count_offset);
}

static void
probe_create(int kind, const char *suffix, const char *grp_name, uint64_t count, uint64_t mult)
{
	char name[LOCKPROF_LEN];
	lck_mtx_lock(&lck_grp_lock);
	lck_grp_t *grp = (lck_grp_t*)queue_first(&lck_grp_queue);
	uint64_t limit = count * mult;

	if (probes[kind].flags & TIME_EVENT) {
		nanoseconds_to_absolutetime(limit, &limit);
	}

	for (unsigned int i = 0; i < lck_grp_cnt; i++, grp = (lck_grp_t*)queue_next((queue_entry_t)grp)) {
		if (!grp_name || grp_name[0] == '\0' || strcmp(grp_name, grp->lck_grp_name) == 0) {
			snprintf(name, sizeof(name), "%s%llu%s", probes[kind].prefix, count, suffix ?: "");

			if (dtrace_probe_lookup(lockprof_id, grp->lck_grp_name, NULL, name) != 0) {
				continue;
			}
			if (lockprof_lock_count(grp, kind) == 0) {
				continue;
			}
			if ((probes[kind].flags & STAT_NEEDED) && !(grp->lck_grp_attr & LCK_GRP_ATTR_STAT)) {
				continue;
			}
			if (lockprof_count >= LOCKPROF_MAX) {
				break;
			}

			lockprof_probe_t *probe = kmem_zalloc(sizeof(lockprof_probe_t), KM_SLEEP);
			probe->lockprof_kind = kind;
			probe->lockprof_limit = limit;
			probe->lockprof_grp = grp;

			lck_grp_reference(grp);

			probe->lockprof_id = dtrace_probe_create(lockprof_id, grp->lck_grp_name, NULL, name,
			    LOCKPROF_AFRAMES, probe);

			lockprof_count++;
		}
	}
	lck_mtx_unlock(&lck_grp_lock);
}

static void
lockprof_provide(void *arg, const dtrace_probedesc_t *desc)
{
#pragma unused(arg)
	size_t event_id, i, j, len;

	if (desc == NULL) {
		for (i = 0; i < sizeof(hold_defaults) / sizeof(hold_defaults[0]); i++) {
			for (j = 0; probes[j].prefix != NULL; j++) {
				if (!(probes[j].flags & TIME_EVENT)) {
					probe_create(j, NULL, NULL, hold_defaults[i], 1);
				}
			}
		}
		for (i = 0; i < sizeof(cont_defaults) / sizeof(cont_defaults[0]); i++) {
			for (j = 0; probes[j].prefix != NULL; j++) {
				if (probes[j].flags & TIME_EVENT) {
					probe_create(j, cont_defaults[i].suffix, NULL, cont_defaults[i].time, cont_defaults[i].mult);
				}
			}
		}
		return;
	}

	const char *name, *suffix = NULL;
	hrtime_t val = 0, mult = 1;

	const struct {
		const char *name;
		hrtime_t mult;
	} suffixes[] = {
		{ "us", NANOSEC / MICROSEC },
		{ "usec", NANOSEC / MICROSEC },
		{ "ms", NANOSEC / MILLISEC },
		{ "msec", NANOSEC / MILLISEC },
		{ "s", NANOSEC / SEC },
		{ "sec", NANOSEC / SEC },
		{ NULL, 0 }
	};

	name = desc->dtpd_name;

	for (event_id = 0; probes[event_id].prefix != NULL; event_id++) {
		len = strlen(probes[event_id].prefix);

		if (strncmp(name, probes[event_id].prefix, len) != 0) {
			continue;
		}
		break;
	}

	if (probes[event_id].prefix == NULL) {
		return;
	}


	/*
	 * We need to start before any time suffix.
	 */
	for (i = strlen(name); i >= len; i--) {
		if (name[i] >= '0' && name[i] <= '9') {
			break;
		}
		suffix = &name[i];
	}

	/*
	 * Now determine the numerical value present in the probe name.
	 */
	for (uint64_t m = 1; i >= len; i--) {
		if (name[i] < '0' || name[i] > '9') {
			return;
		}

		val += (name[i] - '0') * m;
		m *= (hrtime_t)10;
	}

	if (val == 0) {
		return;
	}

	if (probes[event_id].flags & TIME_EVENT) {
		for (i = 0, mult = 0; suffixes[i].name != NULL; i++) {
			if (strncasecmp(suffixes[i].name, suffix, strlen(suffixes[i].name) + 1) == 0) {
				mult = suffixes[i].mult;
				break;
			}
		}
		if (suffixes[i].name == NULL) {
			return;
		}
	} else if (*suffix != '\0') {
		return;
	}

	probe_create(event_id, suffix, desc->dtpd_mod, val, mult);
}


static lck_grp_stat_t*
lockprof_stat(lck_grp_t *grp, int kind)
{
	return (lck_grp_stat_t*)((void*)&grp->lck_grp_stats + probes[kind].stat_offset);
}

static int
lockprof_enable(void *arg, dtrace_id_t id, void *parg)
{
#pragma unused(arg, id, parg)
	lockprof_probe_t *probe = (lockprof_probe_t*)parg;
	lck_grp_t *grp = probe->lockprof_grp;
	lck_grp_stat_t *stat;

	if (grp == NULL) {
		return -1;
	}

	if ((stat = lockprof_stat(grp, probe->lockprof_kind)) == NULL) {
		return -1;
	}

	/*
	 * lockprof_enable/disable are called with
	 * dtrace_lock held
	 */
	if (stat->lgs_limit != 0) {
		return -1;
	}

	stat->lgs_limit = probe->lockprof_limit;
	stat->lgs_enablings++;
	stat->lgs_probeid = probe->lockprof_id;

	return 0;
}

static void
lockprof_disable(void *arg, dtrace_id_t id, void *parg)
{
#pragma unused(arg, id)
	lockprof_probe_t *probe = (lockprof_probe_t*)parg;
	lck_grp_t *grp = probe->lockprof_grp;
	lck_grp_stat_t *stat;

	if (grp == NULL) {
		return;
	}

	if ((stat = lockprof_stat(grp, probe->lockprof_kind)) == NULL) {
		return;
	}

	if (stat->lgs_limit == 0 || stat->lgs_enablings == 0) {
		return;
	}

	stat->lgs_limit = 0;
	stat->lgs_enablings--;
	stat->lgs_probeid = 0;
}

static void
lockprof_destroy(void *arg, dtrace_id_t id, void *parg)
{
#pragma unused(arg, id)
	lockprof_probe_t *probe = (lockprof_probe_t*)parg;
	lck_grp_deallocate(probe->lockprof_grp);
	kmem_free(probe, sizeof(lockprof_probe_t));
	lockprof_count--;
}

static void
lockprof_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc)
{
#pragma unused(arg, id, parg)
	const char *argdesc = NULL;
	switch (desc->dtargd_ndx) {
	case 0:
		argdesc = "lck_grp_t*";
		break;
	case 1:
		argdesc = "uint64_t";
		break;
	}

	if (argdesc) {
		strlcpy(desc->dtargd_native, argdesc, DTRACE_ARGTYPELEN);
	} else {
		desc->dtargd_ndx = DTRACE_ARGNONE;
	}
}
static dtrace_pattr_t lockprof_attr = {
	{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
	{ DTRACE_STABILITY_UNSTABLE, DTRACE_STABILITY_UNSTABLE, DTRACE_CLASS_UNKNOWN },
	{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
	{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
	{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
};

static dtrace_pops_t lockprof_pops = {
	.dtps_provide =         lockprof_provide,
	.dtps_provide_module =  NULL,
	.dtps_enable =          lockprof_enable,
	.dtps_disable =         lockprof_disable,
	.dtps_suspend =         NULL,
	.dtps_resume =          NULL,
	.dtps_getargdesc =      lockprof_getargdesc,
	.dtps_getargval =       NULL,
	.dtps_usermode =        NULL,
	.dtps_destroy =         lockprof_destroy
};

static int
_lockprof_open(dev_t dev, int flags, int devtype, struct proc *p)
{
#pragma unused(dev,flags,devtype,p)
	return 0;
}

static const struct cdevsw lockprof_cdevsw =
{
	.d_open = _lockprof_open,
	.d_close = eno_opcl,
	.d_read = eno_rdwrt,
	.d_write = eno_rdwrt,
	.d_ioctl = eno_ioctl,
	.d_stop = (stop_fcn_t *)nulldev,
	.d_reset = (reset_fcn_t *)nulldev,
	.d_select = eno_select,
	.d_mmap = eno_mmap,
	.d_strategy = eno_strat,
	.d_reserved_1 = eno_getc,
	.d_reserved_2 = eno_putc,
};


#endif /* LOCK_STATS */
void lockprof_init(void);
void
lockprof_init(void)
{
#if LOCK_STATS
	int majorno = cdevsw_add(-1, &lockprof_cdevsw);

	if (majorno < 0) {
		panic("dtrace: failed to allocate a major number");
		return;
	}

	if (dtrace_register(LP_NODE, &lockprof_attr, DTRACE_PRIV_KERNEL,
	    NULL, &lockprof_pops, NULL, &lockprof_id) != 0) {
		panic("dtrace: failed to register lockprof provider");
	}

	dev_t dev = makedev(majorno, 0);

	if (devfs_make_node( dev, DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0666,
	    LP_NODE, 0 ) == NULL) {
		panic("dtrace: devfs_make_node failed for lockprof");
	}

#endif /* LOCK_STATS */
}
Commit	Line	Data
0a7de745	1	/*
cb323159	2	* Copyright (c) 2019 Apple Inc. All rights reserved.
0a7de745 A	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
f427ee49 A	28	#include <sys/ioctl.h>
	29
	30	#include <sys/stat.h>
	31	#include <miscfs/devfs/devfs.h>
	32	#include <sys/conf.h>
	33	#include <sys/systm.h>
0a7de745 A	34	#include <sys/dtrace.h>
	35	#include <sys/dtrace_impl.h>
	36	#include <kern/lock_group.h>
	37	#include <kern/lock_stat.h>
	38
	39	#if LOCK_STATS
cb323159	40
f427ee49	41	#define LP_NODE "lockprof"
0a7de745	42
f427ee49 A	43	#define LOCKPROF_AFRAMES 3
f427ee49 A	44	#define LOCKPROF_LEN 64
0a7de745 A	45
	46	static dtrace_provider_id_t lockprof_id;
	47
cb323159	48	decl_lck_mtx_data(extern, lck_grp_lock);
0a7de745 A	49	extern queue_head_t lck_grp_queue;
	50	extern unsigned int lck_grp_cnt;
	51
f427ee49 A	52	extern void lck_grp_reference(lck_grp_t *grp);
	53	extern void lck_grp_deallocate(lck_grp_t *grp);
	54
0a7de745 A	55	#define LOCKPROF_MAX 10000 /* maximum number of lockprof probes */
	56	static uint32_t lockprof_count; /* current number of lockprof probes */
	57
f427ee49 A	58	enum probe_flags {
	59	/*
	60	* Counts time spent spinning/blocking
	61	*/
	62	TIME_EVENT = 0x01,
	63	/*
	64	* Requires LCK_GRP_ATTR_STAT to be set on the lock
	65	* group, either via lck_grp_attr_setsta on the lock group,
	66	* or globally via the lcks=3 boot-arg
	67	*/
	68	STAT_NEEDED = 0x02
	69	};
	70
0a7de745	71	static const struct {
0a7de745	72	const char *prefix;
f427ee49 A	73	int flags;
	74	size_t count_offset;
	75	size_t stat_offset;
	76	} probes[] = {
	77	{"spin-held-", 0, offsetof(lck_grp_t, lck_grp_spincnt), offsetof(lck_grp_stats_t, lgss_spin_held)},
	78	{"spin-miss-", 0, offsetof(lck_grp_t, lck_grp_spincnt), offsetof(lck_grp_stats_t, lgss_spin_miss)},
	79	{"spin-spin-", TIME_EVENT, offsetof(lck_grp_t, lck_grp_spincnt), offsetof(lck_grp_stats_t, lgss_spin_spin)},
	80	{"ticket-held-", 0, offsetof(lck_grp_t, lck_grp_ticketcnt), offsetof(lck_grp_stats_t, lgss_ticket_held)},
	81	{"ticket-miss-", 0, offsetof(lck_grp_t, lck_grp_ticketcnt), offsetof(lck_grp_stats_t, lgss_ticket_miss)},
	82	{"ticket-spin-", TIME_EVENT, offsetof(lck_grp_t, lck_grp_ticketcnt), offsetof(lck_grp_stats_t, lgss_ticket_spin)},
	83	#if HAS_EXT_MUTEXES
	84	{"adaptive-held-", STAT_NEEDED, offsetof(lck_grp_t, lck_grp_mtxcnt), offsetof(lck_grp_stats_t, lgss_mtx_held)},
	85	{"adaptive-miss-", STAT_NEEDED, offsetof(lck_grp_t, lck_grp_mtxcnt), offsetof(lck_grp_stats_t, lgss_mtx_miss)},
	86	{"adaptive-wait-", STAT_NEEDED, offsetof(lck_grp_t, lck_grp_mtxcnt), offsetof(lck_grp_stats_t, lgss_mtx_wait)},
	87	{"adaptive-direct-wait-", STAT_NEEDED, offsetof(lck_grp_t, lck_grp_mtxcnt), offsetof(lck_grp_stats_t, lgss_mtx_direct_wait)},
	88	#endif /* HAS_EXT_MUTEXES */
	89	{NULL, false, 0, 0}
0a7de745 A	90	};
0a7de745 A	91
f427ee49 A	92	/*
	93	* Default defined probes for counting events
	94	*/
0a7de745	95	const static int hold_defaults[] = {
f427ee49	96	10000 /* 10000 events */
0a7de745 A	97	};
0a7de745 A	98
f427ee49 A	99	/*
	100	* Default defined probes for time events
	101	*/
0a7de745 A	102	const static struct {
	103	unsigned int time;
	104	const char *suffix;
	105	uint64_t mult;
	106	} cont_defaults[] = {
f427ee49	107	{100, "ms", NANOSEC / MILLISEC} /* 100 ms */
0a7de745 A	108	};
	109
	110	typedef struct lockprof_probe {
	111	int lockprof_kind;
	112	dtrace_id_t lockprof_id;
	113	uint64_t lockprof_limit;
	114	lck_grp_t *lockprof_grp;
	115	} lockprof_probe_t;
	116
	117	void
	118	lockprof_invoke(lck_grp_t grp, lck_grp_stat_t stat, uint64_t val)
	119	{
	120	dtrace_probe(stat->lgs_probeid, (uintptr_t)grp, val, 0, 0, 0);
	121	}
	122
f427ee49 A	123	static int
	124	lockprof_lock_count(lck_grp_t *grp, int kind)
	125	{
	126	return (int)((void*)(grp) + probes[kind].count_offset);
	127	}
	128
0a7de745 A	129	static void
	130	probe_create(int kind, const char suffix, const char grp_name, uint64_t count, uint64_t mult)
	131	{
f427ee49	132	char name[LOCKPROF_LEN];
0a7de745 A	133	lck_mtx_lock(&lck_grp_lock);
	134	lck_grp_t grp = (lck_grp_t)queue_first(&lck_grp_queue);
	135	uint64_t limit = count * mult;
	136
f427ee49	137	if (probes[kind].flags & TIME_EVENT) {
0a7de745 A	138	nanoseconds_to_absolutetime(limit, &limit);
	139	}
	140
	141	for (unsigned int i = 0; i < lck_grp_cnt; i++, grp = (lck_grp_t*)queue_next((queue_entry_t)grp)) {
	142	if (!grp_name \|\| grp_name[0] == '\0' \|\| strcmp(grp_name, grp->lck_grp_name) == 0) {
f427ee49	143	snprintf(name, sizeof(name), "%s%llu%s", probes[kind].prefix, count, suffix ?: "");
0a7de745 A	144
	145	if (dtrace_probe_lookup(lockprof_id, grp->lck_grp_name, NULL, name) != 0) {
	146	continue;
	147	}
f427ee49 A	148	if (lockprof_lock_count(grp, kind) == 0) {
	149	continue;
	150	}
	151	if ((probes[kind].flags & STAT_NEEDED) && !(grp->lck_grp_attr & LCK_GRP_ATTR_STAT)) {
	152	continue;
	153	}
0a7de745 A	154	if (lockprof_count >= LOCKPROF_MAX) {
	155	break;
	156	}
	157
	158	lockprof_probe_t *probe = kmem_zalloc(sizeof(lockprof_probe_t), KM_SLEEP);
	159	probe->lockprof_kind = kind;
	160	probe->lockprof_limit = limit;
	161	probe->lockprof_grp = grp;
	162
f427ee49 A	163	lck_grp_reference(grp);
f427ee49 A	164
0a7de745	165	probe->lockprof_id = dtrace_probe_create(lockprof_id, grp->lck_grp_name, NULL, name,
f427ee49	166	LOCKPROF_AFRAMES, probe);
0a7de745 A	167
	168	lockprof_count++;
	169	}
	170	}
	171	lck_mtx_unlock(&lck_grp_lock);
	172	}
	173
	174	static void
	175	lockprof_provide(void arg, const dtrace_probedesc_t desc)
	176	{
	177	#pragma unused(arg)
f427ee49	178	size_t event_id, i, j, len;
0a7de745 A	179
	180	if (desc == NULL) {
	181	for (i = 0; i < sizeof(hold_defaults) / sizeof(hold_defaults[0]); i++) {
f427ee49 A	182	for (j = 0; probes[j].prefix != NULL; j++) {
	183	if (!(probes[j].flags & TIME_EVENT)) {
	184	probe_create(j, NULL, NULL, hold_defaults[i], 1);
	185	}
	186	}
0a7de745 A	187	}
0a7de745 A	188	for (i = 0; i < sizeof(cont_defaults) / sizeof(cont_defaults[0]); i++) {
f427ee49 A	189	for (j = 0; probes[j].prefix != NULL; j++) {
	190	if (probes[j].flags & TIME_EVENT) {
	191	probe_create(j, cont_defaults[i].suffix, NULL, cont_defaults[i].time, cont_defaults[i].mult);
	192	}
	193	}
0a7de745 A	194	}
	195	return;
	196	}
	197
	198	const char name, suffix = NULL;
	199	hrtime_t val = 0, mult = 1;
	200
	201	const struct {
	202	const char *name;
	203	hrtime_t mult;
	204	} suffixes[] = {
	205	{ "us", NANOSEC / MICROSEC },
	206	{ "usec", NANOSEC / MICROSEC },
	207	{ "ms", NANOSEC / MILLISEC },
	208	{ "msec", NANOSEC / MILLISEC },
	209	{ "s", NANOSEC / SEC },
	210	{ "sec", NANOSEC / SEC },
	211	{ NULL, 0 }
	212	};
	213
	214	name = desc->dtpd_name;
	215
f427ee49 A	216	for (event_id = 0; probes[event_id].prefix != NULL; event_id++) {
f427ee49 A	217	len = strlen(probes[event_id].prefix);
0a7de745	218
f427ee49	219	if (strncmp(name, probes[event_id].prefix, len) != 0) {
0a7de745 A	220	continue;
	221	}
	222	break;
	223	}
	224
f427ee49	225	if (probes[event_id].prefix == NULL) {
0a7de745 A	226	return;
	227	}
	228
	229
	230	/*
	231	* We need to start before any time suffix.
	232	*/
	233	for (i = strlen(name); i >= len; i--) {
	234	if (name[i] >= '0' && name[i] <= '9') {
	235	break;
	236	}
	237	suffix = &name[i];
	238	}
	239
	240	/*
	241	* Now determine the numerical value present in the probe name.
	242	*/
	243	for (uint64_t m = 1; i >= len; i--) {
	244	if (name[i] < '0' \|\| name[i] > '9') {
	245	return;
	246	}
	247
	248	val += (name[i] - '0') * m;
	249	m *= (hrtime_t)10;
	250	}
	251
	252	if (val == 0) {
	253	return;
	254	}
	255
f427ee49	256	if (probes[event_id].flags & TIME_EVENT) {
0a7de745 A	257	for (i = 0, mult = 0; suffixes[i].name != NULL; i++) {
	258	if (strncasecmp(suffixes[i].name, suffix, strlen(suffixes[i].name) + 1) == 0) {
	259	mult = suffixes[i].mult;
	260	break;
	261	}
	262	}
	263	if (suffixes[i].name == NULL) {
	264	return;
	265	}
	266	} else if (*suffix != '\0') {
	267	return;
	268	}
	269
f427ee49	270	probe_create(event_id, suffix, desc->dtpd_mod, val, mult);
0a7de745 A	271	}
	272
	273
	274	static lck_grp_stat_t*
	275	lockprof_stat(lck_grp_t *grp, int kind)
	276	{
f427ee49	277	return (lck_grp_stat_t)((void)&grp->lck_grp_stats + probes[kind].stat_offset);
0a7de745 A	278	}
	279
	280	static int
	281	lockprof_enable(void arg, dtrace_id_t id, void parg)
	282	{
	283	#pragma unused(arg, id, parg)
	284	lockprof_probe_t probe = (lockprof_probe_t)parg;
	285	lck_grp_t *grp = probe->lockprof_grp;
	286	lck_grp_stat_t *stat;
	287
	288	if (grp == NULL) {
	289	return -1;
	290	}
	291
	292	if ((stat = lockprof_stat(grp, probe->lockprof_kind)) == NULL) {
	293	return -1;
	294	}
	295
	296	/*
	297	* lockprof_enable/disable are called with
	298	* dtrace_lock held
	299	*/
	300	if (stat->lgs_limit != 0) {
	301	return -1;
	302	}
	303
	304	stat->lgs_limit = probe->lockprof_limit;
	305	stat->lgs_enablings++;
	306	stat->lgs_probeid = probe->lockprof_id;
	307
	308	return 0;
	309	}
	310
	311	static void
	312	lockprof_disable(void arg, dtrace_id_t id, void parg)
	313	{
	314	#pragma unused(arg, id)
	315	lockprof_probe_t probe = (lockprof_probe_t)parg;
	316	lck_grp_t *grp = probe->lockprof_grp;
	317	lck_grp_stat_t *stat;
	318
	319	if (grp == NULL) {
	320	return;
	321	}
	322
	323	if ((stat = lockprof_stat(grp, probe->lockprof_kind)) == NULL) {
	324	return;
	325	}
	326
	327	if (stat->lgs_limit == 0 \|\| stat->lgs_enablings == 0) {
	328	return;
	329	}
	330
	331	stat->lgs_limit = 0;
	332	stat->lgs_enablings--;
	333	stat->lgs_probeid = 0;
	334	}
	335
	336	static void
	337	lockprof_destroy(void arg, dtrace_id_t id, void parg)
	338	{
	339	#pragma unused(arg, id)
	340	lockprof_probe_t probe = (lockprof_probe_t)parg;
f427ee49	341	lck_grp_deallocate(probe->lockprof_grp);
0a7de745 A	342	kmem_free(probe, sizeof(lockprof_probe_t));
	343	lockprof_count--;
	344	}
	345
	346	static void
	347	lockprof_getargdesc(void arg, dtrace_id_t id, void parg, dtrace_argdesc_t *desc)
	348	{
	349	#pragma unused(arg, id, parg)
	350	const char *argdesc = NULL;
	351	switch (desc->dtargd_ndx) {
	352	case 0:
	353	argdesc = "lck_grp_t*";
	354	break;
	355	case 1:
	356	argdesc = "uint64_t";
	357	break;
	358	}
	359
	360	if (argdesc) {
	361	strlcpy(desc->dtargd_native, argdesc, DTRACE_ARGTYPELEN);
	362	} else {
	363	desc->dtargd_ndx = DTRACE_ARGNONE;
	364	}
	365	}
	366	static dtrace_pattr_t lockprof_attr = {
	367	{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
	368	{ DTRACE_STABILITY_UNSTABLE, DTRACE_STABILITY_UNSTABLE, DTRACE_CLASS_UNKNOWN },
	369	{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
	370	{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
	371	{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
	372	};
	373
	374	static dtrace_pops_t lockprof_pops = {
	375	.dtps_provide = lockprof_provide,
	376	.dtps_provide_module = NULL,
	377	.dtps_enable = lockprof_enable,
	378	.dtps_disable = lockprof_disable,
	379	.dtps_suspend = NULL,
	380	.dtps_resume = NULL,
	381	.dtps_getargdesc = lockprof_getargdesc,
	382	.dtps_getargval = NULL,
	383	.dtps_usermode = NULL,
	384	.dtps_destroy = lockprof_destroy
	385	};
f427ee49 A	386
	387	static int
	388	_lockprof_open(dev_t dev, int flags, int devtype, struct proc *p)
	389	{
	390	#pragma unused(dev,flags,devtype,p)
	391	return 0;
	392	}
	393
	394	static const struct cdevsw lockprof_cdevsw =
	395	{
	396	.d_open = _lockprof_open,
	397	.d_close = eno_opcl,
	398	.d_read = eno_rdwrt,
	399	.d_write = eno_rdwrt,
	400	.d_ioctl = eno_ioctl,
	401	.d_stop = (stop_fcn_t *)nulldev,
	402	.d_reset = (reset_fcn_t *)nulldev,
	403	.d_select = eno_select,
	404	.d_mmap = eno_mmap,
	405	.d_strategy = eno_strat,
	406	.d_reserved_1 = eno_getc,
	407	.d_reserved_2 = eno_putc,
	408	};
	409
	410
0a7de745 A	411	#endif /* LOCK_STATS */
	412	void lockprof_init(void);
	413	void
	414	lockprof_init(void)
	415	{
	416	#if LOCK_STATS
f427ee49 A	417	int majorno = cdevsw_add(-1, &lockprof_cdevsw);
	418
	419	if (majorno < 0) {
	420	panic("dtrace: failed to allocate a major number");
	421	return;
	422	}
	423
	424	if (dtrace_register(LP_NODE, &lockprof_attr, DTRACE_PRIV_KERNEL,
	425	NULL, &lockprof_pops, NULL, &lockprof_id) != 0) {
	426	panic("dtrace: failed to register lockprof provider");
	427	}
	428
	429	dev_t dev = makedev(majorno, 0);
	430
	431	if (devfs_make_node( dev, DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0666,
	432	LP_NODE, 0 ) == NULL) {
	433	panic("dtrace: devfs_make_node failed for lockprof");
	434	}
	435
0a7de745 A	436	#endif /* LOCK_STATS */
0a7de745 A	437	}