[apple/xnu.git] / bsd / dev / dtrace / lockstat.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/* #pragma ident	"@(#)lockstat.c	1.12	08/01/16 SMI" */


#ifdef KERNEL
#ifndef _KERNEL
#define _KERNEL /* Solaris vs. Darwin */
#endif
#endif

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/errno.h>
#include <sys/stat.h>
#include <sys/ioctl.h>
#include <sys/conf.h>
#include <sys/fcntl.h>
#include <miscfs/devfs/devfs.h>

#include <sys/dtrace.h>
#include <sys/dtrace_impl.h>

#include <sys/dtrace_glue.h>

#include <sys/lockstat.h>

#include <kern/processor.h>

#define membar_producer dtrace_membar_producer

#define PROBE_ARGS0(a, b, c, d, e) "\000"
#define PROBE_ARGS1(a, b, c, d, e) a "\000"
#define PROBE_ARGS2(a, b, c, d, e) a "\000" b "\000"
#define PROBE_ARGS3(a, b, c, d, e) a "\000" b "\000" c "\000"
#define PROBE_ARGS4(a, b, c, d, e) a "\000" b "\000" c "\000" d "\000"
#define PROBE_ARGS5(a, b, c, d, e) a "\000" b "\000" c "\000" d "\000" e "\000"
#define PROBE_ARGS_(a, b, c, d, e, n, ...) PROBE_ARGS##n(a, b, c, d, e)
#define PROBE_ARGS(...) PROBE_ARGS_(__VA_ARGS__, 5, 4, 3, 2, 1, 0)

#define LOCKSTAT_PROBE(func, name, probe, ...) \
	{func, name, probe, DTRACE_IDNONE, PROBE_ARGS(__VA_ARGS__)}

/*
 * Hot patch values, x86
 */
#if defined(__x86_64__)
#define	NOP	0x90
#define	RET	0xc3
#define LOCKSTAT_AFRAMES 1
#elif	defined(__arm__)
#define NOP	0xE1A00000
#define BXLR	0xE12FFF1E
#define LOCKSTAT_AFRAMES 2
#elif   defined(__arm64__)
#define NOP	0xD503201F
#define RET	0xD65f03c0
#define LOCKSTAT_AFRAMES 2
#else
#error "not ported to this architecture"
#endif

typedef struct lockstat_probe {
	const char	*lsp_func;
	const char	*lsp_name;
	int		lsp_probe;
	dtrace_id_t	lsp_id;
	const char	*lsp_args;
} lockstat_probe_t;

lockstat_probe_t lockstat_probes[] =
{
#if defined(__x86_64__)
	/* Only provide implemented probes for each architecture  */
	LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_ACQUIRE, "lck_mtx_t"),
	LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_SPIN, LS_LCK_MTX_LOCK_SPIN, "lck_mtx_t", "uint64_t"),
	LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_BLOCK, LS_LCK_MTX_LOCK_BLOCK, "lck_mtx_t", "uint64_t"),
	LOCKSTAT_PROBE(LS_LCK_MTX_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_LOCK_ACQUIRE, "lck_mtx_t"),
	LOCKSTAT_PROBE(LS_LCK_MTX_TRY_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE, "lck_mtx_t"),
	LOCKSTAT_PROBE(LS_LCK_MTX_UNLOCK, LSA_RELEASE, LS_LCK_MTX_UNLOCK_RELEASE, "lck_mtx_t"),
	LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_ACQUIRE, LS_LCK_MTX_EXT_LOCK_ACQUIRE, "lck_mtx_t"),
	LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_SPIN, LS_LCK_MTX_EXT_LOCK_SPIN, "lck_mtx_t", "uint64_t"),
	LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_BLOCK, LS_LCK_MTX_EXT_LOCK_BLOCK, "lck_mtx_t", "uint64_t"),
//	LOCKSTAT_PROBE(LS_LCK_MTX_EXT_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_EXT_LOCK_ACQUIRE)
	LOCKSTAT_PROBE(LS_LCK_MTX_EXT_UNLOCK, LSA_RELEASE, LS_LCK_MTX_EXT_UNLOCK_RELEASE, "lck_mtx_t"),
	LOCKSTAT_PROBE(LS_LCK_MTX_LOCK_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_SPIN_ACQUIRE, "lck_mtx_t"),
	// TODO: This should not be a uint64_t !
	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_LOCK_SHARED_ACQUIRE, "lck_rw_t", "uint64_t"),
	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_SPIN, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
	// TODO: This should NOT be a uint64_t
	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_LOCK_EXCL_ACQUIRE, "lck_rw_t", "uint64_t"),
	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_EXCL_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_EXCL_SPIN, "lck_rw_t", "uint64_t", "int"),
	LOCKSTAT_PROBE(LS_LCK_RW_DONE, LSR_RELEASE, LS_LCK_RW_DONE_RELEASE, "lck_rw_t", "_Bool"),
	// TODO : This should NOT be a uint64_t
	LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, "lck_rw_t", "uint64_t"),
	// See above
	LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, "lck_rw_t", "uint64_t"),
	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_UPGRADE, LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, "lck_rw_t", "_Bool"),
	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN, "lck_rw_t", "uint64_t"),
	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL_TO_SHARED, LSR_DOWNGRADE, LS_LCK_RW_LOCK_EXCL_TO_SHARED_DOWNGRADE, "lck_rw_t"),
	//TODO : Separate the probes for the hw_bit from the probe for the normal hw locks
	LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK, LSS_ACQUIRE, LS_LCK_SPIN_LOCK_ACQUIRE, "hw_lock_t"),
	LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK, LSS_SPIN, LS_LCK_SPIN_LOCK_SPIN, "hw_lock_t", "uint64_t", "uint64_t"),
	LOCKSTAT_PROBE(LS_LCK_SPIN_UNLOCK, LSS_RELEASE, LS_LCK_SPIN_UNLOCK_RELEASE, "hw_lock_t"),
#elif defined(__arm__) || defined(__arm64__)
	LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_ACQUIRE, "lck_mtx_t"),
//	LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_SPIN, LS_LCK_MTX_LOCK_SPIN, "lck_mtx_t", "uint64_t"),
	LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_BLOCK, LS_LCK_MTX_LOCK_BLOCK, "lck_mtx_t", "uint64_t"),
	LOCKSTAT_PROBE(LS_LCK_MTX_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_LOCK_ACQUIRE, "lck_mtx_t"),
//	LOCKSTAT_PROBE(LS_LCK_MTX_TRY_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE, "lck_mtx_t"),
	LOCKSTAT_PROBE(LS_LCK_MTX_UNLOCK, LSA_RELEASE, LS_LCK_MTX_UNLOCK_RELEASE, "lck_mtx_t"),
	LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_ACQUIRE, LS_LCK_MTX_EXT_LOCK_ACQUIRE, "lck_mtx_t"),
//	LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_SPIN, LS_LCK_MTX_EXT_LOCK_SPIN, "lck_mtx_t", "uint64_t"),
	LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_BLOCK, LS_LCK_MTX_EXT_LOCK_BLOCK, "lck_mtx_t", "uint64_t"),
//	LOCKSTAT_PROBE(LS_LCK_MTX_EXT_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_EXT_LOCK_ACQUIRE)
//	LOCKSTAT_PROBE(LS_LCK_MTX_EXT_UNLOCK, LSA_RELEASE, LS_LCK_MTX_EXT_UNLOCK_RELEASE, "lck_mtx_t"),
//	LOCKSTAT_PROBE(LS_LCK_MTX_LOCK_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_SPIN_ACQUIRE, "lck_mtx_t"),
	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_LOCK_SHARED_ACQUIRE, "lck_rw_t", "uint64_t"),
	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_SPIN, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_LOCK_EXCL_ACQUIRE, "lck_rw_t", "uint64_t"),
	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_EXCL_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_EXCL_SPIN, "lck_rw_t", "uint64_t", "int"),
	LOCKSTAT_PROBE(LS_LCK_RW_DONE, LSR_RELEASE, LS_LCK_RW_DONE_RELEASE, "lck_rw_t", "_Bool"),
	// TODO : This should NOT be a uint64_t
	LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, "lck_rw_t", "uint64_t"),
	// See above
	LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, "lck_rw_t", "uint64_t"),
	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_UPGRADE, LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, "lck_rw_t", "_Bool"),
	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN, "lck_rw_t", "uint64_t"),
	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL_TO_SHARED, LSR_DOWNGRADE, LS_LCK_RW_LOCK_EXCL_TO_SHARED_DOWNGRADE, "lck_rw_t"),
	//TODO : Separate the probes for the hw_bit from the probe for the normal hw locks
	LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK, LSS_ACQUIRE, LS_LCK_SPIN_LOCK_ACQUIRE, "hw_lock_t"),
	LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK, LSS_SPIN, LS_LCK_SPIN_LOCK_SPIN, "hw_lock_t", "uint64_t", "uint64_t"),
	LOCKSTAT_PROBE(LS_LCK_SPIN_UNLOCK, LSS_RELEASE, LS_LCK_SPIN_UNLOCK_RELEASE, "hw_lock_t"),
#endif
	/* Interlock measurements would be nice, but later */

#ifdef	LATER
	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL_TO_SHARED, LSA_ILK_SPIN, LS_LCK_RW_LOCK_EXCL_TO_SHARED_ILK_SPIN),
	LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_ILK_SPIN, LS_LCK_MTX_LOCK_ILK_SPIN),
	LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_ILK_SPIN, LS_LCK_MTX_EXT_LOCK_ILK_SPIN),
	LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_EXCL, LSA_ILK_SPIN, LS_LCK_RW_TRY_LOCK_EXCL_ILK_SPIN),
	LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_SHARED, LSA_SPIN, LS_LCK_RW_TRY_LOCK_SHARED_SPIN)
#endif

	{ NULL, NULL, 0, 0, NULL}
};

dtrace_id_t lockstat_probemap[LS_NPROBES];

typedef struct lockstat_assembly_probe {
	int lsap_probe;
	vm_offset_t * lsap_patch_point;
} lockstat_assembly_probe_t;
	

	lockstat_assembly_probe_t assembly_probes[] =
	{
		{ LS_LCK_INVALID, NULL }
	};


/*
 * APPLE NOTE:
 * Hot patch is used to manipulate probe points by swapping between
 * no-op and return instructions.
 * The active flag indicates whether the probe point will turn on or off.
 *	on == plant a NOP and thus fall through to the probe call
 *     off == plant a RET and thus avoid the probe call completely
 * The ls_probe identifies which probe we will patch.
 */
static
void lockstat_hot_patch(boolean_t active, int ls_probe)
{
#pragma unused(active)
	int i;

	/*
	 * Loop through entire table, in case there are
	 * multiple patch points per probe. 
	 */
	for (i = 0; assembly_probes[i].lsap_patch_point; i++) {
		if (ls_probe == assembly_probes[i].lsap_probe)
#if defined(__x86_64__)
		{			
			uint8_t instr;
			instr = (active ? NOP : RET );
			(void) ml_nofault_copy( (vm_offset_t)&instr, *(assembly_probes[i].lsap_patch_point), 
								sizeof(instr));
		}
#elif defined (__arm__)
		{
			uint32_t instr;
			instr = (active ? NOP : BXLR );
			(void) ml_nofault_copy( (vm_offset_t)&instr, *(assembly_probes[i].lsap_patch_point), 
								sizeof(instr));
		}
#elif defined (__arm64__)
		{
			uint32_t instr;
			instr = (active ? NOP : RET );
			(void) ml_nofault_copy( (vm_offset_t)&instr, *(assembly_probes[i].lsap_patch_point), 
								sizeof(instr));
		}
#endif
	} /* for */
}

void (*lockstat_probe)(dtrace_id_t, uint64_t, uint64_t,
				    uint64_t, uint64_t, uint64_t);


/*
 * APPLE NOTE:
 * This wrapper is used only by assembler hot patched probes.
 */
void
lockstat_probe_wrapper(int probe, uintptr_t lp, int rwflag)
{
	dtrace_id_t id;
	id = lockstat_probemap[probe];
	if (id != 0)
	{
		(*lockstat_probe)(id, (uintptr_t)lp, (uint64_t)rwflag, 0,0,0);
	}
}

static dtrace_provider_id_t lockstat_id;

/*ARGSUSED*/
static int
lockstat_enable(void *arg, dtrace_id_t id, void *parg)
{
#pragma unused(arg) /* __APPLE__ */
    
	lockstat_probe_t *probe = parg;

	ASSERT(!lockstat_probemap[probe->lsp_probe]);

	lockstat_probemap[probe->lsp_probe] = id;
	membar_producer();

	lockstat_hot_patch(TRUE, probe->lsp_probe);
	membar_producer();
	return(0);

}

/*ARGSUSED*/
static void
lockstat_disable(void *arg, dtrace_id_t id, void *parg)
{
#pragma unused(arg, id) /* __APPLE__ */

	lockstat_probe_t *probe = parg;
	int i;

	ASSERT(lockstat_probemap[probe->lsp_probe]);

	lockstat_probemap[probe->lsp_probe] = 0;
	lockstat_hot_patch(FALSE, probe->lsp_probe);
	membar_producer();

	/*
	 * See if we have any probes left enabled.
	 */
	for (i = 0; i < LS_NPROBES; i++) {
		if (lockstat_probemap[i]) {
			/*
			 * This probe is still enabled.  We don't need to deal
			 * with waiting for all threads to be out of the
			 * lockstat critical sections; just return.
			 */
			return;
		}
	}

}

/*ARGSUSED*/
static void
lockstat_provide(void *arg, const dtrace_probedesc_t *desc)
{
#pragma unused(arg, desc) /* __APPLE__ */
    
	int i = 0;

	for (i = 0; lockstat_probes[i].lsp_func != NULL; i++) {
		lockstat_probe_t *probe = &lockstat_probes[i];

		if (dtrace_probe_lookup(lockstat_id, "mach_kernel",
		    probe->lsp_func, probe->lsp_name) != 0)
			continue;

		ASSERT(!probe->lsp_id);
		probe->lsp_id = dtrace_probe_create(lockstat_id,
		    "mach_kernel", probe->lsp_func, probe->lsp_name,
		    LOCKSTAT_AFRAMES, probe);
	}
}


/*ARGSUSED*/
static void
lockstat_destroy(void *arg, dtrace_id_t id, void *parg)
{
#pragma unused(arg, id) /* __APPLE__ */
    
	lockstat_probe_t *probe = parg;

	ASSERT(!lockstat_probemap[probe->lsp_probe]);
	probe->lsp_id = 0;
}

static void
lockstat_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc)
{
#pragma unused(arg, id)
	lockstat_probe_t *probe = parg;
	const char* argdesc = probe->lsp_args;
	int narg = 0;

	desc->dtargd_native[0] = '\0';
	desc->dtargd_xlate[0] = '\0';

	while(argdesc[0] != '\0') {
		if (narg == desc->dtargd_ndx) {
			strlcpy(desc->dtargd_native, argdesc, DTRACE_ARGTYPELEN);
			return;
		}
		argdesc += strlen(argdesc) + 1;
		narg++;
	}

	desc->dtargd_ndx = DTRACE_ARGNONE;
}

static dtrace_pattr_t lockstat_attr = {
{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
};

static dtrace_pops_t lockstat_pops = {
	.dtps_provide =		lockstat_provide,
	.dtps_provide_module =	NULL,
	.dtps_enable =		lockstat_enable,
	.dtps_disable =		lockstat_disable,
	.dtps_suspend =		NULL,
	.dtps_resume =		NULL,
	.dtps_getargdesc =	lockstat_getargdesc,
	.dtps_getargval =	NULL,
	.dtps_usermode =	NULL,
	.dtps_destroy =		lockstat_destroy
};

static int
lockstat_attach(dev_info_t *devi)
{
	if (ddi_create_minor_node(devi, "lockstat", S_IFCHR, 0,
	    DDI_PSEUDO, 0) == DDI_FAILURE ||
	    dtrace_register("lockstat", &lockstat_attr, DTRACE_PRIV_KERNEL,
	    NULL, &lockstat_pops, NULL, &lockstat_id) != 0) {
		ddi_remove_minor_node(devi, NULL);
		return (DDI_FAILURE);
	}

	lockstat_probe = dtrace_probe;
	membar_producer();

	return (DDI_SUCCESS);
}

d_open_t _lockstat_open;

int _lockstat_open(dev_t dev, int flags, int devtype, struct proc *p)
{
#pragma unused(dev,flags,devtype,p)
	return 0;
}

#define LOCKSTAT_MAJOR  -24 /* let the kernel pick the device number */

/*
 * A struct describing which functions will get invoked for certain
 * actions.
 */
static struct cdevsw lockstat_cdevsw =
{
	_lockstat_open,		/* open */
	eno_opcl,			/* close */
	eno_rdwrt,			/* read */
	eno_rdwrt,			/* write */
	eno_ioctl,			/* ioctl */
	(stop_fcn_t *)nulldev, /* stop */
	(reset_fcn_t *)nulldev, /* reset */
	NULL,				/* tty's */
	eno_select,			/* select */
	eno_mmap,			/* mmap */
	eno_strat,			/* strategy */
	eno_getc,			/* getc */
	eno_putc,			/* putc */
	0					/* type */
};

void lockstat_init( void );

void lockstat_init( void )
{
	int majdevno = cdevsw_add(LOCKSTAT_MAJOR, &lockstat_cdevsw);

	if (majdevno < 0) {
		printf("lockstat_init: failed to allocate a major number!\n");
		return;
	}

	lockstat_attach((dev_info_t*)(uintptr_t)majdevno);
}
#undef LOCKSTAT_MAJOR
Commit	Line	Data
2d21ac55 A	1	/*
	2	* CDDL HEADER START
	3	*
	4	* The contents of this file are subject to the terms of the
	5	* Common Development and Distribution License (the "License").
	6	* You may not use this file except in compliance with the License.
	7	*
	8	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	9	* or http://www.opensolaris.org/os/licensing.
	10	* See the License for the specific language governing permissions
	11	* and limitations under the License.
	12	*
	13	* When distributing Covered Code, include this CDDL HEADER in each
	14	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	15	* If applicable, add the following below this CDDL HEADER, with the
	16	* fields enclosed by brackets "[]" replaced with your own identifying
	17	* information: Portions Copyright [yyyy] [name of copyright owner]
	18	*
	19	* CDDL HEADER END
	20	*/
	21	/*
6d2010ae	22	* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
2d21ac55 A	23	* Use is subject to license terms.
	24	*/
	25
b0d623f7	26	/* #pragma ident "@(#)lockstat.c 1.12 08/01/16 SMI" */
2d21ac55 A	27
	28
	29	#ifdef KERNEL
	30	#ifndef _KERNEL
	31	#define _KERNEL /* Solaris vs. Darwin */
	32	#endif
	33	#endif
	34
	35	#include <sys/param.h>
	36	#include <sys/systm.h>
	37	#include <sys/errno.h>
	38	#include <sys/stat.h>
	39	#include <sys/ioctl.h>
	40	#include <sys/conf.h>
	41	#include <sys/fcntl.h>
	42	#include <miscfs/devfs/devfs.h>
	43
	44	#include <sys/dtrace.h>
	45	#include <sys/dtrace_impl.h>
	46
	47	#include <sys/dtrace_glue.h>
	48
	49	#include <sys/lockstat.h>
	50
	51	#include <kern/processor.h>
	52
	53	#define membar_producer dtrace_membar_producer
	54
5ba3f43e A	55	#define PROBE_ARGS0(a, b, c, d, e) "\000"
	56	#define PROBE_ARGS1(a, b, c, d, e) a "\000"
	57	#define PROBE_ARGS2(a, b, c, d, e) a "\000" b "\000"
	58	#define PROBE_ARGS3(a, b, c, d, e) a "\000" b "\000" c "\000"
	59	#define PROBE_ARGS4(a, b, c, d, e) a "\000" b "\000" c "\000" d "\000"
	60	#define PROBE_ARGS5(a, b, c, d, e) a "\000" b "\000" c "\000" d "\000" e "\000"
	61	#define PROBE_ARGS_(a, b, c, d, e, n, ...) PROBE_ARGS##n(a, b, c, d, e)
	62	#define PROBE_ARGS(...) PROBE_ARGS_(__VA_ARGS__, 5, 4, 3, 2, 1, 0)
	63
	64	#define LOCKSTAT_PROBE(func, name, probe, ...) \
	65	{func, name, probe, DTRACE_IDNONE, PROBE_ARGS(__VA_ARGS__)}
	66
2d21ac55 A	67	/*
	68	* Hot patch values, x86
	69	*/
39236c6e	70	#if defined(__x86_64__)
2d21ac55 A	71	#define NOP 0x90
	72	#define RET 0xc3
	73	#define LOCKSTAT_AFRAMES 1
5ba3f43e A	74	#elif defined(__arm__)
	75	#define NOP 0xE1A00000
	76	#define BXLR 0xE12FFF1E
	77	#define LOCKSTAT_AFRAMES 2
	78	#elif defined(__arm64__)
	79	#define NOP 0xD503201F
	80	#define RET 0xD65f03c0
	81	#define LOCKSTAT_AFRAMES 2
2d21ac55 A	82	#else
	83	#error "not ported to this architecture"
	84	#endif
	85
2d21ac55 A	86	typedef struct lockstat_probe {
	87	const char *lsp_func;
	88	const char *lsp_name;
	89	int lsp_probe;
	90	dtrace_id_t lsp_id;
5ba3f43e	91	const char *lsp_args;
2d21ac55 A	92	} lockstat_probe_t;
	93
	94	lockstat_probe_t lockstat_probes[] =
	95	{
39236c6e	96	#if defined(__x86_64__)
316670eb	97	/* Only provide implemented probes for each architecture */
5ba3f43e A	98	LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_ACQUIRE, "lck_mtx_t"),
	99	LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_SPIN, LS_LCK_MTX_LOCK_SPIN, "lck_mtx_t", "uint64_t"),
	100	LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_BLOCK, LS_LCK_MTX_LOCK_BLOCK, "lck_mtx_t", "uint64_t"),
	101	LOCKSTAT_PROBE(LS_LCK_MTX_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_LOCK_ACQUIRE, "lck_mtx_t"),
	102	LOCKSTAT_PROBE(LS_LCK_MTX_TRY_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE, "lck_mtx_t"),
	103	LOCKSTAT_PROBE(LS_LCK_MTX_UNLOCK, LSA_RELEASE, LS_LCK_MTX_UNLOCK_RELEASE, "lck_mtx_t"),
	104	LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_ACQUIRE, LS_LCK_MTX_EXT_LOCK_ACQUIRE, "lck_mtx_t"),
	105	LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_SPIN, LS_LCK_MTX_EXT_LOCK_SPIN, "lck_mtx_t", "uint64_t"),
	106	LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_BLOCK, LS_LCK_MTX_EXT_LOCK_BLOCK, "lck_mtx_t", "uint64_t"),
	107	// LOCKSTAT_PROBE(LS_LCK_MTX_EXT_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_EXT_LOCK_ACQUIRE)
	108	LOCKSTAT_PROBE(LS_LCK_MTX_EXT_UNLOCK, LSA_RELEASE, LS_LCK_MTX_EXT_UNLOCK_RELEASE, "lck_mtx_t"),
	109	LOCKSTAT_PROBE(LS_LCK_MTX_LOCK_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_SPIN_ACQUIRE, "lck_mtx_t"),
	110	// TODO: This should not be a uint64_t !
	111	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_LOCK_SHARED_ACQUIRE, "lck_rw_t", "uint64_t"),
	112	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
	113	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_SPIN, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
	114	// TODO: This should NOT be a uint64_t
	115	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_LOCK_EXCL_ACQUIRE, "lck_rw_t", "uint64_t"),
	116	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_EXCL_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
	117	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_EXCL_SPIN, "lck_rw_t", "uint64_t", "int"),
	118	LOCKSTAT_PROBE(LS_LCK_RW_DONE, LSR_RELEASE, LS_LCK_RW_DONE_RELEASE, "lck_rw_t", "_Bool"),
	119	// TODO : This should NOT be a uint64_t
	120	LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, "lck_rw_t", "uint64_t"),
	121	// See above
	122	LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, "lck_rw_t", "uint64_t"),
	123	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_UPGRADE, LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, "lck_rw_t", "_Bool"),
	124	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN, "lck_rw_t", "uint64_t"),
	125	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
	126	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL_TO_SHARED, LSR_DOWNGRADE, LS_LCK_RW_LOCK_EXCL_TO_SHARED_DOWNGRADE, "lck_rw_t"),
	127	//TODO : Separate the probes for the hw_bit from the probe for the normal hw locks
	128	LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK, LSS_ACQUIRE, LS_LCK_SPIN_LOCK_ACQUIRE, "hw_lock_t"),
	129	LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK, LSS_SPIN, LS_LCK_SPIN_LOCK_SPIN, "hw_lock_t", "uint64_t", "uint64_t"),
	130	LOCKSTAT_PROBE(LS_LCK_SPIN_UNLOCK, LSS_RELEASE, LS_LCK_SPIN_UNLOCK_RELEASE, "hw_lock_t"),
	131	#elif defined(__arm__) \|\| defined(__arm64__)
	132	LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_ACQUIRE, "lck_mtx_t"),
	133	// LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_SPIN, LS_LCK_MTX_LOCK_SPIN, "lck_mtx_t", "uint64_t"),
	134	LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_BLOCK, LS_LCK_MTX_LOCK_BLOCK, "lck_mtx_t", "uint64_t"),
	135	LOCKSTAT_PROBE(LS_LCK_MTX_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_LOCK_ACQUIRE, "lck_mtx_t"),
	136	// LOCKSTAT_PROBE(LS_LCK_MTX_TRY_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE, "lck_mtx_t"),
	137	LOCKSTAT_PROBE(LS_LCK_MTX_UNLOCK, LSA_RELEASE, LS_LCK_MTX_UNLOCK_RELEASE, "lck_mtx_t"),
	138	LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_ACQUIRE, LS_LCK_MTX_EXT_LOCK_ACQUIRE, "lck_mtx_t"),
	139	// LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_SPIN, LS_LCK_MTX_EXT_LOCK_SPIN, "lck_mtx_t", "uint64_t"),
	140	LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_BLOCK, LS_LCK_MTX_EXT_LOCK_BLOCK, "lck_mtx_t", "uint64_t"),
	141	// LOCKSTAT_PROBE(LS_LCK_MTX_EXT_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_EXT_LOCK_ACQUIRE)
	142	// LOCKSTAT_PROBE(LS_LCK_MTX_EXT_UNLOCK, LSA_RELEASE, LS_LCK_MTX_EXT_UNLOCK_RELEASE, "lck_mtx_t"),
	143	// LOCKSTAT_PROBE(LS_LCK_MTX_LOCK_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_SPIN_ACQUIRE, "lck_mtx_t"),
	144	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_LOCK_SHARED_ACQUIRE, "lck_rw_t", "uint64_t"),
	145	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
	146	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_SPIN, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
	147	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_LOCK_EXCL_ACQUIRE, "lck_rw_t", "uint64_t"),
	148	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_EXCL_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
	149	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_EXCL_SPIN, "lck_rw_t", "uint64_t", "int"),
	150	LOCKSTAT_PROBE(LS_LCK_RW_DONE, LSR_RELEASE, LS_LCK_RW_DONE_RELEASE, "lck_rw_t", "_Bool"),
	151	// TODO : This should NOT be a uint64_t
	152	LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, "lck_rw_t", "uint64_t"),
	153	// See above
	154	LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, "lck_rw_t", "uint64_t"),
	155	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_UPGRADE, LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, "lck_rw_t", "_Bool"),
	156	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN, "lck_rw_t", "uint64_t"),
	157	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
	158	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL_TO_SHARED, LSR_DOWNGRADE, LS_LCK_RW_LOCK_EXCL_TO_SHARED_DOWNGRADE, "lck_rw_t"),
	159	//TODO : Separate the probes for the hw_bit from the probe for the normal hw locks
	160	LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK, LSS_ACQUIRE, LS_LCK_SPIN_LOCK_ACQUIRE, "hw_lock_t"),
	161	LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK, LSS_SPIN, LS_LCK_SPIN_LOCK_SPIN, "hw_lock_t", "uint64_t", "uint64_t"),
162	LOCKSTAT_PROBE(LS_LCK_SPIN_UNLOCK, LSS_RELEASE, LS_LCK_SPIN_UNLOCK_RELEASE, "hw_lock_t"),
316670eb	163	#endif
5ba3f43e A	164	/* Interlock measurements would be nice, but later */
5ba3f43e A	165
2d21ac55	166	#ifdef LATER
5ba3f43e A	167	LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL_TO_SHARED, LSA_ILK_SPIN, LS_LCK_RW_LOCK_EXCL_TO_SHARED_ILK_SPIN),
	168	LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_ILK_SPIN, LS_LCK_MTX_LOCK_ILK_SPIN),
	169	LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_ILK_SPIN, LS_LCK_MTX_EXT_LOCK_ILK_SPIN),
	170	LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_EXCL, LSA_ILK_SPIN, LS_LCK_RW_TRY_LOCK_EXCL_ILK_SPIN),
	171	LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_SHARED, LSA_SPIN, LS_LCK_RW_TRY_LOCK_SHARED_SPIN)
2d21ac55 A	172	#endif
2d21ac55 A	173
5ba3f43e	174	{ NULL, NULL, 0, 0, NULL}
2d21ac55 A	175	};
	176
	177	dtrace_id_t lockstat_probemap[LS_NPROBES];
	178
316670eb A	179	typedef struct lockstat_assembly_probe {
	180	int lsap_probe;
	181	vm_offset_t * lsap_patch_point;
	182	} lockstat_assembly_probe_t;
	183
	184
	185	lockstat_assembly_probe_t assembly_probes[] =
	186	{
316670eb	187	{ LS_LCK_INVALID, NULL }
d9a64523	188	};
39236c6e A	189
39236c6e A	190
2d21ac55	191	/*
fe8ab488 A	192	* APPLE NOTE:
	193	* Hot patch is used to manipulate probe points by swapping between
	194	* no-op and return instructions.
	195	* The active flag indicates whether the probe point will turn on or off.
316670eb A	196	* on == plant a NOP and thus fall through to the probe call
316670eb A	197	* off == plant a RET and thus avoid the probe call completely
fe8ab488	198	* The ls_probe identifies which probe we will patch.
2d21ac55	199	*/
b0d623f7	200	static
316670eb	201	void lockstat_hot_patch(boolean_t active, int ls_probe)
2d21ac55	202	{
c910b4d9	203	#pragma unused(active)
2d21ac55 A	204	int i;
2d21ac55 A	205
316670eb A	206	/*
	207	* Loop through entire table, in case there are
	208	* multiple patch points per probe.
	209	*/
	210	for (i = 0; assembly_probes[i].lsap_patch_point; i++) {
	211	if (ls_probe == assembly_probes[i].lsap_probe)
39236c6e	212	#if defined(__x86_64__)
316670eb A	213	{
	214	uint8_t instr;
	215	instr = (active ? NOP : RET );
	216	(void) ml_nofault_copy( (vm_offset_t)&instr, *(assembly_probes[i].lsap_patch_point),
2d21ac55	217	sizeof(instr));
316670eb	218	}
5ba3f43e A	219	#elif defined (__arm__)
	220	{
	221	uint32_t instr;
	222	instr = (active ? NOP : BXLR );
	223	(void) ml_nofault_copy( (vm_offset_t)&instr, *(assembly_probes[i].lsap_patch_point),
	224	sizeof(instr));
	225	}
	226	#elif defined (__arm64__)
	227	{
	228	uint32_t instr;
	229	instr = (active ? NOP : RET );
	230	(void) ml_nofault_copy( (vm_offset_t)&instr, *(assembly_probes[i].lsap_patch_point),
	231	sizeof(instr));
	232	}
2d21ac55	233	#endif
316670eb	234	} /* for */
2d21ac55	235	}
2d21ac55 A	236
	237	void (*lockstat_probe)(dtrace_id_t, uint64_t, uint64_t,
	238	uint64_t, uint64_t, uint64_t);
	239
fe8ab488 A	240
	241	/*
	242	* APPLE NOTE:
	243	* This wrapper is used only by assembler hot patched probes.
	244	*/
316670eb A	245	void
	246	lockstat_probe_wrapper(int probe, uintptr_t lp, int rwflag)
	247	{
	248	dtrace_id_t id;
	249	id = lockstat_probemap[probe];
	250	if (id != 0)
	251	{
	252	(*lockstat_probe)(id, (uintptr_t)lp, (uint64_t)rwflag, 0,0,0);
	253	}
	254	}
316670eb	255
2d21ac55 A	256	static dtrace_provider_id_t lockstat_id;
	257
	258	/ARGSUSED/
6d2010ae	259	static int
2d21ac55 A	260	lockstat_enable(void arg, dtrace_id_t id, void parg)
2d21ac55 A	261	{
b0d623f7 A	262	#pragma unused(arg) /* __APPLE__ */
b0d623f7 A	263
2d21ac55 A	264	lockstat_probe_t *probe = parg;
	265
	266	ASSERT(!lockstat_probemap[probe->lsp_probe]);
	267
	268	lockstat_probemap[probe->lsp_probe] = id;
	269	membar_producer();
	270
316670eb	271	lockstat_hot_patch(TRUE, probe->lsp_probe);
2d21ac55	272	membar_producer();
6d2010ae	273	return(0);
2d21ac55 A	274
	275	}
	276
	277	/ARGSUSED/
	278	static void
	279	lockstat_disable(void arg, dtrace_id_t id, void parg)
	280	{
b0d623f7 A	281	#pragma unused(arg, id) /* __APPLE__ */
b0d623f7 A	282
2d21ac55 A	283	lockstat_probe_t *probe = parg;
	284	int i;
	285
	286	ASSERT(lockstat_probemap[probe->lsp_probe]);
	287
	288	lockstat_probemap[probe->lsp_probe] = 0;
316670eb	289	lockstat_hot_patch(FALSE, probe->lsp_probe);
2d21ac55 A	290	membar_producer();
	291
	292	/*
	293	* See if we have any probes left enabled.
	294	*/
	295	for (i = 0; i < LS_NPROBES; i++) {
	296	if (lockstat_probemap[i]) {
	297	/*
	298	* This probe is still enabled. We don't need to deal
	299	* with waiting for all threads to be out of the
	300	* lockstat critical sections; just return.
	301	*/
	302	return;
	303	}
	304	}
	305
	306	}
	307
	308	/ARGSUSED/
	309	static void
	310	lockstat_provide(void arg, const dtrace_probedesc_t desc)
	311	{
b0d623f7 A	312	#pragma unused(arg, desc) /* __APPLE__ */
b0d623f7 A	313
2d21ac55 A	314	int i = 0;
	315
	316	for (i = 0; lockstat_probes[i].lsp_func != NULL; i++) {
	317	lockstat_probe_t *probe = &lockstat_probes[i];
	318
	319	if (dtrace_probe_lookup(lockstat_id, "mach_kernel",
	320	probe->lsp_func, probe->lsp_name) != 0)
	321	continue;
	322
	323	ASSERT(!probe->lsp_id);
	324	probe->lsp_id = dtrace_probe_create(lockstat_id,
	325	"mach_kernel", probe->lsp_func, probe->lsp_name,
	326	LOCKSTAT_AFRAMES, probe);
	327	}
	328	}
	329
	330
	331	/ARGSUSED/
	332	static void
	333	lockstat_destroy(void arg, dtrace_id_t id, void parg)
	334	{
b0d623f7 A	335	#pragma unused(arg, id) /* __APPLE__ */
b0d623f7 A	336
2d21ac55 A	337	lockstat_probe_t *probe = parg;
	338
	339	ASSERT(!lockstat_probemap[probe->lsp_probe]);
	340	probe->lsp_id = 0;
	341	}
	342
5ba3f43e A	343	static void
	344	lockstat_getargdesc(void arg, dtrace_id_t id, void parg, dtrace_argdesc_t *desc)
	345	{
	346	#pragma unused(arg, id)
	347	lockstat_probe_t *probe = parg;
	348	const char* argdesc = probe->lsp_args;
	349	int narg = 0;
	350
	351	desc->dtargd_native[0] = '\0';
	352	desc->dtargd_xlate[0] = '\0';
	353
	354	while(argdesc[0] != '\0') {
	355	if (narg == desc->dtargd_ndx) {
	356	strlcpy(desc->dtargd_native, argdesc, DTRACE_ARGTYPELEN);
	357	return;
	358	}
	359	argdesc += strlen(argdesc) + 1;
	360	narg++;
	361	}
	362
	363	desc->dtargd_ndx = DTRACE_ARGNONE;
	364	}
	365
2d21ac55 A	366	static dtrace_pattr_t lockstat_attr = {
	367	{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
	368	{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
	369	{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
	370	{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
	371	{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
	372	};
	373
	374	static dtrace_pops_t lockstat_pops = {
d9a64523 A	375	.dtps_provide = lockstat_provide,
	376	.dtps_provide_module = NULL,
	377	.dtps_enable = lockstat_enable,
	378	.dtps_disable = lockstat_disable,
	379	.dtps_suspend = NULL,
	380	.dtps_resume = NULL,
	381	.dtps_getargdesc = lockstat_getargdesc,
	382	.dtps_getargval = NULL,
	383	.dtps_usermode = NULL,
	384	.dtps_destroy = lockstat_destroy
2d21ac55 A	385	};
	386
	387	static int
d9a64523	388	lockstat_attach(dev_info_t *devi)
2d21ac55	389	{
2d21ac55 A	390	if (ddi_create_minor_node(devi, "lockstat", S_IFCHR, 0,
	391	DDI_PSEUDO, 0) == DDI_FAILURE \|\|
	392	dtrace_register("lockstat", &lockstat_attr, DTRACE_PRIV_KERNEL,
	393	NULL, &lockstat_pops, NULL, &lockstat_id) != 0) {
	394	ddi_remove_minor_node(devi, NULL);
	395	return (DDI_FAILURE);
	396	}
	397
b0d623f7 A	398	lockstat_probe = dtrace_probe;
	399	membar_producer();
	400
2d21ac55 A	401	return (DDI_SUCCESS);
	402	}
	403
	404	d_open_t _lockstat_open;
	405
	406	int _lockstat_open(dev_t dev, int flags, int devtype, struct proc *p)
	407	{
	408	#pragma unused(dev,flags,devtype,p)
	409	return 0;
	410	}
	411
	412	#define LOCKSTAT_MAJOR -24 /* let the kernel pick the device number */
	413
	414	/*
	415	* A struct describing which functions will get invoked for certain
	416	* actions.
	417	*/
	418	static struct cdevsw lockstat_cdevsw =
	419	{
	420	_lockstat_open, /* open */
	421	eno_opcl, /* close */
	422	eno_rdwrt, /* read */
	423	eno_rdwrt, /* write */
	424	eno_ioctl, /* ioctl */
	425	(stop_fcn_t )nulldev, / stop */
	426	(reset_fcn_t )nulldev, / reset */
	427	NULL, /* tty's */
	428	eno_select, /* select */
	429	eno_mmap, /* mmap */
	430	eno_strat, /* strategy */
	431	eno_getc, /* getc */
	432	eno_putc, /* putc */
	433	0 /* type */
	434	};
	435
2d21ac55 A	436	void lockstat_init( void );
	437
	438	void lockstat_init( void )
	439	{
d9a64523 A	440	int majdevno = cdevsw_add(LOCKSTAT_MAJOR, &lockstat_cdevsw);
	441
	442	if (majdevno < 0) {
	443	printf("lockstat_init: failed to allocate a major number!\n");
	444	return;
	445	}
2d21ac55	446
d9a64523	447	lockstat_attach((dev_info_t*)(uintptr_t)majdevno);
2d21ac55 A	448	}
2d21ac55 A	449	#undef LOCKSTAT_MAJOR