[apple/xnu.git] / bsd / dev / dtrace / sdt.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/* #pragma ident	"@(#)sdt.c	1.6	06/03/24 SMI" */

#ifdef KERNEL
#ifndef _KERNEL
#define _KERNEL /* Solaris vs. Darwin */
#endif
#endif

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/errno.h>
#include <sys/stat.h>
#include <sys/ioctl.h>
#include <sys/conf.h>
#include <sys/fcntl.h>
#include <miscfs/devfs/devfs.h>

#include <sys/dtrace.h>
#include <sys/dtrace_impl.h>

#include <sys/dtrace_glue.h>

#include <sys/sdt_impl.h>

struct savearea_t; /* Used anonymously */
typedef kern_return_t (*perfCallback)(int, struct savearea_t *, int, int);

#if defined (__ppc__) || defined (__ppc64__)
extern perfCallback tempDTraceTrapHook, tempDTraceIntHook;
extern kern_return_t fbt_perfCallback(int, struct savearea_t *, int, int);
extern kern_return_t fbt_perfIntCallback(int, struct savearea_t *, int, int);

#define	SDT_PATCHVAL	0x7c810808
#define SDT_AFRAMES     6
#elif defined(__i386__) || defined(__x86_64__)
extern perfCallback tempDTraceTrapHook;
extern kern_return_t fbt_perfCallback(int, struct savearea_t *, int, int);

#define	SDT_PATCHVAL	0xf0
#define	SDT_AFRAMES		6
#else
#error Unknown architecture
#endif

#define	SDT_PROBETAB_SIZE	0x1000		/* 4k entries -- 16K total */

static dev_info_t		*sdt_devi;
static int			sdt_verbose = 0;
sdt_probe_t		**sdt_probetab;
int			sdt_probetab_size;
int			sdt_probetab_mask;

/*ARGSUSED*/
static void
__sdt_provide_module(void *arg, struct modctl *ctl)
{
#pragma unused(arg)
	struct module *mp = (struct module *)ctl->address;
	char *modname = ctl->mod_modname;
	sdt_probedesc_t *sdpd;
	sdt_probe_t *sdp, *old;
	sdt_provider_t *prov;
	int len;

	/*
	 * One for all, and all for one:  if we haven't yet registered all of
	 * our providers, we'll refuse to provide anything.
	 */
	for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
		if (prov->sdtp_id == DTRACE_PROVNONE)
			return;
	}

	if (mp->sdt_nprobes != 0 || (sdpd = mp->sdt_probes) == NULL)
		return;

	for (sdpd = mp->sdt_probes; sdpd != NULL; sdpd = sdpd->sdpd_next) {
		char *name = sdpd->sdpd_name, *func, *nname;
		int i, j;
		dtrace_id_t id;

		for (prov = sdt_providers; prov->sdtp_prefix != NULL; prov++) {
			char *prefpart, *prefix = prov->sdtp_prefix;

			if ((prefpart = strstr(name, prefix))) {
				name = prefpart + strlen(prefix);
				break;
			}
		}

		nname = kmem_alloc(len = strlen(name) + 1, KM_SLEEP);

		for (i = 0, j = 0; name[j] != '\0'; i++) {
			if (name[j] == '_' && name[j + 1] == '_') {
				nname[i] = '-';
				j += 2;
			} else {
				nname[i] = name[j++];
			}
		}

		nname[i] = '\0';

		sdp = kmem_zalloc(sizeof (sdt_probe_t), KM_SLEEP);
		sdp->sdp_loadcnt = ctl->mod_loadcnt;
		sdp->sdp_ctl = ctl;
		sdp->sdp_name = nname;
		sdp->sdp_namelen = len;
		sdp->sdp_provider = prov;

		func = sdpd->sdpd_func;

		if (func == NULL)
			func = "<unknown>";

		/*
		 * We have our provider.  Now create the probe.
		 */
		if ((id = dtrace_probe_lookup(prov->sdtp_id, modname,
		    func, nname)) != DTRACE_IDNONE) {
			old = dtrace_probe_arg(prov->sdtp_id, id);
			ASSERT(old != NULL);

			sdp->sdp_next = old->sdp_next;
			sdp->sdp_id = id;
			old->sdp_next = sdp;
		} else {
			sdp->sdp_id = dtrace_probe_create(prov->sdtp_id,
			    modname, func, nname, SDT_AFRAMES, sdp);

			mp->sdt_nprobes++;
		}

		sdp->sdp_hashnext =
		    sdt_probetab[SDT_ADDR2NDX(sdpd->sdpd_offset)];
		sdt_probetab[SDT_ADDR2NDX(sdpd->sdpd_offset)] = sdp;

		sdp->sdp_patchval = SDT_PATCHVAL;
		sdp->sdp_patchpoint = (sdt_instr_t *)sdpd->sdpd_offset;
		sdp->sdp_savedval = *sdp->sdp_patchpoint;
	}
}

/*ARGSUSED*/
static void
sdt_destroy(void *arg, dtrace_id_t id, void *parg)
{
#pragma unused(arg,id)
	sdt_probe_t *sdp = parg, *old, *last, *hash;
	int ndx;
#if !defined(__APPLE__)
	struct modctl *ctl = sdp->sdp_ctl;

	if (ctl != NULL && ctl->mod_loadcnt == sdp->sdp_loadcnt) {
		if ((ctl->mod_loadcnt == sdp->sdp_loadcnt &&
		    ctl->mod_loaded)) {
			((struct module *)(ctl->mod_mp))->sdt_nprobes--;
		}
	}
#endif /* __APPLE__ */

	while (sdp != NULL) {
		old = sdp;

		/*
		 * Now we need to remove this probe from the sdt_probetab.
		 */
		ndx = SDT_ADDR2NDX(sdp->sdp_patchpoint);
		last = NULL;
		hash = sdt_probetab[ndx];

		while (hash != sdp) {
			ASSERT(hash != NULL);
			last = hash;
			hash = hash->sdp_hashnext;
		}

		if (last != NULL) {
			last->sdp_hashnext = sdp->sdp_hashnext;
		} else {
			sdt_probetab[ndx] = sdp->sdp_hashnext;
		}

		kmem_free(sdp->sdp_name, sdp->sdp_namelen);
		sdp = sdp->sdp_next;
		kmem_free(old, sizeof (sdt_probe_t));
	}
}

/*ARGSUSED*/
static void
sdt_enable(void *arg, dtrace_id_t id, void *parg)
{
#pragma unused(arg,id)
	sdt_probe_t *sdp = parg;
	struct modctl *ctl = sdp->sdp_ctl;

#if !defined(__APPLE__)
	ctl->mod_nenabled++;

	/*
	 * If this module has disappeared since we discovered its probes,
	 * refuse to enable it.
	 */
	if (!ctl->mod_loaded) {
		if (sdt_verbose) {
			cmn_err(CE_NOTE, "sdt is failing for probe %s "
			    "(module %s unloaded)",
			    sdp->sdp_name, ctl->mod_modname);
		}
		goto err;
	}

	/*
	 * Now check that our modctl has the expected load count.  If it
	 * doesn't, this module must have been unloaded and reloaded -- and
	 * we're not going to touch it.
	 */
	if (ctl->mod_loadcnt != sdp->sdp_loadcnt) {
		if (sdt_verbose) {
			cmn_err(CE_NOTE, "sdt is failing for probe %s "
			    "(module %s reloaded)",
			    sdp->sdp_name, ctl->mod_modname);
		}
		goto err;
	}
#endif /* __APPLE__ */

#if defined (__ppc__) || defined (__ppc64__)
	dtrace_casptr(&tempDTraceIntHook, NULL, fbt_perfIntCallback);
	if (tempDTraceIntHook != (perfCallback)fbt_perfIntCallback) {
		if (sdt_verbose) {
			cmn_err(CE_NOTE, "sdt_enable is failing for probe %s "
			    "in module %s: tempDTraceIntHook already occupied.",
			    sdp->sdp_name, ctl->mod_modname);
		}
		return;
	}
#endif
	
	dtrace_casptr(&tempDTraceTrapHook, NULL, fbt_perfCallback);
	if (tempDTraceTrapHook != (perfCallback)fbt_perfCallback) {
		if (sdt_verbose) {
			cmn_err(CE_NOTE, "sdt_enable is failing for probe %s "
			    "in module %s: tempDTraceTrapHook already occupied.",
			    sdp->sdp_name, ctl->mod_modname);
		}
		return;
	}

	while (sdp != NULL) {
		(void)ml_nofault_copy( (vm_offset_t)&sdp->sdp_patchval, (vm_offset_t)sdp->sdp_patchpoint, 
								sizeof(sdp->sdp_patchval));
		sdp = sdp->sdp_next;
	}
err:
	;
}

/*ARGSUSED*/
static void
sdt_disable(void *arg, dtrace_id_t id, void *parg)
{
#pragma unused(arg,id)
	sdt_probe_t *sdp = parg;
#if !defined(__APPLE__)
	struct modctl *ctl = sdp->sdp_ctl;

	ctl->mod_nenabled--;

	if (!ctl->mod_loaded || ctl->mod_loadcnt != sdp->sdp_loadcnt)
		goto err;
#endif /* __APPLE__ */

	while (sdp != NULL) {
		(void)ml_nofault_copy( (vm_offset_t)&sdp->sdp_savedval, (vm_offset_t)sdp->sdp_patchpoint, 
								sizeof(sdp->sdp_savedval));
		sdp = sdp->sdp_next;
	}

err:
	;
}

static dtrace_pops_t sdt_pops = {
	NULL,
	sdt_provide_module,
	sdt_enable,
	sdt_disable,
	NULL,
	NULL,
	sdt_getargdesc,
	NULL,
	NULL,
	sdt_destroy
};

/*ARGSUSED*/
static int
sdt_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
{
#pragma unused(cmd)
	sdt_provider_t *prov;

	if (ddi_create_minor_node(devi, "sdt", S_IFCHR,
	    0, DDI_PSEUDO, 0) == DDI_FAILURE) {
		cmn_err(CE_NOTE, "/dev/sdt couldn't create minor node");
		ddi_remove_minor_node(devi, NULL);
		return (DDI_FAILURE);
	}

	ddi_report_dev(devi);
	sdt_devi = devi;

	if (sdt_probetab_size == 0)
		sdt_probetab_size = SDT_PROBETAB_SIZE;

	sdt_probetab_mask = sdt_probetab_size - 1;
	sdt_probetab =
	    kmem_zalloc(sdt_probetab_size * sizeof (sdt_probe_t *), KM_SLEEP);
	dtrace_invop_add(sdt_invop);

	for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
		if (dtrace_register(prov->sdtp_name, prov->sdtp_attr,
		    DTRACE_PRIV_KERNEL, NULL,
		    &sdt_pops, prov, &prov->sdtp_id) != 0) {
			cmn_err(CE_WARN, "failed to register sdt provider %s",
			    prov->sdtp_name);
		}
	}

	return (DDI_SUCCESS);
}

#if !defined(__APPLE__)
/*ARGSUSED*/
static int
sdt_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
	sdt_provider_t *prov;

	switch (cmd) {
	case DDI_DETACH:
		break;

	case DDI_SUSPEND:
		return (DDI_SUCCESS);

	default:
		return (DDI_FAILURE);
	}

	for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
		if (prov->sdtp_id != DTRACE_PROVNONE) {
			if (dtrace_unregister(prov->sdtp_id) != 0)
				return (DDI_FAILURE);

			prov->sdtp_id = DTRACE_PROVNONE;
		}
	}

	dtrace_invop_remove(sdt_invop);
	kmem_free(sdt_probetab, sdt_probetab_size * sizeof (sdt_probe_t *));

	return (DDI_SUCCESS);
}

/*ARGSUSED*/
static int
sdt_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
{
	int error;

	switch (infocmd) {
	case DDI_INFO_DEVT2DEVINFO:
		*result = (void *)sdt_devi;
		error = DDI_SUCCESS;
		break;
	case DDI_INFO_DEVT2INSTANCE:
		*result = (void *)0;
		error = DDI_SUCCESS;
		break;
	default:
		error = DDI_FAILURE;
	}
	return (error);
}

/*ARGSUSED*/
static int
sdt_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
{
	return (0);
}

static struct cb_ops sdt_cb_ops = {
	sdt_open,		/* open */
	nodev,			/* close */
	nulldev,		/* strategy */
	nulldev,		/* print */
	nodev,			/* dump */
	nodev,			/* read */
	nodev,			/* write */
	nodev,			/* ioctl */
	nodev,			/* devmap */
	nodev,			/* mmap */
	nodev,			/* segmap */
	nochpoll,		/* poll */
	ddi_prop_op,		/* cb_prop_op */
	0,			/* streamtab  */
	D_NEW | D_MP		/* Driver compatibility flag */
};

static struct dev_ops sdt_ops = {
	DEVO_REV,		/* devo_rev, */
	0,			/* refcnt  */
	sdt_info,		/* get_dev_info */
	nulldev,		/* identify */
	nulldev,		/* probe */
	sdt_attach,		/* attach */
	sdt_detach,		/* detach */
	nodev,			/* reset */
	&sdt_cb_ops,		/* driver operations */
	NULL,			/* bus operations */
	nodev			/* dev power */
};

/*
 * Module linkage information for the kernel.
 */
static struct modldrv modldrv = {
	&mod_driverops,		/* module type (this is a pseudo driver) */
	"Statically Defined Tracing",	/* name of module */
	&sdt_ops,		/* driver ops */
};

static struct modlinkage modlinkage = {
	MODREV_1,
	(void *)&modldrv,
	NULL
};

int
_init(void)
{
	return (mod_install(&modlinkage));
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}

int
_fini(void)
{
	return (mod_remove(&modlinkage));
}
#else
d_open_t _sdt_open;

int _sdt_open(dev_t dev, int flags, int devtype, struct proc *p)
{
#pragma unused(dev,flags,devtype,p)
	return 0;
}

#define SDT_MAJOR  -24 /* let the kernel pick the device number */

/*
 * A struct describing which functions will get invoked for certain
 * actions.
 */
static struct cdevsw sdt_cdevsw =
{
	_sdt_open,		/* open */
	eno_opcl,			/* close */
	eno_rdwrt,			/* read */
	eno_rdwrt,			/* write */
	eno_ioctl,			/* ioctl */
	(stop_fcn_t *)nulldev, /* stop */
	(reset_fcn_t *)nulldev, /* reset */
	NULL,				/* tty's */
	eno_select,			/* select */
	eno_mmap,			/* mmap */
	eno_strat,			/* strategy */
	eno_getc,			/* getc */
	eno_putc,			/* putc */
	0					/* type */
};

static int gSDTInited = 0;
static struct modctl g_sdt_kernctl;
static struct module g_sdt_mach_module;

#include <mach-o/loader.h>
#include <mach-o/nlist.h>

extern struct mach_header _mh_execute_header; /* the kernel's mach header */

void sdt_init( void )
{
	if (0 == gSDTInited)
	{
		int majdevno = cdevsw_add(SDT_MAJOR, &sdt_cdevsw);
		
		if (majdevno < 0) {
			printf("sdt_init: failed to allocate a major number!\n");
			gSDTInited = 0;
			return;
		}

		if (MH_MAGIC != _mh_execute_header.magic) {
        	g_sdt_kernctl.address = (vm_address_t)NULL;
        	g_sdt_kernctl.size = 0;
		} else {
		struct mach_header          *mh;
    		struct load_command         *cmd;
    		struct segment_command      *orig_ts = NULL, *orig_le = NULL;
    		struct symtab_command       *orig_st = NULL;
    		struct nlist                *sym = NULL;
    		char                        *strings;
    		unsigned int i;

		g_sdt_mach_module.sdt_nprobes = 0;
		g_sdt_mach_module.sdt_probes = NULL;

        	g_sdt_kernctl.address = (vm_address_t)&g_sdt_mach_module;
        	g_sdt_kernctl.size = 0;
		strncpy((char *)&(g_sdt_kernctl.mod_modname), "mach_kernel", KMOD_MAX_NAME);

		mh = &_mh_execute_header;
    		cmd = (struct load_command *) &mh[1];
    		for (i = 0; i < mh->ncmds; i++) {
        		if (cmd->cmd == LC_SEGMENT) {
            		struct segment_command *orig_sg = (struct segment_command *) cmd;

            		if (strcmp(SEG_TEXT, orig_sg->segname) == 0)
                		orig_ts = orig_sg;
            		else if (strcmp(SEG_LINKEDIT, orig_sg->segname) == 0)
                		orig_le = orig_sg;
            		else if (strcmp("", orig_sg->segname) == 0)
                		orig_ts = orig_sg; /* kexts have a single unnamed segment */
        		}
        		else if (cmd->cmd == LC_SYMTAB)
            		orig_st = (struct symtab_command *) cmd;
	
        		cmd = (struct load_command *) ((caddr_t) cmd + cmd->cmdsize);
    		}
	
    		if ((orig_ts == NULL) || (orig_st == NULL) || (orig_le == NULL))
        		return;

    		sym = (struct nlist *)orig_le->vmaddr;
    		strings = ((char *)sym) + orig_st->nsyms * sizeof(struct nlist);

    		for (i = 0; i < orig_st->nsyms; i++) {
        		uint8_t n_type = sym[i].n_type & (N_TYPE | N_EXT);
        		char *name = strings + sym[i].n_un.n_strx;
				char *prev_name;
				unsigned long best;
				unsigned int j;

        		/* Check that the symbol is a global and that it has a name. */
        		if (((N_SECT | N_EXT) != n_type && (N_ABS | N_EXT) != n_type))
            		continue;

        		if (0 == sym[i].n_un.n_strx) /* iff a null, "", name. */
            		continue;

        		/* Lop off omnipresent leading underscore. */
        		if (*name == '_')
            		name += 1;

				if (strstr(name, "_dtrace_probe$")) {
					sdt_probedesc_t *sdpd = kmem_alloc(sizeof(sdt_probedesc_t), KM_SLEEP);
					int len = strlen(name) + 1;

					sdpd->sdpd_name = kmem_alloc(len, KM_SLEEP);
					strncpy(sdpd->sdpd_name, name, len); /* NUL termination is ensured. */

					prev_name = "<unknown>";
					best = 0;
					for (j = 0; j < orig_st->nsyms; j++) {
						uint8_t n_type = sym[j].n_type & (N_TYPE | N_EXT);
						char *name = strings + sym[j].n_un.n_strx;

						if (((N_SECT | N_EXT) != n_type && (N_ABS | N_EXT) != n_type))
							continue;

						if (0 == sym[j].n_un.n_strx) /* iff a null, "", name. */
							continue;

						if (*name == '_')
							name += 1;
						if (strstr(name, "_dtrace_probe$"))
							continue;

						if (*(unsigned long *)sym[i].n_value <= (unsigned long)sym[j].n_value)
							continue;

						if ((unsigned long)sym[j].n_value > best) {
							best = (unsigned long)sym[j].n_value;
							prev_name = name;
						}
					}

					sdpd->sdpd_func = kmem_alloc((len = strlen(prev_name) + 1), KM_SLEEP);
					strncpy(sdpd->sdpd_func, prev_name, len); /* NUL termination is ensured. */

					sdpd->sdpd_offset = *(unsigned long *)sym[i].n_value;

					sdpd->sdpd_next = g_sdt_mach_module.sdt_probes;
					g_sdt_mach_module.sdt_probes = sdpd;
				} else {
					prev_name = name;
				}
			}
		}

		sdt_attach( (dev_info_t	*)majdevno, DDI_ATTACH );

		gSDTInited = 1;
	} else
		panic("sdt_init: called twice!\n");
}
#undef SDT_MAJOR

/*ARGSUSED*/
void
sdt_provide_module(void *arg, struct modctl *ctl)
{
#pragma unused(ctl)
    __sdt_provide_module(arg, &g_sdt_kernctl);

	sdt_probedesc_t *sdpd = g_sdt_mach_module.sdt_probes;
	while (sdpd) {
		sdt_probedesc_t *this_sdpd = sdpd;
		kmem_free((void *)sdpd->sdpd_name, strlen(sdpd->sdpd_name) + 1);
		kmem_free((void *)sdpd->sdpd_func, strlen(sdpd->sdpd_func) + 1);
		sdpd = sdpd->sdpd_next;
		kmem_free((void *)this_sdpd, sizeof(sdt_probedesc_t));
	}
	g_sdt_mach_module.sdt_probes = NULL;
}

#endif /* __APPLE__ */
Commit	Line	Data
2d21ac55 A	1	/*
	2	* CDDL HEADER START
	3	*
	4	* The contents of this file are subject to the terms of the
	5	* Common Development and Distribution License (the "License").
	6	* You may not use this file except in compliance with the License.
	7	*
	8	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	9	* or http://www.opensolaris.org/os/licensing.
	10	* See the License for the specific language governing permissions
	11	* and limitations under the License.
	12	*
	13	* When distributing Covered Code, include this CDDL HEADER in each
	14	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	15	* If applicable, add the following below this CDDL HEADER, with the
	16	* fields enclosed by brackets "[]" replaced with your own identifying
	17	* information: Portions Copyright [yyyy] [name of copyright owner]
	18	*
	19	* CDDL HEADER END
	20	*/
	21	/*
	22	* Copyright 2006 Sun Microsystems, Inc. All rights reserved.
	23	* Use is subject to license terms.
	24	*/
	25
	26	/* #pragma ident "@(#)sdt.c 1.6 06/03/24 SMI" */
	27
	28	#ifdef KERNEL
	29	#ifndef _KERNEL
	30	#define _KERNEL /* Solaris vs. Darwin */
	31	#endif
	32	#endif
	33
	34	#include <sys/param.h>
	35	#include <sys/systm.h>
	36	#include <sys/errno.h>
	37	#include <sys/stat.h>
	38	#include <sys/ioctl.h>
	39	#include <sys/conf.h>
	40	#include <sys/fcntl.h>
	41	#include <miscfs/devfs/devfs.h>
	42
	43	#include <sys/dtrace.h>
	44	#include <sys/dtrace_impl.h>
	45
	46	#include <sys/dtrace_glue.h>
	47
	48	#include <sys/sdt_impl.h>
	49
	50	struct savearea_t; /* Used anonymously */
	51	typedef kern_return_t (perfCallback)(int, struct savearea_t , int, int);
	52
	53	#if defined (__ppc__) \|\| defined (__ppc64__)
	54	extern perfCallback tempDTraceTrapHook, tempDTraceIntHook;
	55	extern kern_return_t fbt_perfCallback(int, struct savearea_t *, int, int);
	56	extern kern_return_t fbt_perfIntCallback(int, struct savearea_t *, int, int);
	57
	58	#define SDT_PATCHVAL 0x7c810808
	59	#define SDT_AFRAMES 6
	60	#elif defined(__i386__) \|\| defined(__x86_64__)
	61	extern perfCallback tempDTraceTrapHook;
	62	extern kern_return_t fbt_perfCallback(int, struct savearea_t *, int, int);
	63
	64	#define SDT_PATCHVAL 0xf0
65	#define SDT_AFRAMES 6
66	#else
67	#error Unknown architecture
68	#endif
69
70	#define SDT_PROBETAB_SIZE 0x1000 /* 4k entries -- 16K total */
71
72	static dev_info_t *sdt_devi;
73	static int sdt_verbose = 0;
74	sdt_probe_t **sdt_probetab;
75	int sdt_probetab_size;
76	int sdt_probetab_mask;
77
78	/ARGSUSED/
79	static void
80	__sdt_provide_module(void arg, struct modctl ctl)
81	{
82	#pragma unused(arg)
83	struct module mp = (struct module )ctl->address;
84	char *modname = ctl->mod_modname;
85	sdt_probedesc_t *sdpd;
86	sdt_probe_t sdp, old;
87	sdt_provider_t *prov;
88	int len;
89
90	/*
91	* One for all, and all for one: if we haven't yet registered all of
92	* our providers, we'll refuse to provide anything.
93	*/
94	for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
95	if (prov->sdtp_id == DTRACE_PROVNONE)
96	return;
97	}
98
99	if (mp->sdt_nprobes != 0 \|\| (sdpd = mp->sdt_probes) == NULL)
100	return;
101
102	for (sdpd = mp->sdt_probes; sdpd != NULL; sdpd = sdpd->sdpd_next) {
103	char name = sdpd->sdpd_name, func, *nname;
104	int i, j;
105	dtrace_id_t id;
106
107	for (prov = sdt_providers; prov->sdtp_prefix != NULL; prov++) {
108	char prefpart, prefix = prov->sdtp_prefix;
109
110	if ((prefpart = strstr(name, prefix))) {
111	name = prefpart + strlen(prefix);
112	break;
113	}
114	}
115
116	nname = kmem_alloc(len = strlen(name) + 1, KM_SLEEP);
117
118	for (i = 0, j = 0; name[j] != '\0'; i++) {
119	if (name[j] == '_' && name[j + 1] == '_') {
120	nname[i] = '-';
121	j += 2;
122	} else {
123	nname[i] = name[j++];
124	}
125	}
126
127	nname[i] = '\0';
128
129	sdp = kmem_zalloc(sizeof (sdt_probe_t), KM_SLEEP);
130	sdp->sdp_loadcnt = ctl->mod_loadcnt;
131	sdp->sdp_ctl = ctl;
132	sdp->sdp_name = nname;
133	sdp->sdp_namelen = len;
134	sdp->sdp_provider = prov;
135
136	func = sdpd->sdpd_func;
137
138	if (func == NULL)
139	func = "<unknown>";
140
141	/*
142	* We have our provider. Now create the probe.
143	*/
144	if ((id = dtrace_probe_lookup(prov->sdtp_id, modname,
145	func, nname)) != DTRACE_IDNONE) {
146	old = dtrace_probe_arg(prov->sdtp_id, id);
147	ASSERT(old != NULL);
148
149	sdp->sdp_next = old->sdp_next;
150	sdp->sdp_id = id;
151	old->sdp_next = sdp;
152	} else {
153	sdp->sdp_id = dtrace_probe_create(prov->sdtp_id,
154	modname, func, nname, SDT_AFRAMES, sdp);
155
156	mp->sdt_nprobes++;
157	}
158
159	sdp->sdp_hashnext =
160	sdt_probetab[SDT_ADDR2NDX(sdpd->sdpd_offset)];
161	sdt_probetab[SDT_ADDR2NDX(sdpd->sdpd_offset)] = sdp;
162
163	sdp->sdp_patchval = SDT_PATCHVAL;
164	sdp->sdp_patchpoint = (sdt_instr_t *)sdpd->sdpd_offset;
165	sdp->sdp_savedval = *sdp->sdp_patchpoint;
166	}
167	}
168
169	/ARGSUSED/
170	static void
171	sdt_destroy(void arg, dtrace_id_t id, void parg)
172	{
173	#pragma unused(arg,id)
174	sdt_probe_t sdp = parg, old, last, hash;
175	int ndx;
176	#if !defined(__APPLE__)
177	struct modctl *ctl = sdp->sdp_ctl;
178
179	if (ctl != NULL && ctl->mod_loadcnt == sdp->sdp_loadcnt) {
180	if ((ctl->mod_loadcnt == sdp->sdp_loadcnt &&
181	ctl->mod_loaded)) {
182	((struct module *)(ctl->mod_mp))->sdt_nprobes--;
183	}
184	}
185	#endif /* __APPLE__ */
186
187	while (sdp != NULL) {
188	old = sdp;
189
190	/*
191	* Now we need to remove this probe from the sdt_probetab.
192	*/
193	ndx = SDT_ADDR2NDX(sdp->sdp_patchpoint);
194	last = NULL;
195	hash = sdt_probetab[ndx];
196
197	while (hash != sdp) {
198	ASSERT(hash != NULL);
199	last = hash;
200	hash = hash->sdp_hashnext;
201	}
202
203	if (last != NULL) {
204	last->sdp_hashnext = sdp->sdp_hashnext;
205	} else {
206	sdt_probetab[ndx] = sdp->sdp_hashnext;
207	}
208
209	kmem_free(sdp->sdp_name, sdp->sdp_namelen);
210	sdp = sdp->sdp_next;
211	kmem_free(old, sizeof (sdt_probe_t));
212	}
213	}
214
215	/ARGSUSED/
216	static void
217	sdt_enable(void arg, dtrace_id_t id, void parg)
218	{
219	#pragma unused(arg,id)
220	sdt_probe_t *sdp = parg;
221	struct modctl *ctl = sdp->sdp_ctl;
222
223	#if !defined(__APPLE__)
224	ctl->mod_nenabled++;
225
226	/*
227	* If this module has disappeared since we discovered its probes,
228	* refuse to enable it.
229	*/
230	if (!ctl->mod_loaded) {
231	if (sdt_verbose) {
232	cmn_err(CE_NOTE, "sdt is failing for probe %s "
233	"(module %s unloaded)",
234	sdp->sdp_name, ctl->mod_modname);
235	}
236	goto err;
237	}
238
239	/*
240	* Now check that our modctl has the expected load count. If it
241	* doesn't, this module must have been unloaded and reloaded -- and
242	* we're not going to touch it.
243	*/
244	if (ctl->mod_loadcnt != sdp->sdp_loadcnt) {
245	if (sdt_verbose) {
246	cmn_err(CE_NOTE, "sdt is failing for probe %s "
247	"(module %s reloaded)",
248	sdp->sdp_name, ctl->mod_modname);
249	}
250	goto err;
251	}
252	#endif /* __APPLE__ */
253
254	#if defined (__ppc__) \|\| defined (__ppc64__)
255	dtrace_casptr(&tempDTraceIntHook, NULL, fbt_perfIntCallback);
256	if (tempDTraceIntHook != (perfCallback)fbt_perfIntCallback) {
257	if (sdt_verbose) {
258	cmn_err(CE_NOTE, "sdt_enable is failing for probe %s "
259	"in module %s: tempDTraceIntHook already occupied.",
260	sdp->sdp_name, ctl->mod_modname);
261	}
262	return;
263	}
264	#endif
265
266	dtrace_casptr(&tempDTraceTrapHook, NULL, fbt_perfCallback);
267	if (tempDTraceTrapHook != (perfCallback)fbt_perfCallback) {
268	if (sdt_verbose) {
269	cmn_err(CE_NOTE, "sdt_enable is failing for probe %s "
270	"in module %s: tempDTraceTrapHook already occupied.",
271	sdp->sdp_name, ctl->mod_modname);
272	}
273	return;
274	}
275
276	while (sdp != NULL) {
277	(void)ml_nofault_copy( (vm_offset_t)&sdp->sdp_patchval, (vm_offset_t)sdp->sdp_patchpoint,
278	sizeof(sdp->sdp_patchval));
279	sdp = sdp->sdp_next;
280	}
281	err:
282	;
283	}
284
285	/ARGSUSED/
286	static void
287	sdt_disable(void arg, dtrace_id_t id, void parg)
288	{
289	#pragma unused(arg,id)
290	sdt_probe_t *sdp = parg;
291	#if !defined(__APPLE__)
292	struct modctl *ctl = sdp->sdp_ctl;
293
294	ctl->mod_nenabled--;
295
296	if (!ctl->mod_loaded \|\| ctl->mod_loadcnt != sdp->sdp_loadcnt)
297	goto err;
298	#endif /* __APPLE__ */
299
300	while (sdp != NULL) {
301	(void)ml_nofault_copy( (vm_offset_t)&sdp->sdp_savedval, (vm_offset_t)sdp->sdp_patchpoint,
302	sizeof(sdp->sdp_savedval));
303	sdp = sdp->sdp_next;
304	}
305
306	err:
307	;
308	}
309
310	static dtrace_pops_t sdt_pops = {
311	NULL,
312	sdt_provide_module,
313	sdt_enable,
314	sdt_disable,
315	NULL,
316	NULL,
317	sdt_getargdesc,
318	NULL,
319	NULL,
320	sdt_destroy
321	};
322
323	/ARGSUSED/
324	static int
325	sdt_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
326	{
327	#pragma unused(cmd)
328	sdt_provider_t *prov;
329
330	if (ddi_create_minor_node(devi, "sdt", S_IFCHR,
331	0, DDI_PSEUDO, 0) == DDI_FAILURE) {
332	cmn_err(CE_NOTE, "/dev/sdt couldn't create minor node");
333	ddi_remove_minor_node(devi, NULL);
334	return (DDI_FAILURE);
335	}
336
337	ddi_report_dev(devi);
338	sdt_devi = devi;
339
340	if (sdt_probetab_size == 0)
341	sdt_probetab_size = SDT_PROBETAB_SIZE;
342
343	sdt_probetab_mask = sdt_probetab_size - 1;
344	sdt_probetab =
345	kmem_zalloc(sdt_probetab_size * sizeof (sdt_probe_t *), KM_SLEEP);
346	dtrace_invop_add(sdt_invop);
347
348	for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
349	if (dtrace_register(prov->sdtp_name, prov->sdtp_attr,
350	DTRACE_PRIV_KERNEL, NULL,
351	&sdt_pops, prov, &prov->sdtp_id) != 0) {
352	cmn_err(CE_WARN, "failed to register sdt provider %s",
353	prov->sdtp_name);
354	}
355	}
356
357	return (DDI_SUCCESS);
358	}
359
360	#if !defined(__APPLE__)
361	/ARGSUSED/
362	static int
363	sdt_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
364	{
365	sdt_provider_t *prov;
366
367	switch (cmd) {
368	case DDI_DETACH:
369	break;
370
371	case DDI_SUSPEND:
372	return (DDI_SUCCESS);
373
374	default:
375	return (DDI_FAILURE);
376	}
377
378	for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
379	if (prov->sdtp_id != DTRACE_PROVNONE) {
380	if (dtrace_unregister(prov->sdtp_id) != 0)
381	return (DDI_FAILURE);
382
383	prov->sdtp_id = DTRACE_PROVNONE;
384	}
385	}
386
387	dtrace_invop_remove(sdt_invop);
388	kmem_free(sdt_probetab, sdt_probetab_size * sizeof (sdt_probe_t *));
389
390	return (DDI_SUCCESS);
391	}
392
393	/ARGSUSED/
394	static int
395	sdt_info(dev_info_t dip, ddi_info_cmd_t infocmd, void arg, void **result)
396	{
397	int error;
398
399	switch (infocmd) {
400	case DDI_INFO_DEVT2DEVINFO:
401	result = (void )sdt_devi;
402	error = DDI_SUCCESS;
403	break;
404	case DDI_INFO_DEVT2INSTANCE:
405	result = (void )0;
406	error = DDI_SUCCESS;
407	break;
408	default:
409	error = DDI_FAILURE;
410	}
411	return (error);
412	}
413
414	/ARGSUSED/
415	static int
416	sdt_open(dev_t devp, int flag, int otyp, cred_t cred_p)
417	{
418	return (0);
419	}
420
421	static struct cb_ops sdt_cb_ops = {
422	sdt_open, /* open */
423	nodev, /* close */
424	nulldev, /* strategy */
425	nulldev, /* print */
426	nodev, /* dump */
427	nodev, /* read */
428	nodev, /* write */
429	nodev, /* ioctl */
430	nodev, /* devmap */
431	nodev, /* mmap */
432	nodev, /* segmap */
433	nochpoll, /* poll */
434	ddi_prop_op, /* cb_prop_op */
435	0, /* streamtab */
436	D_NEW \| D_MP /* Driver compatibility flag */
437	};
438
439	static struct dev_ops sdt_ops = {
440	DEVO_REV, /* devo_rev, */
441	0, /* refcnt */
442	sdt_info, /* get_dev_info */
443	nulldev, /* identify */
444	nulldev, /* probe */
445	sdt_attach, /* attach */
446	sdt_detach, /* detach */
447	nodev, /* reset */
448	&sdt_cb_ops, /* driver operations */
449	NULL, /* bus operations */
450	nodev /* dev power */
451	};
452
453	/*
454	* Module linkage information for the kernel.
455	*/
456	static struct modldrv modldrv = {
457	&mod_driverops, /* module type (this is a pseudo driver) */
458	"Statically Defined Tracing", /* name of module */
459	&sdt_ops, /* driver ops */
460	};
461
462	static struct modlinkage modlinkage = {
463	MODREV_1,
464	(void *)&modldrv,
465	NULL
466	};
467
468	int
469	_init(void)
470	{
471	return (mod_install(&modlinkage));
472	}
473
474	int
475	_info(struct modinfo *modinfop)
476	{
477	return (mod_info(&modlinkage, modinfop));
478	}
479
480	int
481	_fini(void)
482	{
483	return (mod_remove(&modlinkage));
484	}
485	#else
486	d_open_t _sdt_open;
487
488	int _sdt_open(dev_t dev, int flags, int devtype, struct proc *p)
489	{
490	#pragma unused(dev,flags,devtype,p)
491	return 0;
492	}
493
494	#define SDT_MAJOR -24 /* let the kernel pick the device number */
495
496	/*
497	* A struct describing which functions will get invoked for certain
498	* actions.
499	*/
500	static struct cdevsw sdt_cdevsw =
501	{
502	_sdt_open, /* open */
503	eno_opcl, /* close */
504	eno_rdwrt, /* read */
505	eno_rdwrt, /* write */
506	eno_ioctl, /* ioctl */
507	(stop_fcn_t )nulldev, / stop */
508	(reset_fcn_t )nulldev, / reset */
509	NULL, /* tty's */
510	eno_select, /* select */
511	eno_mmap, /* mmap */
512	eno_strat, /* strategy */
513	eno_getc, /* getc */
514	eno_putc, /* putc */
515	0 /* type */
516	};
517
518	static int gSDTInited = 0;
519	static struct modctl g_sdt_kernctl;
520	static struct module g_sdt_mach_module;
521
522	#include <mach-o/loader.h>
523	#include <mach-o/nlist.h>
524
525	extern struct mach_header _mh_execute_header; /* the kernel's mach header */
526
527	void sdt_init( void )
528	{
529	if (0 == gSDTInited)
530	{
531	int majdevno = cdevsw_add(SDT_MAJOR, &sdt_cdevsw);
532
533	if (majdevno < 0) {
534	printf("sdt_init: failed to allocate a major number!\n");
535	gSDTInited = 0;
536	return;
537	}
538
539	if (MH_MAGIC != _mh_execute_header.magic) {
540	g_sdt_kernctl.address = (vm_address_t)NULL;
541	g_sdt_kernctl.size = 0;
542	} else {
543	struct mach_header *mh;
544	struct load_command *cmd;
545	struct segment_command orig_ts = NULL, orig_le = NULL;
546	struct symtab_command *orig_st = NULL;
547	struct nlist *sym = NULL;
548	char *strings;
549	unsigned int i;
550
551	g_sdt_mach_module.sdt_nprobes = 0;
552	g_sdt_mach_module.sdt_probes = NULL;
553
554	g_sdt_kernctl.address = (vm_address_t)&g_sdt_mach_module;
555	g_sdt_kernctl.size = 0;
556	strncpy((char *)&(g_sdt_kernctl.mod_modname), "mach_kernel", KMOD_MAX_NAME);
557
558	mh = &_mh_execute_header;
559	cmd = (struct load_command *) &mh[1];
560	for (i = 0; i < mh->ncmds; i++) {
561	if (cmd->cmd == LC_SEGMENT) {
562	struct segment_command orig_sg = (struct segment_command ) cmd;
563
564	if (strcmp(SEG_TEXT, orig_sg->segname) == 0)
565	orig_ts = orig_sg;
566	else if (strcmp(SEG_LINKEDIT, orig_sg->segname) == 0)
567	orig_le = orig_sg;
568	else if (strcmp("", orig_sg->segname) == 0)
569	orig_ts = orig_sg; /* kexts have a single unnamed segment */
570	}
571	else if (cmd->cmd == LC_SYMTAB)
572	orig_st = (struct symtab_command *) cmd;
573
574	cmd = (struct load_command *) ((caddr_t) cmd + cmd->cmdsize);
575	}
576
577	if ((orig_ts == NULL) \|\| (orig_st == NULL) \|\| (orig_le == NULL))
578	return;
579
580	sym = (struct nlist *)orig_le->vmaddr;
581	strings = ((char )sym) + orig_st->nsyms sizeof(struct nlist);
582
583	for (i = 0; i < orig_st->nsyms; i++) {
584	uint8_t n_type = sym[i].n_type & (N_TYPE \| N_EXT);
585	char *name = strings + sym[i].n_un.n_strx;
586	char *prev_name;
587	unsigned long best;
588	unsigned int j;
589
590	/* Check that the symbol is a global and that it has a name. */
591	if (((N_SECT \| N_EXT) != n_type && (N_ABS \| N_EXT) != n_type))
592	continue;
593
594	if (0 == sym[i].n_un.n_strx) /* iff a null, "", name. */
595	continue;
596
597	/* Lop off omnipresent leading underscore. */
598	if (*name == '_')
599	name += 1;
600
601	if (strstr(name, "_dtrace_probe$")) {
602	sdt_probedesc_t *sdpd = kmem_alloc(sizeof(sdt_probedesc_t), KM_SLEEP);
603	int len = strlen(name) + 1;
604
605	sdpd->sdpd_name = kmem_alloc(len, KM_SLEEP);
606	strncpy(sdpd->sdpd_name, name, len); /* NUL termination is ensured. */
607
608	prev_name = "<unknown>";
609	best = 0;
610	for (j = 0; j < orig_st->nsyms; j++) {
611	uint8_t n_type = sym[j].n_type & (N_TYPE \| N_EXT);
612	char *name = strings + sym[j].n_un.n_strx;
613
614	if (((N_SECT \| N_EXT) != n_type && (N_ABS \| N_EXT) != n_type))
615	continue;
616
617	if (0 == sym[j].n_un.n_strx) /* iff a null, "", name. */
618	continue;
619
620	if (*name == '_')
621	name += 1;
622	if (strstr(name, "_dtrace_probe$"))
623	continue;
624
625	if ((unsigned long )sym[i].n_value <= (unsigned long)sym[j].n_value)
626	continue;
627
628	if ((unsigned long)sym[j].n_value > best) {
629	best = (unsigned long)sym[j].n_value;
630	prev_name = name;
631	}
632	}
633
634	sdpd->sdpd_func = kmem_alloc((len = strlen(prev_name) + 1), KM_SLEEP);
635	strncpy(sdpd->sdpd_func, prev_name, len); /* NUL termination is ensured. */
636
637	sdpd->sdpd_offset = (unsigned long )sym[i].n_value;
638
639	sdpd->sdpd_next = g_sdt_mach_module.sdt_probes;
640	g_sdt_mach_module.sdt_probes = sdpd;
641	} else {
642	prev_name = name;
643	}
644	}
645	}
646
647	sdt_attach( (dev_info_t *)majdevno, DDI_ATTACH );
648
649	gSDTInited = 1;
650	} else
651	panic("sdt_init: called twice!\n");
652	}
653	#undef SDT_MAJOR
654
655	/ARGSUSED/
656	void
657	sdt_provide_module(void arg, struct modctl ctl)
658	{
659	#pragma unused(ctl)
660	__sdt_provide_module(arg, &g_sdt_kernctl);
661
662	sdt_probedesc_t *sdpd = g_sdt_mach_module.sdt_probes;
663	while (sdpd) {
664	sdt_probedesc_t *this_sdpd = sdpd;
665	kmem_free((void *)sdpd->sdpd_name, strlen(sdpd->sdpd_name) + 1);
666	kmem_free((void *)sdpd->sdpd_func, strlen(sdpd->sdpd_func) + 1);
667	sdpd = sdpd->sdpd_next;
668	kmem_free((void *)this_sdpd, sizeof(sdt_probedesc_t));
669	}
670	g_sdt_mach_module.sdt_probes = NULL;
671	}
672
673	#endif /* __APPLE__ */