X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/c910b4d9d2451126ae3917b931cd4390c11e1d52..refs/heads/master:/bsd/dev/dtrace/sdt.c diff --git a/bsd/dev/dtrace/sdt.c b/bsd/dev/dtrace/sdt.c index 946c6a4c6..a3722f61e 100644 --- a/bsd/dev/dtrace/sdt.c +++ b/bsd/dev/dtrace/sdt.c @@ -19,18 +19,10 @@ * CDDL HEADER END */ /* - * Copyright 2006 Sun Microsystems, Inc. All rights reserved. + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ -/* #pragma ident "@(#)sdt.c 1.6 06/03/24 SMI" */ - -#ifdef KERNEL -#ifndef _KERNEL -#define _KERNEL /* Solaris vs. Darwin */ -#endif -#endif - #include #include #include @@ -40,109 +32,110 @@ #include #include +#if defined(__arm__) || defined(__arm64__) +#include +#endif /* defined(__arm__) || defined(__arm64__) */ + #include #include #include #include +extern int dtrace_kernel_symbol_mode; -struct savearea_t; /* Used anonymously */ -typedef kern_return_t (*perfCallback)(int, struct savearea_t *, int, int); +#include -#if defined (__ppc__) || defined (__ppc64__) -extern perfCallback tempDTraceTrapHook, tempDTraceIntHook; -extern kern_return_t fbt_perfCallback(int, struct savearea_t *, int, int); -extern kern_return_t fbt_perfIntCallback(int, struct savearea_t *, int, int); +/* #include address; + struct module *mp = (struct module *)ctl->mod_address; char *modname = ctl->mod_modname; sdt_probedesc_t *sdpd; sdt_probe_t *sdp, *old; sdt_provider_t *prov; - int len; /* * One for all, and all for one: if we haven't yet registered all of * our providers, we'll refuse to provide anything. */ for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) { - if (prov->sdtp_id == DTRACE_PROVNONE) + if (prov->sdtp_id == DTRACE_PROVNONE) { return; + } } - if (mp->sdt_nprobes != 0 || (sdpd = mp->sdt_probes) == NULL) + if (!mp || mp->sdt_nprobes != 0 || (sdpd = mp->sdt_probes) == NULL) { return; + } for (sdpd = mp->sdt_probes; sdpd != NULL; sdpd = sdpd->sdpd_next) { - char *name = sdpd->sdpd_name, *func, *nname; - int i, j; + const char *func; dtrace_id_t id; - for (prov = sdt_providers; prov->sdtp_prefix != NULL; prov++) { - char *prefpart, *prefix = prov->sdtp_prefix; - - if ((prefpart = strstr(name, prefix))) { - name = prefpart + strlen(prefix); + /* Validate probe's provider name. Do not provide probes for unknown providers. */ + for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) { + if (strcmp(prov->sdtp_prefix, sdpd->sdpd_prov) == 0) { break; } } - nname = kmem_alloc(len = strlen(name) + 1, KM_SLEEP); - - for (i = 0, j = 0; name[j] != '\0'; i++) { - if (name[j] == '_' && name[j + 1] == '_') { - nname[i] = '-'; - j += 2; - } else { - nname[i] = name[j++]; - } + if (prov->sdtp_name == NULL) { + printf("Ignoring probes from unsupported provider %s\n", sdpd->sdpd_prov); + continue; } - nname[i] = '\0'; - - sdp = kmem_zalloc(sizeof (sdt_probe_t), KM_SLEEP); + sdp = kmem_zalloc(sizeof(sdt_probe_t), KM_SLEEP); sdp->sdp_loadcnt = ctl->mod_loadcnt; sdp->sdp_ctl = ctl; - sdp->sdp_name = nname; - sdp->sdp_namelen = len; + sdp->sdp_name = kmem_alloc(strlen(sdpd->sdpd_name) + 1, KM_SLEEP); + strncpy(sdp->sdp_name, sdpd->sdpd_name, strlen(sdpd->sdpd_name) + 1); + sdp->sdp_namelen = strlen(sdpd->sdpd_name) + 1; sdp->sdp_provider = prov; - func = sdpd->sdpd_func; - - if (func == NULL) - func = ""; + func = (sdpd->sdpd_func != NULL) ? sdpd->sdpd_func : ""; /* * We have our provider. Now create the probe. */ if ((id = dtrace_probe_lookup(prov->sdtp_id, modname, - func, nname)) != DTRACE_IDNONE) { + func, sdp->sdp_name)) != DTRACE_IDNONE) { old = dtrace_probe_arg(prov->sdtp_id, id); ASSERT(old != NULL); @@ -151,11 +144,16 @@ __sdt_provide_module(void *arg, struct modctl *ctl) old->sdp_next = sdp; } else { sdp->sdp_id = dtrace_probe_create(prov->sdtp_id, - modname, func, nname, SDT_AFRAMES, sdp); + modname, func, sdp->sdp_name, SDT_AFRAMES, sdp); mp->sdt_nprobes++; } +#if 0 + printf("__sdt_provide_module: sdpd=0x%p sdp=0x%p name=%s, id=%d\n", sdpd, sdp, + sdp->sdp_name, sdp->sdp_id); +#endif + sdp->sdp_hashnext = sdt_probetab[SDT_ADDR2NDX(sdpd->sdpd_offset)]; sdt_probetab[SDT_ADDR2NDX(sdpd->sdpd_offset)] = sdp; @@ -173,7 +171,11 @@ sdt_destroy(void *arg, dtrace_id_t id, void *parg) #pragma unused(arg,id) sdt_probe_t *sdp = parg, *old, *last, *hash; int ndx; + #if !defined(__APPLE__) + /* + * APPLE NOTE: sdt probes for kexts not yet implemented + */ struct modctl *ctl = sdp->sdp_ctl; if (ctl != NULL && ctl->mod_loadcnt == sdp->sdp_loadcnt) { @@ -208,19 +210,18 @@ sdt_destroy(void *arg, dtrace_id_t id, void *parg) kmem_free(sdp->sdp_name, sdp->sdp_namelen); sdp = sdp->sdp_next; - kmem_free(old, sizeof (sdt_probe_t)); + kmem_free(old, sizeof(sdt_probe_t)); } } /*ARGSUSED*/ -static void +static int sdt_enable(void *arg, dtrace_id_t id, void *parg) { #pragma unused(arg,id) sdt_probe_t *sdp = parg; struct modctl *ctl = sdp->sdp_ctl; -#if !defined(__APPLE__) ctl->mod_nenabled++; /* @@ -249,37 +250,33 @@ sdt_enable(void *arg, dtrace_id_t id, void *parg) } goto err; } -#endif /* __APPLE__ */ -#if defined (__ppc__) || defined (__ppc64__) - dtrace_casptr(&tempDTraceIntHook, NULL, fbt_perfIntCallback); - if (tempDTraceIntHook != (perfCallback)fbt_perfIntCallback) { - if (sdt_verbose) { - cmn_err(CE_NOTE, "sdt_enable is failing for probe %s " - "in module %s: tempDTraceIntHook already occupied.", - sdp->sdp_name, ctl->mod_modname); - } - return; - } -#endif - - dtrace_casptr(&tempDTraceTrapHook, NULL, fbt_perfCallback); + dtrace_casptr(&tempDTraceTrapHook, NULL, ptrauth_nop_cast(void *, &fbt_perfCallback)); if (tempDTraceTrapHook != (perfCallback)fbt_perfCallback) { if (sdt_verbose) { cmn_err(CE_NOTE, "sdt_enable is failing for probe %s " "in module %s: tempDTraceTrapHook already occupied.", sdp->sdp_name, ctl->mod_modname); } - return; + return 0; } while (sdp != NULL) { - (void)ml_nofault_copy( (vm_offset_t)&sdp->sdp_patchval, (vm_offset_t)sdp->sdp_patchpoint, - sizeof(sdp->sdp_patchval)); + (void)ml_nofault_copy((vm_offset_t)&sdp->sdp_patchval, (vm_offset_t)sdp->sdp_patchpoint, + (vm_size_t)sizeof(sdp->sdp_patchval)); + + /* + * Make the patched instruction visible via a data + instruction + * cache fush on platforms that need it + */ + flush_dcache((vm_offset_t)sdp->sdp_patchpoint, (vm_size_t)sizeof(sdp->sdp_patchval), 0); + invalidate_icache((vm_offset_t)sdp->sdp_patchpoint, (vm_size_t)sizeof(sdp->sdp_patchval), 0); + sdp = sdp->sdp_next; } + err: - ; + return 0; } /*ARGSUSED*/ @@ -288,18 +285,23 @@ sdt_disable(void *arg, dtrace_id_t id, void *parg) { #pragma unused(arg,id) sdt_probe_t *sdp = parg; -#if !defined(__APPLE__) struct modctl *ctl = sdp->sdp_ctl; ctl->mod_nenabled--; - if (!ctl->mod_loaded || ctl->mod_loadcnt != sdp->sdp_loadcnt) + if (!ctl->mod_loaded || ctl->mod_loadcnt != sdp->sdp_loadcnt) { goto err; -#endif /* __APPLE__ */ + } while (sdp != NULL) { - (void)ml_nofault_copy( (vm_offset_t)&sdp->sdp_savedval, (vm_offset_t)sdp->sdp_patchpoint, - sizeof(sdp->sdp_savedval)); + (void)ml_nofault_copy((vm_offset_t)&sdp->sdp_savedval, (vm_offset_t)sdp->sdp_patchpoint, + (vm_size_t)sizeof(sdp->sdp_savedval)); + /* + * Make the patched instruction visible via a data + instruction + * cache flush on platforms that need it + */ + flush_dcache((vm_offset_t)sdp->sdp_patchpoint, (vm_size_t)sizeof(sdp->sdp_savedval), 0); + invalidate_icache((vm_offset_t)sdp->sdp_patchpoint, (vm_size_t)sizeof(sdp->sdp_savedval), 0); sdp = sdp->sdp_next; } @@ -308,41 +310,38 @@ err: } static dtrace_pops_t sdt_pops = { - NULL, - sdt_provide_module, - sdt_enable, - sdt_disable, - NULL, - NULL, - sdt_getargdesc, - NULL, - NULL, - sdt_destroy + .dtps_provide = NULL, + .dtps_provide_module = sdt_provide_module, + .dtps_enable = sdt_enable, + .dtps_disable = sdt_disable, + .dtps_suspend = NULL, + .dtps_resume = NULL, + .dtps_getargdesc = sdt_getargdesc, + .dtps_getargval = sdt_getarg, + .dtps_usermode = NULL, + .dtps_destroy = sdt_destroy, }; /*ARGSUSED*/ static int -sdt_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) +sdt_attach(dev_info_t *devi) { -#pragma unused(cmd) sdt_provider_t *prov; if (ddi_create_minor_node(devi, "sdt", S_IFCHR, 0, DDI_PSEUDO, 0) == DDI_FAILURE) { cmn_err(CE_NOTE, "/dev/sdt couldn't create minor node"); ddi_remove_minor_node(devi, NULL); - return (DDI_FAILURE); + return DDI_FAILURE; } - ddi_report_dev(devi); - sdt_devi = devi; - - if (sdt_probetab_size == 0) + if (sdt_probetab_size == 0) { sdt_probetab_size = SDT_PROBETAB_SIZE; + } sdt_probetab_mask = sdt_probetab_size - 1; sdt_probetab = - kmem_zalloc(sdt_probetab_size * sizeof (sdt_probe_t *), KM_SLEEP); + kmem_zalloc(sdt_probetab_size * sizeof(sdt_probe_t *), KM_SLEEP); dtrace_invop_add(sdt_invop); for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) { @@ -354,9 +353,12 @@ sdt_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) } } - return (DDI_SUCCESS); + return DDI_SUCCESS; } +/* + * APPLE NOTE: sdt_detach not implemented + */ #if !defined(__APPLE__) /*ARGSUSED*/ static int @@ -369,123 +371,33 @@ sdt_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) break; case DDI_SUSPEND: - return (DDI_SUCCESS); + return DDI_SUCCESS; default: - return (DDI_FAILURE); + return DDI_FAILURE; } for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) { if (prov->sdtp_id != DTRACE_PROVNONE) { - if (dtrace_unregister(prov->sdtp_id) != 0) - return (DDI_FAILURE); + if (dtrace_unregister(prov->sdtp_id) != 0) { + return DDI_FAILURE; + } prov->sdtp_id = DTRACE_PROVNONE; } } dtrace_invop_remove(sdt_invop); - kmem_free(sdt_probetab, sdt_probetab_size * sizeof (sdt_probe_t *)); - - return (DDI_SUCCESS); -} - -/*ARGSUSED*/ -static int -sdt_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result) -{ - int error; - - switch (infocmd) { - case DDI_INFO_DEVT2DEVINFO: - *result = (void *)sdt_devi; - error = DDI_SUCCESS; - break; - case DDI_INFO_DEVT2INSTANCE: - *result = (void *)0; - error = DDI_SUCCESS; - break; - default: - error = DDI_FAILURE; - } - return (error); -} - -/*ARGSUSED*/ -static int -sdt_open(dev_t *devp, int flag, int otyp, cred_t *cred_p) -{ - return (0); -} - -static struct cb_ops sdt_cb_ops = { - sdt_open, /* open */ - nodev, /* close */ - nulldev, /* strategy */ - nulldev, /* print */ - nodev, /* dump */ - nodev, /* read */ - nodev, /* write */ - nodev, /* ioctl */ - nodev, /* devmap */ - nodev, /* mmap */ - nodev, /* segmap */ - nochpoll, /* poll */ - ddi_prop_op, /* cb_prop_op */ - 0, /* streamtab */ - D_NEW | D_MP /* Driver compatibility flag */ -}; - -static struct dev_ops sdt_ops = { - DEVO_REV, /* devo_rev, */ - 0, /* refcnt */ - sdt_info, /* get_dev_info */ - nulldev, /* identify */ - nulldev, /* probe */ - sdt_attach, /* attach */ - sdt_detach, /* detach */ - nodev, /* reset */ - &sdt_cb_ops, /* driver operations */ - NULL, /* bus operations */ - nodev /* dev power */ -}; - -/* - * Module linkage information for the kernel. - */ -static struct modldrv modldrv = { - &mod_driverops, /* module type (this is a pseudo driver) */ - "Statically Defined Tracing", /* name of module */ - &sdt_ops, /* driver ops */ -}; - -static struct modlinkage modlinkage = { - MODREV_1, - (void *)&modldrv, - NULL -}; - -int -_init(void) -{ - return (mod_install(&modlinkage)); -} + kmem_free(sdt_probetab, sdt_probetab_size * sizeof(sdt_probe_t *)); -int -_info(struct modinfo *modinfop) -{ - return (mod_info(&modlinkage, modinfop)); + return DDI_SUCCESS; } +#endif /* __APPLE__ */ -int -_fini(void) -{ - return (mod_remove(&modlinkage)); -} -#else d_open_t _sdt_open; -int _sdt_open(dev_t dev, int flags, int devtype, struct proc *p) +int +_sdt_open(dev_t dev, int flags, int devtype, struct proc *p) { #pragma unused(dev,flags,devtype,p) return 0; @@ -493,182 +405,244 @@ int _sdt_open(dev_t dev, int flags, int devtype, struct proc *p) #define SDT_MAJOR -24 /* let the kernel pick the device number */ -/* - * A struct describing which functions will get invoked for certain - * actions. - */ -static struct cdevsw sdt_cdevsw = +static const struct cdevsw sdt_cdevsw = { - _sdt_open, /* open */ - eno_opcl, /* close */ - eno_rdwrt, /* read */ - eno_rdwrt, /* write */ - eno_ioctl, /* ioctl */ - (stop_fcn_t *)nulldev, /* stop */ - (reset_fcn_t *)nulldev, /* reset */ - NULL, /* tty's */ - eno_select, /* select */ - eno_mmap, /* mmap */ - eno_strat, /* strategy */ - eno_getc, /* getc */ - eno_putc, /* putc */ - 0 /* type */ + .d_open = _sdt_open, + .d_close = eno_opcl, + .d_read = eno_rdwrt, + .d_write = eno_rdwrt, + .d_ioctl = eno_ioctl, + .d_stop = (stop_fcn_t *)nulldev, + .d_reset = (reset_fcn_t *)nulldev, + .d_select = eno_select, + .d_mmap = eno_mmap, + .d_strategy = eno_strat, + .d_reserved_1 = eno_getc, + .d_reserved_2 = eno_putc, }; -static int gSDTInited = 0; static struct modctl g_sdt_kernctl; static struct module g_sdt_mach_module; -#include #include +#include -extern struct mach_header _mh_execute_header; /* the kernel's mach header */ +/* + * Represents single record in __DATA,__sdt section. + */ +typedef struct dtrace_sdt_def { + uintptr_t dsd_addr; /* probe site location */ + const char *dsd_prov; /* provider's name */ + const char *dsd_name; /* probe's name */ +} __attribute__((__packed__)) dtrace_sdt_def_t; -void sdt_init( void ) +/* + * Creates a copy of name and unescapes '-' characters. + */ +static char * +sdt_strdup_name(const char *name) { - if (0 == gSDTInited) - { - int majdevno = cdevsw_add(SDT_MAJOR, &sdt_cdevsw); - - if (majdevno < 0) { - printf("sdt_init: failed to allocate a major number!\n"); - gSDTInited = 0; - return; + size_t len = strlen(name) + 1; + size_t i, j; + char *nname = kmem_alloc(len, KM_SLEEP); + + for (i = 0, j = 0; name[j] != '\0'; i++) { + if (name[j] == '_' && name[j + 1] == '_') { + nname[i] = '-'; + j += 2; + } else { + nname[i] = name[j++]; } + } - if (MH_MAGIC != _mh_execute_header.magic) { - g_sdt_kernctl.address = (vm_address_t)NULL; - g_sdt_kernctl.size = 0; - } else { - struct mach_header *mh; - struct load_command *cmd; - struct segment_command *orig_ts = NULL, *orig_le = NULL; - struct symtab_command *orig_st = NULL; - struct nlist *sym = NULL; - char *strings; - unsigned int i; + nname[i] = '\0'; + return nname; +} + +void +sdt_early_init( void ) +{ + if (dtrace_sdt_probes_restricted()) { + return; + } + if (MH_MAGIC_KERNEL != _mh_execute_header.magic) { + g_sdt_kernctl.mod_address = (vm_address_t)NULL; + g_sdt_kernctl.mod_size = 0; + } else { + kernel_mach_header_t *mh; + struct load_command *cmd; + kernel_segment_command_t *orig_ts = NULL, *orig_le = NULL; + kernel_section_t *orig_dt = NULL; + struct symtab_command *orig_st = NULL; + kernel_nlist_t *sym = NULL; + char *strings; + unsigned int i; + unsigned int len; g_sdt_mach_module.sdt_nprobes = 0; g_sdt_mach_module.sdt_probes = NULL; - g_sdt_kernctl.address = (vm_address_t)&g_sdt_mach_module; - g_sdt_kernctl.size = 0; + g_sdt_kernctl.mod_address = (vm_address_t)&g_sdt_mach_module; + g_sdt_kernctl.mod_size = 0; strncpy((char *)&(g_sdt_kernctl.mod_modname), "mach_kernel", KMOD_MAX_NAME); + g_sdt_kernctl.mod_next = NULL; + g_sdt_kernctl.mod_stale = NULL; + g_sdt_kernctl.mod_id = 0; + g_sdt_kernctl.mod_loadcnt = 1; + g_sdt_kernctl.mod_loaded = 1; + g_sdt_kernctl.mod_flags = 0; + g_sdt_kernctl.mod_nenabled = 0; + mh = &_mh_execute_header; - cmd = (struct load_command *) &mh[1]; - for (i = 0; i < mh->ncmds; i++) { - if (cmd->cmd == LC_SEGMENT) { - struct segment_command *orig_sg = (struct segment_command *) cmd; - - if (strcmp(SEG_TEXT, orig_sg->segname) == 0) - orig_ts = orig_sg; - else if (strcmp(SEG_LINKEDIT, orig_sg->segname) == 0) - orig_le = orig_sg; - else if (strcmp("", orig_sg->segname) == 0) - orig_ts = orig_sg; /* kexts have a single unnamed segment */ - } - else if (cmd->cmd == LC_SYMTAB) - orig_st = (struct symtab_command *) cmd; - - cmd = (struct load_command *) ((caddr_t) cmd + cmd->cmdsize); - } - - if ((orig_ts == NULL) || (orig_st == NULL) || (orig_le == NULL)) - return; - - sym = (struct nlist *)orig_le->vmaddr; - strings = ((char *)sym) + orig_st->nsyms * sizeof(struct nlist); - - for (i = 0; i < orig_st->nsyms; i++) { - uint8_t n_type = sym[i].n_type & (N_TYPE | N_EXT); - char *name = strings + sym[i].n_un.n_strx; - char *prev_name; - unsigned long best; - unsigned int j; - - /* Check that the symbol is a global and that it has a name. */ - if (((N_SECT | N_EXT) != n_type && (N_ABS | N_EXT) != n_type)) - continue; - - if (0 == sym[i].n_un.n_strx) /* iff a null, "", name. */ - continue; - - /* Lop off omnipresent leading underscore. */ - if (*name == '_') - name += 1; - - if (strstr(name, "_dtrace_probe$")) { - sdt_probedesc_t *sdpd = kmem_alloc(sizeof(sdt_probedesc_t), KM_SLEEP); - int len = strlen(name) + 1; - - sdpd->sdpd_name = kmem_alloc(len, KM_SLEEP); - strncpy(sdpd->sdpd_name, name, len); /* NUL termination is ensured. */ - - prev_name = ""; - best = 0; - for (j = 0; j < orig_st->nsyms; j++) { - uint8_t n_type = sym[j].n_type & (N_TYPE | N_EXT); - char *name = strings + sym[j].n_un.n_strx; - - if (((N_SECT | N_EXT) != n_type && (N_ABS | N_EXT) != n_type)) - continue; - - if (0 == sym[j].n_un.n_strx) /* iff a null, "", name. */ - continue; - - if (*name == '_') - name += 1; - if (strstr(name, "_dtrace_probe$")) - continue; - - if (*(unsigned long *)sym[i].n_value <= (unsigned long)sym[j].n_value) - continue; - - if ((unsigned long)sym[j].n_value > best) { - best = (unsigned long)sym[j].n_value; - prev_name = name; - } - } - - sdpd->sdpd_func = kmem_alloc((len = strlen(prev_name) + 1), KM_SLEEP); - strncpy(sdpd->sdpd_func, prev_name, len); /* NUL termination is ensured. */ - - sdpd->sdpd_offset = *(unsigned long *)sym[i].n_value; - - sdpd->sdpd_next = g_sdt_mach_module.sdt_probes; - g_sdt_mach_module.sdt_probes = sdpd; - } else { - prev_name = name; + cmd = (struct load_command*) &mh[1]; + for (i = 0; i < mh->ncmds; i++) { + if (cmd->cmd == LC_SEGMENT_KERNEL) { + kernel_segment_command_t *orig_sg = (kernel_segment_command_t *) cmd; + + if (LIT_STRNEQL(orig_sg->segname, SEG_TEXT)) { + orig_ts = orig_sg; + } else if (LIT_STRNEQL(orig_sg->segname, SEG_LINKEDIT)) { + orig_le = orig_sg; + } else if (LIT_STRNEQL(orig_sg->segname, "")) { + orig_ts = orig_sg; /* kexts have a single unnamed segment */ + } + } else if (cmd->cmd == LC_SYMTAB) { + orig_st = (struct symtab_command *) cmd; + } + + cmd = (struct load_command *) ((uintptr_t) cmd + cmd->cmdsize); + } + + /* Locate DTrace SDT section in the object. */ + if ((orig_dt = getsectbyname("__DATA", "__sdt")) == NULL) { + printf("DTrace section not found.\n"); + return; + } + + if ((orig_ts == NULL) || (orig_st == NULL) || (orig_le == NULL)) { + return; + } + + sym = (kernel_nlist_t *)(orig_le->vmaddr + orig_st->symoff - orig_le->fileoff); + strings = (char *)(orig_le->vmaddr + orig_st->stroff - orig_le->fileoff); + + /* + * Iterate over SDT section and establish all SDT probes. + */ + dtrace_sdt_def_t *sdtdef = (dtrace_sdt_def_t *)(orig_dt->addr); + for (size_t k = 0; k < orig_dt->size / sizeof(dtrace_sdt_def_t); k++, sdtdef++) { + const char *funcname; + unsigned long best; + + sdt_probedesc_t *sdpd = kmem_alloc(sizeof(sdt_probedesc_t), KM_SLEEP); + + /* Unescape probe name and keep a note of the size of original memory allocation. */ + sdpd->sdpd_name = sdt_strdup_name(sdtdef->dsd_name); + sdpd->sdpd_namelen = strlen(sdtdef->dsd_name) + 1; + + /* Used only for provider structure lookup so there is no need to make dynamic copy. */ + sdpd->sdpd_prov = sdtdef->dsd_prov; + + /* + * Find the symbol immediately preceding the sdt probe site just discovered, + * that symbol names the function containing the sdt probe. + */ + funcname = ""; + for (i = 0; i < orig_st->nsyms; i++) { + uint8_t jn_type = sym[i].n_type & N_TYPE; + char *jname = strings + sym[i].n_un.n_strx; + + if ((N_SECT != jn_type && N_ABS != jn_type)) { + continue; + } + + if (0 == sym[i].n_un.n_strx) { /* iff a null, "", name. */ + continue; + } + + if (*jname == '_') { + jname += 1; + } + + if (sdtdef->dsd_addr <= (unsigned long)sym[i].n_value) { + continue; + } + + if ((unsigned long)sym[i].n_value > best) { + best = (unsigned long)sym[i].n_value; + funcname = jname; } } + + len = strlen(funcname) + 1; + sdpd->sdpd_func = kmem_alloc(len, KM_SLEEP); + strncpy(sdpd->sdpd_func, funcname, len); + + sdpd->sdpd_offset = sdtdef->dsd_addr; +#if defined(__arm__) + /* PR8353094 - mask off thumb-bit */ + sdpd->sdpd_offset &= ~0x1U; +#elif defined(__arm64__) + sdpd->sdpd_offset &= ~0x1LU; +#endif /* __arm__ */ + + sdpd->sdpd_next = g_sdt_mach_module.sdt_probes; + g_sdt_mach_module.sdt_probes = sdpd; } + } +} - sdt_attach( (dev_info_t *)majdevno, DDI_ATTACH ); +void +sdt_init( void ) +{ + int majdevno = cdevsw_add(SDT_MAJOR, &sdt_cdevsw); - gSDTInited = 1; - } else - panic("sdt_init: called twice!\n"); + if (majdevno < 0) { + printf("sdt_init: failed to allocate a major number!\n"); + return; + } + + if (dtrace_sdt_probes_restricted()) { + return; + } + + sdt_attach((dev_info_t*)(uintptr_t)majdevno); } + #undef SDT_MAJOR /*ARGSUSED*/ void sdt_provide_module(void *arg, struct modctl *ctl) { -#pragma unused(ctl) #pragma unused(arg) - __sdt_provide_module(arg, &g_sdt_kernctl); - - sdt_probedesc_t *sdpd = g_sdt_mach_module.sdt_probes; - while (sdpd) { - sdt_probedesc_t *this_sdpd = sdpd; - kmem_free((void *)sdpd->sdpd_name, strlen(sdpd->sdpd_name) + 1); - kmem_free((void *)sdpd->sdpd_func, strlen(sdpd->sdpd_func) + 1); - sdpd = sdpd->sdpd_next; - kmem_free((void *)this_sdpd, sizeof(sdt_probedesc_t)); + ASSERT(ctl != NULL); + ASSERT(dtrace_kernel_symbol_mode != DTRACE_KERNEL_SYMBOLS_NEVER); + LCK_MTX_ASSERT(&mod_lock, LCK_MTX_ASSERT_OWNED); + + if (MOD_SDT_DONE(ctl)) { + return; } - g_sdt_mach_module.sdt_probes = NULL; -} -#endif /* __APPLE__ */ + if (MOD_IS_MACH_KERNEL(ctl)) { + __sdt_provide_module(arg, &g_sdt_kernctl); + + sdt_probedesc_t *sdpd = g_sdt_mach_module.sdt_probes; + while (sdpd) { + sdt_probedesc_t *this_sdpd = sdpd; + kmem_free((void *)sdpd->sdpd_name, sdpd->sdpd_namelen); + kmem_free((void *)sdpd->sdpd_func, strlen(sdpd->sdpd_func) + 1); + sdpd = sdpd->sdpd_next; + kmem_free((void *)this_sdpd, sizeof(sdt_probedesc_t)); + } + g_sdt_mach_module.sdt_probes = NULL; + } else { + /* + * APPLE NOTE: sdt probes for kexts not yet implemented + */ + } + + /* Need to mark this module as completed */ + ctl->mod_flags |= MODCTL_SDT_PROBES_PROVIDED; +}