xnu-7195.101.1.tar.gz

[apple/xnu.git] / bsd / dev / dtrace / sdt.c

diff --git a/bsd/dev/dtrace/sdt.c b/bsd/dev/dtrace/sdt.c
index 610de106b6ea4e55fbb7469e4baa3e5a16746f4f..a3722f61ebe61ecd41d0c080f1586442235303b4 100644 (file)
--- a/bsd/dev/dtrace/sdt.c
+++ b/bsd/dev/dtrace/sdt.c
@@ -23,14 +23,6 @@
  * Use is subject to license terms.
  */
 
  * Use is subject to license terms.
  */
 

-/* #pragma ident       "@(#)sdt.c      1.9     08/07/01 SMI" */
-
-#ifdef KERNEL
-#ifndef _KERNEL
-#define _KERNEL /* Solaris vs. Darwin */
-#endif
-#endif
-

 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/errno.h>
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/errno.h>


@@ -40,6 +32,9 @@
 #include <sys/fcntl.h>
 #include <miscfs/devfs/devfs.h>
 
 #include <sys/fcntl.h>
 #include <miscfs/devfs/devfs.h>
 

+#if defined(__arm__) || defined(__arm64__)
+#include <arm/caches_internal.h>
+#endif /* defined(__arm__) || defined(__arm64__) */

 
 #include <sys/dtrace.h>
 #include <sys/dtrace_impl.h>
 
 #include <sys/dtrace.h>
 #include <sys/dtrace_impl.h>


@@ -49,28 +44,41 @@
 #include <sys/sdt_impl.h>
 extern int dtrace_kernel_symbol_mode;
 
 #include <sys/sdt_impl.h>
 extern int dtrace_kernel_symbol_mode;
 

+#include <ptrauth.h>
+

 /* #include <machine/trap.h */
 struct savearea_t; /* Used anonymously */
 
 /* #include <machine/trap.h */
 struct savearea_t; /* Used anonymously */
 

-#if   defined(__x86_64__)
+#if defined(__arm__)
+typedef kern_return_t (*perfCallback)(int, struct savearea_t *, __unused int, __unused int);
+extern perfCallback tempDTraceTrapHook;
+extern kern_return_t fbt_perfCallback(int, struct savearea_t *, __unused int, __unused int);
+#define SDT_PATCHVAL    0xdefc
+#define SDT_AFRAMES             7
+#elif defined(__arm64__)
+typedef kern_return_t (*perfCallback)(int, struct savearea_t *, __unused int, __unused int);
+extern perfCallback tempDTraceTrapHook;
+extern kern_return_t fbt_perfCallback(int, struct savearea_t *, __unused int, __unused int);
+#define SDT_PATCHVAL    0xe7eeee7e
+#define SDT_AFRAMES             7
+#elif defined(__x86_64__)

 typedef kern_return_t (*perfCallback)(int, struct savearea_t *, uintptr_t *, int);
 extern perfCallback tempDTraceTrapHook;
 extern kern_return_t fbt_perfCallback(int, struct savearea_t *, uintptr_t *, int);
 typedef kern_return_t (*perfCallback)(int, struct savearea_t *, uintptr_t *, int);
 extern perfCallback tempDTraceTrapHook;
 extern kern_return_t fbt_perfCallback(int, struct savearea_t *, uintptr_t *, int);

-#define        SDT_PATCHVAL    0xf0
-#define        SDT_AFRAMES             6
+#define SDT_PATCHVAL    0xf0
+#define SDT_AFRAMES             6

 #else
 #error Unknown architecture
 #endif
 
 #else
 #error Unknown architecture
 #endif
 

-#define        SDT_PROBETAB_SIZE       0x1000          /* 4k entries -- 16K total */
+#define SDT_PROBETAB_SIZE       0x1000          /* 4k entries -- 16K total */

 
 #define DTRACE_PROBE_PREFIX "_dtrace_probe$"
 
 
 #define DTRACE_PROBE_PREFIX "_dtrace_probe$"
 

-static dev_info_t              *sdt_devi;
-static int                     sdt_verbose = 0;
-sdt_probe_t            **sdt_probetab;
-int                    sdt_probetab_size;
-int                    sdt_probetab_mask;
+static int                      sdt_verbose = 0;
+sdt_probe_t             **sdt_probetab;
+int                     sdt_probetab_size;
+int                     sdt_probetab_mask;

 
 /*ARGSUSED*/
 static void
 
 /*ARGSUSED*/
 static void


@@ -82,65 +90,52 @@ __sdt_provide_module(void *arg, struct modctl *ctl)
        sdt_probedesc_t *sdpd;
        sdt_probe_t *sdp, *old;
        sdt_provider_t *prov;
        sdt_probedesc_t *sdpd;
        sdt_probe_t *sdp, *old;
        sdt_provider_t *prov;

-       int len;

 
        /*
         * One for all, and all for one:  if we haven't yet registered all of
         * our providers, we'll refuse to provide anything.
         */
        for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
 
        /*
         * One for all, and all for one:  if we haven't yet registered all of
         * our providers, we'll refuse to provide anything.
         */
        for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {

-               if (prov->sdtp_id == DTRACE_PROVNONE)
+               if (prov->sdtp_id == DTRACE_PROVNONE) {

                        return;
                        return;

+               }

        }
 
        }
 

-       if (!mp || mp->sdt_nprobes != 0 || (sdpd = mp->sdt_probes) == NULL)
+       if (!mp || mp->sdt_nprobes != 0 || (sdpd = mp->sdt_probes) == NULL) {

                return;
                return;

+       }

 
        for (sdpd = mp->sdt_probes; sdpd != NULL; sdpd = sdpd->sdpd_next) {
 
        for (sdpd = mp->sdt_probes; sdpd != NULL; sdpd = sdpd->sdpd_next) {

-           const char *name = sdpd->sdpd_name, *func;
-           char *nname;
-               int i, j;
+               const char *func;

                dtrace_id_t id;
 
                dtrace_id_t id;
 

-               for (prov = sdt_providers; prov->sdtp_prefix != NULL; prov++) {
-                       const char *prefpart, *prefix = prov->sdtp_prefix;
-
-                       if ((prefpart = strstr(name, prefix))) {
-                               name = prefpart + strlen(prefix);
+               /* Validate probe's provider name. Do not provide probes for unknown providers. */
+               for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
+                       if (strcmp(prov->sdtp_prefix, sdpd->sdpd_prov) == 0) {

                                break;
                        }
                }
 
                                break;
                        }
                }
 

-               nname = kmem_alloc(len = strlen(name) + 1, KM_SLEEP);
-
-               for (i = 0, j = 0; name[j] != '\0'; i++) {
-                       if (name[j] == '_' && name[j + 1] == '_') {
-                               nname[i] = '-';
-                               j += 2;
-                       } else {
-                               nname[i] = name[j++];
-                       }
+               if (prov->sdtp_name == NULL) {
+                       printf("Ignoring probes from unsupported provider %s\n", sdpd->sdpd_prov);
+                       continue;

                }
 
                }
 

-               nname[i] = '\0';
-
-               sdp = kmem_zalloc(sizeof (sdt_probe_t), KM_SLEEP);
+               sdp = kmem_zalloc(sizeof(sdt_probe_t), KM_SLEEP);

                sdp->sdp_loadcnt = ctl->mod_loadcnt;
                sdp->sdp_ctl = ctl;
                sdp->sdp_loadcnt = ctl->mod_loadcnt;
                sdp->sdp_ctl = ctl;

-               sdp->sdp_name = nname;
-               sdp->sdp_namelen = len;
+               sdp->sdp_name = kmem_alloc(strlen(sdpd->sdpd_name) + 1, KM_SLEEP);
+               strncpy(sdp->sdp_name, sdpd->sdpd_name, strlen(sdpd->sdpd_name) + 1);
+               sdp->sdp_namelen = strlen(sdpd->sdpd_name) + 1;

                sdp->sdp_provider = prov;
 
                sdp->sdp_provider = prov;
 

-               func = sdpd->sdpd_func;
-
-               if (func == NULL)
-                       func = "<unknown>";
+               func = (sdpd->sdpd_func != NULL) ? sdpd->sdpd_func : "<unknown>";

 
                /*
                 * We have our provider.  Now create the probe.
                 */
                if ((id = dtrace_probe_lookup(prov->sdtp_id, modname,
 
                /*
                 * We have our provider.  Now create the probe.
                 */
                if ((id = dtrace_probe_lookup(prov->sdtp_id, modname,

-                   func, nname)) != DTRACE_IDNONE) {
+                   func, sdp->sdp_name)) != DTRACE_IDNONE) {

                        old = dtrace_probe_arg(prov->sdtp_id, id);
                        ASSERT(old != NULL);
 
                        old = dtrace_probe_arg(prov->sdtp_id, id);
                        ASSERT(old != NULL);
 


@@ -149,14 +144,15 @@ __sdt_provide_module(void *arg, struct modctl *ctl)
                        old->sdp_next = sdp;
                } else {
                        sdp->sdp_id = dtrace_probe_create(prov->sdtp_id,
                        old->sdp_next = sdp;
                } else {
                        sdp->sdp_id = dtrace_probe_create(prov->sdtp_id,

-                           modname, func, nname, SDT_AFRAMES, sdp);
+                           modname, func, sdp->sdp_name, SDT_AFRAMES, sdp);

 
                        mp->sdt_nprobes++;
                }
 
 
                        mp->sdt_nprobes++;
                }
 

-#if 0          
-               printf ("__sdt_provide_module:  sdpd=0x%p  sdp=0x%p  name=%s, id=%d\n", sdpd, sdp, nname, sdp->sdp_id);
-#endif         
+#if 0
+               printf("__sdt_provide_module:  sdpd=0x%p  sdp=0x%p  name=%s, id=%d\n", sdpd, sdp,
+                   sdp->sdp_name, sdp->sdp_id);
+#endif

 
                sdp->sdp_hashnext =
                    sdt_probetab[SDT_ADDR2NDX(sdpd->sdpd_offset)];
 
                sdp->sdp_hashnext =
                    sdt_probetab[SDT_ADDR2NDX(sdpd->sdpd_offset)];


@@ -214,7 +210,7 @@ sdt_destroy(void *arg, dtrace_id_t id, void *parg)
 
                kmem_free(sdp->sdp_name, sdp->sdp_namelen);
                sdp = sdp->sdp_next;
 
                kmem_free(sdp->sdp_name, sdp->sdp_namelen);
                sdp = sdp->sdp_next;

-               kmem_free(old, sizeof (sdt_probe_t));
+               kmem_free(old, sizeof(sdt_probe_t));

        }
 }
 
        }
 }
 


@@ -255,24 +251,32 @@ sdt_enable(void *arg, dtrace_id_t id, void *parg)
                goto err;
        }
 
                goto err;
        }
 

-       dtrace_casptr(&tempDTraceTrapHook, NULL, fbt_perfCallback);
+       dtrace_casptr(&tempDTraceTrapHook, NULL, ptrauth_nop_cast(void *, &fbt_perfCallback));

        if (tempDTraceTrapHook != (perfCallback)fbt_perfCallback) {
                if (sdt_verbose) {
                        cmn_err(CE_NOTE, "sdt_enable is failing for probe %s "
                            "in module %s: tempDTraceTrapHook already occupied.",
                            sdp->sdp_name, ctl->mod_modname);
                }
        if (tempDTraceTrapHook != (perfCallback)fbt_perfCallback) {
                if (sdt_verbose) {
                        cmn_err(CE_NOTE, "sdt_enable is failing for probe %s "
                            "in module %s: tempDTraceTrapHook already occupied.",
                            sdp->sdp_name, ctl->mod_modname);
                }

-               return (0);
+               return 0;

        }
 
        while (sdp != NULL) {
        }
 
        while (sdp != NULL) {

-               (void)ml_nofault_copy( (vm_offset_t)&sdp->sdp_patchval, (vm_offset_t)sdp->sdp_patchpoint, 
-                                      (vm_size_t)sizeof(sdp->sdp_patchval));
+               (void)ml_nofault_copy((vm_offset_t)&sdp->sdp_patchval, (vm_offset_t)sdp->sdp_patchpoint,
+                   (vm_size_t)sizeof(sdp->sdp_patchval));
+
+               /*
+                * Make the patched instruction visible via a data + instruction
+                * cache fush on platforms that need it
+                */
+               flush_dcache((vm_offset_t)sdp->sdp_patchpoint, (vm_size_t)sizeof(sdp->sdp_patchval), 0);
+               invalidate_icache((vm_offset_t)sdp->sdp_patchpoint, (vm_size_t)sizeof(sdp->sdp_patchval), 0);
+

                sdp = sdp->sdp_next;
        }
 
 err:
                sdp = sdp->sdp_next;
        }
 
 err:

-       return (0);
+       return 0;

 }
 
 /*ARGSUSED*/
 }
 
 /*ARGSUSED*/


@@ -285,12 +289,19 @@ sdt_disable(void *arg, dtrace_id_t id, void *parg)
 
        ctl->mod_nenabled--;
 
 
        ctl->mod_nenabled--;
 

-       if (!ctl->mod_loaded || ctl->mod_loadcnt != sdp->sdp_loadcnt)
+       if (!ctl->mod_loaded || ctl->mod_loadcnt != sdp->sdp_loadcnt) {

                goto err;
                goto err;

+       }

 
        while (sdp != NULL) {
 
        while (sdp != NULL) {

-               (void)ml_nofault_copy( (vm_offset_t)&sdp->sdp_savedval, (vm_offset_t)sdp->sdp_patchpoint, 
-                                      (vm_size_t)sizeof(sdp->sdp_savedval));
+               (void)ml_nofault_copy((vm_offset_t)&sdp->sdp_savedval, (vm_offset_t)sdp->sdp_patchpoint,
+                   (vm_size_t)sizeof(sdp->sdp_savedval));
+               /*
+                * Make the patched instruction visible via a data + instruction
+                * cache flush on platforms that need it
+                */
+               flush_dcache((vm_offset_t)sdp->sdp_patchpoint, (vm_size_t)sizeof(sdp->sdp_savedval), 0);
+               invalidate_icache((vm_offset_t)sdp->sdp_patchpoint, (vm_size_t)sizeof(sdp->sdp_savedval), 0);

                sdp = sdp->sdp_next;
        }
 
                sdp = sdp->sdp_next;
        }
 


@@ -299,41 +310,38 @@ err:
 }
 
 static dtrace_pops_t sdt_pops = {
 }
 
 static dtrace_pops_t sdt_pops = {

-       NULL,
-       sdt_provide_module,
-       sdt_enable,
-       sdt_disable,
-       NULL,
-       NULL,
-       sdt_getargdesc,
-       sdt_getarg,
-       NULL,
-       sdt_destroy
+       .dtps_provide =         NULL,
+       .dtps_provide_module =  sdt_provide_module,
+       .dtps_enable =          sdt_enable,
+       .dtps_disable =         sdt_disable,
+       .dtps_suspend =         NULL,
+       .dtps_resume =          NULL,
+       .dtps_getargdesc =      sdt_getargdesc,
+       .dtps_getargval =       sdt_getarg,
+       .dtps_usermode =        NULL,
+       .dtps_destroy =         sdt_destroy,

 };
 
 /*ARGSUSED*/
 static int
 };
 
 /*ARGSUSED*/
 static int

-sdt_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
+sdt_attach(dev_info_t *devi)

 {
 {

-#pragma unused(cmd)

        sdt_provider_t *prov;
 
        if (ddi_create_minor_node(devi, "sdt", S_IFCHR,
            0, DDI_PSEUDO, 0) == DDI_FAILURE) {
                cmn_err(CE_NOTE, "/dev/sdt couldn't create minor node");
                ddi_remove_minor_node(devi, NULL);
        sdt_provider_t *prov;
 
        if (ddi_create_minor_node(devi, "sdt", S_IFCHR,
            0, DDI_PSEUDO, 0) == DDI_FAILURE) {
                cmn_err(CE_NOTE, "/dev/sdt couldn't create minor node");
                ddi_remove_minor_node(devi, NULL);

-               return (DDI_FAILURE);
+               return DDI_FAILURE;

        }
 
        }
 

-       ddi_report_dev(devi);
-       sdt_devi = devi;
-
-       if (sdt_probetab_size == 0)
+       if (sdt_probetab_size == 0) {

                sdt_probetab_size = SDT_PROBETAB_SIZE;
                sdt_probetab_size = SDT_PROBETAB_SIZE;

+       }

 
        sdt_probetab_mask = sdt_probetab_size - 1;
        sdt_probetab =
 
        sdt_probetab_mask = sdt_probetab_size - 1;
        sdt_probetab =

-           kmem_zalloc(sdt_probetab_size * sizeof (sdt_probe_t *), KM_SLEEP);
+           kmem_zalloc(sdt_probetab_size * sizeof(sdt_probe_t *), KM_SLEEP);

        dtrace_invop_add(sdt_invop);
 
        for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
        dtrace_invop_add(sdt_invop);
 
        for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {


@@ -345,7 +353,7 @@ sdt_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
                }
        }
 
                }
        }
 

-       return (DDI_SUCCESS);
+       return DDI_SUCCESS;

 }
 
 /*
 }
 
 /*


@@ -363,31 +371,33 @@ sdt_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
                break;
 
        case DDI_SUSPEND:
                break;
 
        case DDI_SUSPEND:

-               return (DDI_SUCCESS);
+               return DDI_SUCCESS;

 
        default:
 
        default:

-               return (DDI_FAILURE);
+               return DDI_FAILURE;

        }
 
        for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
                if (prov->sdtp_id != DTRACE_PROVNONE) {
        }
 
        for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
                if (prov->sdtp_id != DTRACE_PROVNONE) {

-                       if (dtrace_unregister(prov->sdtp_id) != 0)
-                               return (DDI_FAILURE);
+                       if (dtrace_unregister(prov->sdtp_id) != 0) {
+                               return DDI_FAILURE;
+                       }

 
                        prov->sdtp_id = DTRACE_PROVNONE;
                }
        }
 
        dtrace_invop_remove(sdt_invop);
 
                        prov->sdtp_id = DTRACE_PROVNONE;
                }
        }
 
        dtrace_invop_remove(sdt_invop);

-       kmem_free(sdt_probetab, sdt_probetab_size * sizeof (sdt_probe_t *));
+       kmem_free(sdt_probetab, sdt_probetab_size * sizeof(sdt_probe_t *));

 
 

-       return (DDI_SUCCESS);
+       return DDI_SUCCESS;

 }
 #endif /* __APPLE__ */
 
 d_open_t _sdt_open;
 
 }
 #endif /* __APPLE__ */
 
 d_open_t _sdt_open;
 

-int _sdt_open(dev_t dev, int flags, int devtype, struct proc *p)
+int
+_sdt_open(dev_t dev, int flags, int devtype, struct proc *p)

 {
 #pragma unused(dev,flags,devtype,p)
        return 0;
 {
 #pragma unused(dev,flags,devtype,p)
        return 0;


@@ -395,176 +405,209 @@ int _sdt_open(dev_t dev, int flags, int devtype, struct proc *p)
 
 #define SDT_MAJOR  -24 /* let the kernel pick the device number */
 
 
 #define SDT_MAJOR  -24 /* let the kernel pick the device number */
 

-/*
- * A struct describing which functions will get invoked for certain
- * actions.
- */
-static struct cdevsw sdt_cdevsw =
+static const struct cdevsw sdt_cdevsw =

 {
 {

-       _sdt_open,              /* open */
-       eno_opcl,                       /* close */
-       eno_rdwrt,                      /* read */
-       eno_rdwrt,                      /* write */
-       eno_ioctl,                      /* ioctl */
-       (stop_fcn_t *)nulldev, /* stop */
-       (reset_fcn_t *)nulldev, /* reset */
-       NULL,                           /* tty's */
-       eno_select,                     /* select */
-       eno_mmap,                       /* mmap */
-       eno_strat,                      /* strategy */
-       eno_getc,                       /* getc */
-       eno_putc,                       /* putc */
-       0                                       /* type */
+       .d_open = _sdt_open,
+       .d_close = eno_opcl,
+       .d_read = eno_rdwrt,
+       .d_write = eno_rdwrt,
+       .d_ioctl = eno_ioctl,
+       .d_stop = (stop_fcn_t *)nulldev,
+       .d_reset = (reset_fcn_t *)nulldev,
+       .d_select = eno_select,
+       .d_mmap = eno_mmap,
+       .d_strategy = eno_strat,
+       .d_reserved_1 = eno_getc,
+       .d_reserved_2 = eno_putc,

 };
 
 };
 

-static int gSDTInited = 0;

 static struct modctl g_sdt_kernctl;
 static struct module g_sdt_mach_module;
 
 #include <mach-o/nlist.h>
 #include <libkern/kernel_mach_header.h>
 
 static struct modctl g_sdt_kernctl;
 static struct module g_sdt_mach_module;
 
 #include <mach-o/nlist.h>
 #include <libkern/kernel_mach_header.h>
 

-void sdt_init( void )
+/*
+ * Represents single record in __DATA,__sdt section.
+ */
+typedef struct dtrace_sdt_def {
+       uintptr_t      dsd_addr;    /* probe site location */
+       const char     *dsd_prov;   /* provider's name */
+       const char     *dsd_name;   /* probe's name */
+} __attribute__((__packed__))  dtrace_sdt_def_t;
+
+/*
+ * Creates a copy of name and unescapes '-' characters.
+ */
+static char *
+sdt_strdup_name(const char *name)

 {
 {

-       if (0 == gSDTInited)
-       {
-               int majdevno = cdevsw_add(SDT_MAJOR, &sdt_cdevsw);
-               
-               if (majdevno < 0) {
-                       printf("sdt_init: failed to allocate a major number!\n");
-                       gSDTInited = 0;
-                       return;
+       size_t len = strlen(name) + 1;
+       size_t i, j;
+       char *nname = kmem_alloc(len, KM_SLEEP);
+
+       for (i = 0, j = 0; name[j] != '\0'; i++) {
+               if (name[j] == '_' && name[j + 1] == '_') {
+                       nname[i] = '-';
+                       j += 2;
+               } else {
+                       nname[i] = name[j++];

                }
                }

+       }

 
 

-               if (MH_MAGIC_KERNEL != _mh_execute_header.magic) {
-                       g_sdt_kernctl.mod_address = (vm_address_t)NULL;
-                       g_sdt_kernctl.mod_size = 0;
-               } else {
-                       kernel_mach_header_t        *mh;
-                       struct load_command         *cmd;
-                       kernel_segment_command_t    *orig_ts = NULL, *orig_le = NULL;
-                       struct symtab_command       *orig_st = NULL;
-                       kernel_nlist_t              *sym = NULL;
-                       char                        *strings;
-                       unsigned int                i;
-                       
-                       g_sdt_mach_module.sdt_nprobes = 0;
-                       g_sdt_mach_module.sdt_probes = NULL;
-                       
-                       g_sdt_kernctl.mod_address = (vm_address_t)&g_sdt_mach_module;
-                       g_sdt_kernctl.mod_size = 0;
-                       strncpy((char *)&(g_sdt_kernctl.mod_modname), "mach_kernel", KMOD_MAX_NAME);
-                       
-                       g_sdt_kernctl.mod_next = NULL;
-                       g_sdt_kernctl.mod_stale = NULL;
-                       g_sdt_kernctl.mod_id = 0;
-                       g_sdt_kernctl.mod_loadcnt = 1;
-                       g_sdt_kernctl.mod_loaded = 1;
-                       g_sdt_kernctl.mod_flags = 0;
-                       g_sdt_kernctl.mod_nenabled = 0;
-                       
-                       mh = &_mh_execute_header;
-                       cmd = (struct load_command*) &mh[1];
-                       for (i = 0; i < mh->ncmds; i++) {
-                               if (cmd->cmd == LC_SEGMENT_KERNEL) {
-                                       kernel_segment_command_t *orig_sg = (kernel_segment_command_t *) cmd;
-                                       
-                                       if (LIT_STRNEQL(orig_sg->segname, SEG_TEXT))
-                                               orig_ts = orig_sg;
-                                       else if (LIT_STRNEQL(orig_sg->segname, SEG_LINKEDIT))
-                                               orig_le = orig_sg;
-                                       else if (LIT_STRNEQL(orig_sg->segname, ""))
-                                               orig_ts = orig_sg; /* kexts have a single unnamed segment */
+       nname[i] = '\0';
+       return nname;
+}
+
+void
+sdt_early_init( void )
+{
+       if (dtrace_sdt_probes_restricted()) {
+               return;
+       }
+       if (MH_MAGIC_KERNEL != _mh_execute_header.magic) {
+               g_sdt_kernctl.mod_address = (vm_address_t)NULL;
+               g_sdt_kernctl.mod_size = 0;
+       } else {
+               kernel_mach_header_t        *mh;
+               struct load_command         *cmd;
+               kernel_segment_command_t    *orig_ts = NULL, *orig_le = NULL;
+               kernel_section_t            *orig_dt = NULL;
+               struct symtab_command       *orig_st = NULL;
+               kernel_nlist_t              *sym = NULL;
+               char                        *strings;
+               unsigned int                i;
+               unsigned int                len;
+
+               g_sdt_mach_module.sdt_nprobes = 0;
+               g_sdt_mach_module.sdt_probes = NULL;
+
+               g_sdt_kernctl.mod_address = (vm_address_t)&g_sdt_mach_module;
+               g_sdt_kernctl.mod_size = 0;
+               strncpy((char *)&(g_sdt_kernctl.mod_modname), "mach_kernel", KMOD_MAX_NAME);
+
+               g_sdt_kernctl.mod_next = NULL;
+               g_sdt_kernctl.mod_stale = NULL;
+               g_sdt_kernctl.mod_id = 0;
+               g_sdt_kernctl.mod_loadcnt = 1;
+               g_sdt_kernctl.mod_loaded = 1;
+               g_sdt_kernctl.mod_flags = 0;
+               g_sdt_kernctl.mod_nenabled = 0;
+
+               mh = &_mh_execute_header;
+               cmd = (struct load_command*) &mh[1];
+               for (i = 0; i < mh->ncmds; i++) {
+                       if (cmd->cmd == LC_SEGMENT_KERNEL) {
+                               kernel_segment_command_t *orig_sg = (kernel_segment_command_t *) cmd;
+
+                               if (LIT_STRNEQL(orig_sg->segname, SEG_TEXT)) {
+                                       orig_ts = orig_sg;
+                               } else if (LIT_STRNEQL(orig_sg->segname, SEG_LINKEDIT)) {
+                                       orig_le = orig_sg;
+                               } else if (LIT_STRNEQL(orig_sg->segname, "")) {
+                                       orig_ts = orig_sg; /* kexts have a single unnamed segment */

                                }
                                }

-                               else if (cmd->cmd == LC_SYMTAB)
-                                       orig_st = (struct symtab_command *) cmd;
-                               
-                               cmd = (struct load_command *) ((uintptr_t) cmd + cmd->cmdsize);
+                       } else if (cmd->cmd == LC_SYMTAB) {
+                               orig_st = (struct symtab_command *) cmd;

                        }
                        }

-                       
-                       if ((orig_ts == NULL) || (orig_st == NULL) || (orig_le == NULL))
-                               return;
-                       
-                       sym = (kernel_nlist_t *)(orig_le->vmaddr + orig_st->symoff - orig_le->fileoff);
-                       strings = (char *)(orig_le->vmaddr + orig_st->stroff - orig_le->fileoff);
-                       
+
+                       cmd = (struct load_command *) ((uintptr_t) cmd + cmd->cmdsize);
+               }
+
+               /* Locate DTrace SDT section in the object. */
+               if ((orig_dt = getsectbyname("__DATA", "__sdt")) == NULL) {
+                       printf("DTrace section not found.\n");
+                       return;
+               }
+
+               if ((orig_ts == NULL) || (orig_st == NULL) || (orig_le == NULL)) {
+                       return;
+               }
+
+               sym = (kernel_nlist_t *)(orig_le->vmaddr + orig_st->symoff - orig_le->fileoff);
+               strings = (char *)(orig_le->vmaddr + orig_st->stroff - orig_le->fileoff);
+
+               /*
+                * Iterate over SDT section and establish all SDT probes.
+                */
+               dtrace_sdt_def_t *sdtdef = (dtrace_sdt_def_t *)(orig_dt->addr);
+               for (size_t k = 0; k < orig_dt->size / sizeof(dtrace_sdt_def_t); k++, sdtdef++) {
+                       const char *funcname;
+                       unsigned long best;
+
+                       sdt_probedesc_t *sdpd = kmem_alloc(sizeof(sdt_probedesc_t), KM_SLEEP);
+
+                       /* Unescape probe name and keep a note of the size of original memory allocation. */
+                       sdpd->sdpd_name = sdt_strdup_name(sdtdef->dsd_name);
+                       sdpd->sdpd_namelen = strlen(sdtdef->dsd_name) + 1;
+
+                       /* Used only for provider structure lookup so there is no need to make dynamic copy. */
+                       sdpd->sdpd_prov = sdtdef->dsd_prov;
+
+                       /*
+                        * Find the symbol immediately preceding the sdt probe site just discovered,
+                        * that symbol names the function containing the sdt probe.
+                        */
+                       funcname = "<unknown>";

                        for (i = 0; i < orig_st->nsyms; i++) {
                        for (i = 0; i < orig_st->nsyms; i++) {

-                               uint8_t n_type = sym[i].n_type & (N_TYPE | N_EXT);
-                               char *name = strings + sym[i].n_un.n_strx;
-                               const char *prev_name;
-                               unsigned long best;
-                               unsigned int j;
-                               
-                               /* Check that the symbol is a global and that it has a name. */
-                               if (((N_SECT | N_EXT) != n_type && (N_ABS | N_EXT) != n_type))
+                               uint8_t jn_type = sym[i].n_type & N_TYPE;
+                               char *jname = strings + sym[i].n_un.n_strx;
+
+                               if ((N_SECT != jn_type && N_ABS != jn_type)) {

                                        continue;
                                        continue;

-                               
-                               if (0 == sym[i].n_un.n_strx) /* iff a null, "", name. */
+                               }
+
+                               if (0 == sym[i].n_un.n_strx) { /* iff a null, "", name. */

                                        continue;
                                        continue;

-                               
-                               /* Lop off omnipresent leading underscore. */
-                               if (*name == '_')
-                                       name += 1;
-                               
-                               if (strncmp(name, DTRACE_PROBE_PREFIX, sizeof(DTRACE_PROBE_PREFIX) - 1) == 0) {
-                                       sdt_probedesc_t *sdpd = kmem_alloc(sizeof(sdt_probedesc_t), KM_SLEEP);
-                                       int len = strlen(name) + 1;
-                                       
-                                       sdpd->sdpd_name = kmem_alloc(len, KM_SLEEP);
-                                       strncpy(sdpd->sdpd_name, name, len); /* NUL termination is ensured. */
-                                       
-                                       prev_name = "<unknown>";
-                                       best = 0;
-                                       
-                                       /*
-                                        * Find the symbol immediately preceding the sdt probe site just discovered,
-                                        * that symbol names the function containing the sdt probe.
-                                        */
-                                       for (j = 0; j < orig_st->nsyms; j++) {
-                                               uint8_t jn_type = sym[j].n_type & (N_TYPE | N_EXT);
-                                               char *jname = strings + sym[j].n_un.n_strx;
-                                               
-                                               if (((N_SECT | N_EXT) != jn_type && (N_ABS | N_EXT) != jn_type))
-                                                       continue;
-                                               
-                                               if (0 == sym[j].n_un.n_strx) /* iff a null, "", name. */
-                                                       continue;
-                                               
-                                               if (*jname == '_')
-                                                       jname += 1;
-                                               
-                                               if (*(unsigned long *)sym[i].n_value <= (unsigned long)sym[j].n_value)
-                                                       continue;
-                                               
-                                               if ((unsigned long)sym[j].n_value > best) {
-                                                       best = (unsigned long)sym[j].n_value;
-                                                       prev_name = jname;
-                                               }
-                                       }
-                                       
-                                       sdpd->sdpd_func = kmem_alloc((len = strlen(prev_name) + 1), KM_SLEEP);
-                                       strncpy(sdpd->sdpd_func, prev_name, len); /* NUL termination is ensured. */
-                                       
-                                       sdpd->sdpd_offset = *(unsigned long *)sym[i].n_value;
+                               }

 
 

-#if 0
-                                       printf("sdt_init: sdpd_offset=0x%lx, n_value=0x%lx, name=%s\n",
-                                           sdpd->sdpd_offset,  *(unsigned long *)sym[i].n_value, name);
-#endif
+                               if (*jname == '_') {
+                                       jname += 1;
+                               }

 
 

-                                       sdpd->sdpd_next = g_sdt_mach_module.sdt_probes;
-                                       g_sdt_mach_module.sdt_probes = sdpd;
-                               } else {
-                                       prev_name = name;
+                               if (sdtdef->dsd_addr <= (unsigned long)sym[i].n_value) {
+                                       continue;
+                               }
+
+                               if ((unsigned long)sym[i].n_value > best) {
+                                       best = (unsigned long)sym[i].n_value;
+                                       funcname = jname;

                                }
                        }
                                }
                        }

+
+                       len = strlen(funcname) + 1;
+                       sdpd->sdpd_func = kmem_alloc(len, KM_SLEEP);
+                       strncpy(sdpd->sdpd_func, funcname, len);
+
+                       sdpd->sdpd_offset = sdtdef->dsd_addr;
+#if defined(__arm__)
+                       /* PR8353094 - mask off thumb-bit */
+                       sdpd->sdpd_offset &= ~0x1U;
+#elif defined(__arm64__)
+                       sdpd->sdpd_offset &= ~0x1LU;
+#endif  /* __arm__ */
+
+                       sdpd->sdpd_next = g_sdt_mach_module.sdt_probes;
+                       g_sdt_mach_module.sdt_probes = sdpd;

                }
                }

-               
-               sdt_attach( (dev_info_t *)(uintptr_t)majdevno, DDI_ATTACH );
-               
-               gSDTInited = 1;
-       } else
-               panic("sdt_init: called twice!\n");
+       }
+}
+
+void
+sdt_init( void )
+{
+       int majdevno = cdevsw_add(SDT_MAJOR, &sdt_cdevsw);
+
+       if (majdevno < 0) {
+               printf("sdt_init: failed to allocate a major number!\n");
+               return;
+       }
+
+       if (dtrace_sdt_probes_restricted()) {
+               return;
+       }
+
+       sdt_attach((dev_info_t*)(uintptr_t)majdevno);

 }
 
 #undef SDT_MAJOR
 }
 
 #undef SDT_MAJOR


@@ -576,18 +619,19 @@ sdt_provide_module(void *arg, struct modctl *ctl)
 #pragma unused(arg)
        ASSERT(ctl != NULL);
        ASSERT(dtrace_kernel_symbol_mode != DTRACE_KERNEL_SYMBOLS_NEVER);
 #pragma unused(arg)
        ASSERT(ctl != NULL);
        ASSERT(dtrace_kernel_symbol_mode != DTRACE_KERNEL_SYMBOLS_NEVER);

-       lck_mtx_assert(&mod_lock, LCK_MTX_ASSERT_OWNED);
-       
-       if (MOD_SDT_DONE(ctl))
+       LCK_MTX_ASSERT(&mod_lock, LCK_MTX_ASSERT_OWNED);
+
+       if (MOD_SDT_DONE(ctl)) {

                return;
                return;

-               
+       }
+

        if (MOD_IS_MACH_KERNEL(ctl)) {
                __sdt_provide_module(arg, &g_sdt_kernctl);
        if (MOD_IS_MACH_KERNEL(ctl)) {
                __sdt_provide_module(arg, &g_sdt_kernctl);

-               
+

                sdt_probedesc_t *sdpd = g_sdt_mach_module.sdt_probes;
                while (sdpd) {
                        sdt_probedesc_t *this_sdpd = sdpd;
                sdt_probedesc_t *sdpd = g_sdt_mach_module.sdt_probes;
                while (sdpd) {
                        sdt_probedesc_t *this_sdpd = sdpd;

-                       kmem_free((void *)sdpd->sdpd_name, strlen(sdpd->sdpd_name) + 1);
+                       kmem_free((void *)sdpd->sdpd_name, sdpd->sdpd_namelen);

                        kmem_free((void *)sdpd->sdpd_func, strlen(sdpd->sdpd_func) + 1);
                        sdpd = sdpd->sdpd_next;
                        kmem_free((void *)this_sdpd, sizeof(sdt_probedesc_t));
                        kmem_free((void *)sdpd->sdpd_func, strlen(sdpd->sdpd_func) + 1);
                        sdpd = sdpd->sdpd_next;
                        kmem_free((void *)this_sdpd, sizeof(sdt_probedesc_t));


@@ -598,7 +642,7 @@ sdt_provide_module(void *arg, struct modctl *ctl)
                 * APPLE NOTE:  sdt probes for kexts not yet implemented
                 */
        }
                 * APPLE NOTE:  sdt probes for kexts not yet implemented
                 */
        }

-       
+

        /* Need to mark this module as completed */
        ctl->mod_flags |= MODCTL_SDT_PROBES_PROVIDED;
 }
        /* Need to mark this module as completed */
        ctl->mod_flags |= MODCTL_SDT_PROBES_PROVIDED;
 }