bsd/dev/dtrace/sdt.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25
  26 /* #pragma ident        "@(#)sdt.c      1.9     08/07/01 SMI" */
  27
  28 #ifdef KERNEL
  29 #ifndef _KERNEL
  30 #define _KERNEL /* Solaris vs. Darwin */
  31 #endif
  32 #endif
  33
  34 #include <sys/param.h>
  35 #include <sys/systm.h>
  36 #include <sys/errno.h>
  37 #include <sys/stat.h>
  38 #include <sys/ioctl.h>
  39 #include <sys/conf.h>
  40 #include <sys/fcntl.h>
  41 #include <miscfs/devfs/devfs.h>
  42
  43 #if CONFIG_EMBEDDED
  44 #include <arm/caches_internal.h>
  45 #endif
  46
  47 #include <sys/dtrace.h>
  48 #include <sys/dtrace_impl.h>
  49
  50 #include <sys/dtrace_glue.h>
  51
  52 #include <sys/sdt_impl.h>
  53 extern int dtrace_kernel_symbol_mode;
  54
  55 /* #include <machine/trap.h */
  56 struct savearea_t; /* Used anonymously */
  57
  58 #if defined(__arm__)
  59 typedef kern_return_t (*perfCallback)(int, struct savearea_t *, __unused int, __unused int);
  60 extern perfCallback tempDTraceTrapHook;
  61 extern kern_return_t fbt_perfCallback(int, struct savearea_t *, __unused int, __unused int);
  62 #define SDT_PATCHVAL    0xdefc
  63 #define SDT_AFRAMES             7
  64 #elif defined(__arm64__)
  65 typedef kern_return_t (*perfCallback)(int, struct savearea_t *, __unused int, __unused int);
  66 extern perfCallback tempDTraceTrapHook;
  67 extern kern_return_t fbt_perfCallback(int, struct savearea_t *, __unused int, __unused int);
  68 #define SDT_PATCHVAL    0xe7eeee7e
  69 #define SDT_AFRAMES             7
  70 #elif defined(__x86_64__)
  71 typedef kern_return_t (*perfCallback)(int, struct savearea_t *, uintptr_t *, int);
  72 extern perfCallback tempDTraceTrapHook;
  73 extern kern_return_t fbt_perfCallback(int, struct savearea_t *, uintptr_t *, int);
  74 #define SDT_PATCHVAL    0xf0
  75 #define SDT_AFRAMES             6
  76 #else
  77 #error Unknown architecture
  78 #endif
  79
  80 #define SDT_PROBETAB_SIZE       0x1000          /* 4k entries -- 16K total */
  81
  82 #define DTRACE_PROBE_PREFIX "_dtrace_probe$"
  83
  84 static dev_info_t               *sdt_devi;
  85 static int                      sdt_verbose = 0;
  86 sdt_probe_t             **sdt_probetab;
  87 int                     sdt_probetab_size;
  88 int                     sdt_probetab_mask;
  89
  90 /*ARGSUSED*/
  91 static void
  92 __sdt_provide_module(void *arg, struct modctl *ctl)
  93 {
  94 #pragma unused(arg)
  95         struct module *mp = (struct module *)ctl->mod_address;
  96         char *modname = ctl->mod_modname;
  97         sdt_probedesc_t *sdpd;
  98         sdt_probe_t *sdp, *old;
  99         sdt_provider_t *prov;
 100         int len;
 101
 102         /*
 103          * One for all, and all for one:  if we haven't yet registered all of
 104          * our providers, we'll refuse to provide anything.
 105          */
 106         for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
 107                 if (prov->sdtp_id == DTRACE_PROVNONE)
 108                         return;
 109         }
 110
 111         if (!mp || mp->sdt_nprobes != 0 || (sdpd = mp->sdt_probes) == NULL)
 112                 return;
 113
 114         for (sdpd = mp->sdt_probes; sdpd != NULL; sdpd = sdpd->sdpd_next) {
 115             const char *name = sdpd->sdpd_name, *func;
 116             char *nname;
 117                 int i, j;
 118                 dtrace_id_t id;
 119
 120                 for (prov = sdt_providers; prov->sdtp_prefix != NULL; prov++) {
 121                         const char *prefpart, *prefix = prov->sdtp_prefix;
 122
 123                         if ((prefpart = strstr(name, prefix))) {
 124                                 name = prefpart + strlen(prefix);
 125                                 break;
 126                         }
 127                 }
 128
 129                 nname = kmem_alloc(len = strlen(name) + 1, KM_SLEEP);
 130
 131                 for (i = 0, j = 0; name[j] != '\0'; i++) {
 132                         if (name[j] == '_' && name[j + 1] == '_') {
 133                                 nname[i] = '-';
 134                                 j += 2;
 135                         } else {
 136                                 nname[i] = name[j++];
 137                         }
 138                 }
 139
 140                 nname[i] = '\0';
 141
 142                 sdp = kmem_zalloc(sizeof (sdt_probe_t), KM_SLEEP);
 143                 sdp->sdp_loadcnt = ctl->mod_loadcnt;
 144                 sdp->sdp_ctl = ctl;
 145                 sdp->sdp_name = nname;
 146                 sdp->sdp_namelen = len;
 147                 sdp->sdp_provider = prov;
 148
 149                 func = sdpd->sdpd_func;
 150
 151                 if (func == NULL)
 152                         func = "<unknown>";
 153
 154                 /*
 155                  * We have our provider.  Now create the probe.
 156                  */
 157                 if ((id = dtrace_probe_lookup(prov->sdtp_id, modname,
 158                     func, nname)) != DTRACE_IDNONE) {
 159                         old = dtrace_probe_arg(prov->sdtp_id, id);
 160                         ASSERT(old != NULL);
 161
 162                         sdp->sdp_next = old->sdp_next;
 163                         sdp->sdp_id = id;
 164                         old->sdp_next = sdp;
 165                 } else {
 166                         sdp->sdp_id = dtrace_probe_create(prov->sdtp_id,
 167                             modname, func, nname, SDT_AFRAMES, sdp);
 168
 169                         mp->sdt_nprobes++;
 170                 }
 171
 172 #if 0
 173                 printf ("__sdt_provide_module:  sdpd=0x%p  sdp=0x%p  name=%s, id=%d\n", sdpd, sdp, nname, sdp->sdp_id);
 174 #endif
 175
 176                 sdp->sdp_hashnext =
 177                     sdt_probetab[SDT_ADDR2NDX(sdpd->sdpd_offset)];
 178                 sdt_probetab[SDT_ADDR2NDX(sdpd->sdpd_offset)] = sdp;
 179
 180                 sdp->sdp_patchval = SDT_PATCHVAL;
 181                 sdp->sdp_patchpoint = (sdt_instr_t *)sdpd->sdpd_offset;
 182                 sdp->sdp_savedval = *sdp->sdp_patchpoint;
 183         }
 184 }
 185
 186 /*ARGSUSED*/
 187 static void
 188 sdt_destroy(void *arg, dtrace_id_t id, void *parg)
 189 {
 190 #pragma unused(arg,id)
 191         sdt_probe_t *sdp = parg, *old, *last, *hash;
 192         int ndx;
 193
 194 #if !defined(__APPLE__)
 195         /*
 196          * APPLE NOTE:  sdt probes for kexts not yet implemented
 197          */
 198         struct modctl *ctl = sdp->sdp_ctl;
 199
 200         if (ctl != NULL && ctl->mod_loadcnt == sdp->sdp_loadcnt) {
 201                 if ((ctl->mod_loadcnt == sdp->sdp_loadcnt &&
 202                     ctl->mod_loaded)) {
 203                         ((struct module *)(ctl->mod_mp))->sdt_nprobes--;
 204                 }
 205         }
 206 #endif /* __APPLE__ */
 207
 208         while (sdp != NULL) {
 209                 old = sdp;
 210
 211                 /*
 212                  * Now we need to remove this probe from the sdt_probetab.
 213                  */
 214                 ndx = SDT_ADDR2NDX(sdp->sdp_patchpoint);
 215                 last = NULL;
 216                 hash = sdt_probetab[ndx];
 217
 218                 while (hash != sdp) {
 219                         ASSERT(hash != NULL);
 220                         last = hash;
 221                         hash = hash->sdp_hashnext;
 222                 }
 223
 224                 if (last != NULL) {
 225                         last->sdp_hashnext = sdp->sdp_hashnext;
 226                 } else {
 227                         sdt_probetab[ndx] = sdp->sdp_hashnext;
 228                 }
 229
 230                 kmem_free(sdp->sdp_name, sdp->sdp_namelen);
 231                 sdp = sdp->sdp_next;
 232                 kmem_free(old, sizeof (sdt_probe_t));
 233         }
 234 }
 235
 236 /*ARGSUSED*/
 237 static int
 238 sdt_enable(void *arg, dtrace_id_t id, void *parg)
 239 {
 240 #pragma unused(arg,id)
 241         sdt_probe_t *sdp = parg;
 242         struct modctl *ctl = sdp->sdp_ctl;
 243
 244         ctl->mod_nenabled++;
 245
 246         /*
 247          * If this module has disappeared since we discovered its probes,
 248          * refuse to enable it.
 249          */
 250         if (!ctl->mod_loaded) {
 251                 if (sdt_verbose) {
 252                         cmn_err(CE_NOTE, "sdt is failing for probe %s "
 253                             "(module %s unloaded)",
 254                             sdp->sdp_name, ctl->mod_modname);
 255                 }
 256                 goto err;
 257         }
 258
 259         /*
 260          * Now check that our modctl has the expected load count.  If it
 261          * doesn't, this module must have been unloaded and reloaded -- and
 262          * we're not going to touch it.
 263          */
 264         if (ctl->mod_loadcnt != sdp->sdp_loadcnt) {
 265                 if (sdt_verbose) {
 266                         cmn_err(CE_NOTE, "sdt is failing for probe %s "
 267                             "(module %s reloaded)",
 268                             sdp->sdp_name, ctl->mod_modname);
 269                 }
 270                 goto err;
 271         }
 272
 273         dtrace_casptr(&tempDTraceTrapHook, NULL, fbt_perfCallback);
 274         if (tempDTraceTrapHook != (perfCallback)fbt_perfCallback) {
 275                 if (sdt_verbose) {
 276                         cmn_err(CE_NOTE, "sdt_enable is failing for probe %s "
 277                             "in module %s: tempDTraceTrapHook already occupied.",
 278                             sdp->sdp_name, ctl->mod_modname);
 279                 }
 280                 return (0);
 281         }
 282
 283         while (sdp != NULL) {
 284                 (void)ml_nofault_copy( (vm_offset_t)&sdp->sdp_patchval, (vm_offset_t)sdp->sdp_patchpoint,
 285                                        (vm_size_t)sizeof(sdp->sdp_patchval));
 286
 287                 /*
 288                  * Make the patched instruction visible via a data + instruction
 289                  * cache fush on platforms that need it
 290                  */
 291                 flush_dcache((vm_offset_t)sdp->sdp_patchpoint,(vm_size_t)sizeof(sdp->sdp_patchval), 0);
 292                 invalidate_icache((vm_offset_t)sdp->sdp_patchpoint,(vm_size_t)sizeof(sdp->sdp_patchval), 0);
 293
 294                 sdp = sdp->sdp_next;
 295         }
 296
 297 err:
 298         return (0);
 299 }
 300
 301 /*ARGSUSED*/
 302 static void
 303 sdt_disable(void *arg, dtrace_id_t id, void *parg)
 304 {
 305 #pragma unused(arg,id)
 306         sdt_probe_t *sdp = parg;
 307         struct modctl *ctl = sdp->sdp_ctl;
 308
 309         ctl->mod_nenabled--;
 310
 311         if (!ctl->mod_loaded || ctl->mod_loadcnt != sdp->sdp_loadcnt)
 312                 goto err;
 313
 314         while (sdp != NULL) {
 315                 (void)ml_nofault_copy( (vm_offset_t)&sdp->sdp_savedval, (vm_offset_t)sdp->sdp_patchpoint,
 316                                        (vm_size_t)sizeof(sdp->sdp_savedval));
 317                 /*
 318                  * Make the patched instruction visible via a data + instruction
 319                  * cache flush on platforms that need it
 320                  */
 321                 flush_dcache((vm_offset_t)sdp->sdp_patchpoint,(vm_size_t)sizeof(sdp->sdp_savedval), 0);
 322                 invalidate_icache((vm_offset_t)sdp->sdp_patchpoint,(vm_size_t)sizeof(sdp->sdp_savedval), 0);
 323                 sdp = sdp->sdp_next;
 324         }
 325
 326 err:
 327         ;
 328 }
 329
 330 static dtrace_pops_t sdt_pops = {
 331         NULL,
 332         sdt_provide_module,
 333         sdt_enable,
 334         sdt_disable,
 335         NULL,
 336         NULL,
 337         sdt_getargdesc,
 338         sdt_getarg,
 339         NULL,
 340         sdt_destroy
 341 };
 342
 343 /*ARGSUSED*/
 344 static int
 345 sdt_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
 346 {
 347 #pragma unused(cmd)
 348         sdt_provider_t *prov;
 349
 350         if (ddi_create_minor_node(devi, "sdt", S_IFCHR,
 351             0, DDI_PSEUDO, 0) == DDI_FAILURE) {
 352                 cmn_err(CE_NOTE, "/dev/sdt couldn't create minor node");
 353                 ddi_remove_minor_node(devi, NULL);
 354                 return (DDI_FAILURE);
 355         }
 356
 357         ddi_report_dev(devi);
 358         sdt_devi = devi;
 359
 360         if (sdt_probetab_size == 0)
 361                 sdt_probetab_size = SDT_PROBETAB_SIZE;
 362
 363         sdt_probetab_mask = sdt_probetab_size - 1;
 364         sdt_probetab =
 365             kmem_zalloc(sdt_probetab_size * sizeof (sdt_probe_t *), KM_SLEEP);
 366         dtrace_invop_add(sdt_invop);
 367
 368         for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
 369                 if (dtrace_register(prov->sdtp_name, prov->sdtp_attr,
 370                     DTRACE_PRIV_KERNEL, NULL,
 371                     &sdt_pops, prov, &prov->sdtp_id) != 0) {
 372                         cmn_err(CE_WARN, "failed to register sdt provider %s",
 373                             prov->sdtp_name);
 374                 }
 375         }
 376
 377         return (DDI_SUCCESS);
 378 }
 379
 380 /*
 381  * APPLE NOTE:  sdt_detach not implemented
 382  */
 383 #if !defined(__APPLE__)
 384 /*ARGSUSED*/
 385 static int
 386 sdt_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
 387 {
 388         sdt_provider_t *prov;
 389
 390         switch (cmd) {
 391         case DDI_DETACH:
 392                 break;
 393
 394         case DDI_SUSPEND:
 395                 return (DDI_SUCCESS);
 396
 397         default:
 398                 return (DDI_FAILURE);
 399         }
 400
 401         for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
 402                 if (prov->sdtp_id != DTRACE_PROVNONE) {
 403                         if (dtrace_unregister(prov->sdtp_id) != 0)
 404                                 return (DDI_FAILURE);
 405
 406                         prov->sdtp_id = DTRACE_PROVNONE;
 407                 }
 408         }
 409
 410         dtrace_invop_remove(sdt_invop);
 411         kmem_free(sdt_probetab, sdt_probetab_size * sizeof (sdt_probe_t *));
 412
 413         return (DDI_SUCCESS);
 414 }
 415 #endif /* __APPLE__ */
 416
 417 d_open_t _sdt_open;
 418
 419 int _sdt_open(dev_t dev, int flags, int devtype, struct proc *p)
 420 {
 421 #pragma unused(dev,flags,devtype,p)
 422         return 0;
 423 }
 424
 425 #define SDT_MAJOR  -24 /* let the kernel pick the device number */
 426
 427 /*
 428  * A struct describing which functions will get invoked for certain
 429  * actions.
 430  */
 431 static struct cdevsw sdt_cdevsw =
 432 {
 433         _sdt_open,              /* open */
 434         eno_opcl,                       /* close */
 435         eno_rdwrt,                      /* read */
 436         eno_rdwrt,                      /* write */
 437         eno_ioctl,                      /* ioctl */
 438         (stop_fcn_t *)nulldev, /* stop */
 439         (reset_fcn_t *)nulldev, /* reset */
 440         NULL,                           /* tty's */
 441         eno_select,                     /* select */
 442         eno_mmap,                       /* mmap */
 443         eno_strat,                      /* strategy */
 444         eno_getc,                       /* getc */
 445         eno_putc,                       /* putc */
 446         0                                       /* type */
 447 };
 448
 449 static int gSDTInited = 0;
 450 static struct modctl g_sdt_kernctl;
 451 static struct module g_sdt_mach_module;
 452
 453 #include <mach-o/nlist.h>
 454 #include <libkern/kernel_mach_header.h>
 455
 456 void sdt_init( void )
 457 {
 458         if (0 == gSDTInited)
 459         {
 460                 int majdevno = cdevsw_add(SDT_MAJOR, &sdt_cdevsw);
 461
 462                 if (majdevno < 0) {
 463                         printf("sdt_init: failed to allocate a major number!\n");
 464                         gSDTInited = 0;
 465                         return;
 466                 }
 467
 468                 if (dtrace_sdt_probes_restricted()) {
 469                         return;
 470                 }
 471
 472                 if (MH_MAGIC_KERNEL != _mh_execute_header.magic) {
 473                         g_sdt_kernctl.mod_address = (vm_address_t)NULL;
 474                         g_sdt_kernctl.mod_size = 0;
 475                 } else {
 476                         kernel_mach_header_t        *mh;
 477                         struct load_command         *cmd;
 478                         kernel_segment_command_t    *orig_ts = NULL, *orig_le = NULL;
 479                         struct symtab_command       *orig_st = NULL;
 480                         kernel_nlist_t              *sym = NULL;
 481                         char                        *strings;
 482                         unsigned int                i;
 483
 484                         g_sdt_mach_module.sdt_nprobes = 0;
 485                         g_sdt_mach_module.sdt_probes = NULL;
 486
 487                         g_sdt_kernctl.mod_address = (vm_address_t)&g_sdt_mach_module;
 488                         g_sdt_kernctl.mod_size = 0;
 489                         strncpy((char *)&(g_sdt_kernctl.mod_modname), "mach_kernel", KMOD_MAX_NAME);
 490
 491                         g_sdt_kernctl.mod_next = NULL;
 492                         g_sdt_kernctl.mod_stale = NULL;
 493                         g_sdt_kernctl.mod_id = 0;
 494                         g_sdt_kernctl.mod_loadcnt = 1;
 495                         g_sdt_kernctl.mod_loaded = 1;
 496                         g_sdt_kernctl.mod_flags = 0;
 497                         g_sdt_kernctl.mod_nenabled = 0;
 498
 499                         mh = &_mh_execute_header;
 500                         cmd = (struct load_command*) &mh[1];
 501                         for (i = 0; i < mh->ncmds; i++) {
 502                                 if (cmd->cmd == LC_SEGMENT_KERNEL) {
 503                                         kernel_segment_command_t *orig_sg = (kernel_segment_command_t *) cmd;
 504
 505                                         if (LIT_STRNEQL(orig_sg->segname, SEG_TEXT))
 506                                                 orig_ts = orig_sg;
 507                                         else if (LIT_STRNEQL(orig_sg->segname, SEG_LINKEDIT))
 508                                                 orig_le = orig_sg;
 509                                         else if (LIT_STRNEQL(orig_sg->segname, ""))
 510                                                 orig_ts = orig_sg; /* kexts have a single unnamed segment */
 511                                 }
 512                                 else if (cmd->cmd == LC_SYMTAB)
 513                                         orig_st = (struct symtab_command *) cmd;
 514
 515                                 cmd = (struct load_command *) ((uintptr_t) cmd + cmd->cmdsize);
 516                         }
 517
 518                         if ((orig_ts == NULL) || (orig_st == NULL) || (orig_le == NULL))
 519                                 return;
 520
 521                         sym = (kernel_nlist_t *)(orig_le->vmaddr + orig_st->symoff - orig_le->fileoff);
 522                         strings = (char *)(orig_le->vmaddr + orig_st->stroff - orig_le->fileoff);
 523
 524                         for (i = 0; i < orig_st->nsyms; i++) {
 525                                 uint8_t n_type = sym[i].n_type & (N_TYPE | N_EXT);
 526                                 char *name = strings + sym[i].n_un.n_strx;
 527                                 const char *prev_name;
 528                                 unsigned long best;
 529                                 unsigned int j;
 530
 531                                 /* Check that the symbol is a global and that it has a name. */
 532                                 if (((N_SECT | N_EXT) != n_type && (N_ABS | N_EXT) != n_type))
 533                                         continue;
 534
 535                                 if (0 == sym[i].n_un.n_strx) /* iff a null, "", name. */
 536                                         continue;
 537
 538                                 /* Lop off omnipresent leading underscore. */
 539                                 if (*name == '_')
 540                                         name += 1;
 541
 542                                 if (strncmp(name, DTRACE_PROBE_PREFIX, sizeof(DTRACE_PROBE_PREFIX) - 1) == 0) {
 543                                         sdt_probedesc_t *sdpd = kmem_alloc(sizeof(sdt_probedesc_t), KM_SLEEP);
 544                                         int len = strlen(name) + 1;
 545
 546                                         sdpd->sdpd_name = kmem_alloc(len, KM_SLEEP);
 547                                         strncpy(sdpd->sdpd_name, name, len); /* NUL termination is ensured. */
 548
 549                                         prev_name = "<unknown>";
 550                                         best = 0;
 551
 552                                         /*
 553                                          * Find the symbol immediately preceding the sdt probe site just discovered,
 554                                          * that symbol names the function containing the sdt probe.
 555                                          */
 556                                         for (j = 0; j < orig_st->nsyms; j++) {
 557                                                 uint8_t jn_type = sym[j].n_type & N_TYPE;
 558                                                 char *jname = strings + sym[j].n_un.n_strx;
 559
 560                                                 if ((N_SECT != jn_type && N_ABS != jn_type))
 561                                                         continue;
 562
 563                                                 if (0 == sym[j].n_un.n_strx) /* iff a null, "", name. */
 564                                                         continue;
 565
 566                                                 if (*jname == '_')
 567                                                         jname += 1;
 568
 569                                                 if (*(unsigned long *)sym[i].n_value <= (unsigned long)sym[j].n_value)
 570                                                         continue;
 571
 572                                                 if ((unsigned long)sym[j].n_value > best) {
 573                                                         best = (unsigned long)sym[j].n_value;
 574                                                         prev_name = jname;
 575                                                 }
 576                                         }
 577
 578                                         sdpd->sdpd_func = kmem_alloc((len = strlen(prev_name) + 1), KM_SLEEP);
 579                                         strncpy(sdpd->sdpd_func, prev_name, len); /* NUL termination is ensured. */
 580
 581                                         sdpd->sdpd_offset = *(unsigned long *)sym[i].n_value;
 582 #if defined(__arm__)
 583                                         /* PR8353094 - mask off thumb-bit */
 584                                         sdpd->sdpd_offset &= ~0x1U;
 585 #elif defined(__arm64__)
 586                                         sdpd->sdpd_offset &= ~0x1LU;
 587 #endif  /* __arm__ */
 588
 589 #if 0
 590                                         printf("sdt_init: sdpd_offset=0x%lx, n_value=0x%lx, name=%s\n",
 591                                             sdpd->sdpd_offset,  *(unsigned long *)sym[i].n_value, name);
 592 #endif
 593
 594                                         sdpd->sdpd_next = g_sdt_mach_module.sdt_probes;
 595                                         g_sdt_mach_module.sdt_probes = sdpd;
 596                                 } else {
 597                                         prev_name = name;
 598                                 }
 599                         }
 600                 }
 601
 602                 sdt_attach( (dev_info_t *)(uintptr_t)majdevno, DDI_ATTACH );
 603
 604                 gSDTInited = 1;
 605         } else
 606                 panic("sdt_init: called twice!\n");
 607 }
 608
 609 #undef SDT_MAJOR
 610
 611 /*ARGSUSED*/
 612 void
 613 sdt_provide_module(void *arg, struct modctl *ctl)
 614 {
 615 #pragma unused(arg)
 616         ASSERT(ctl != NULL);
 617         ASSERT(dtrace_kernel_symbol_mode != DTRACE_KERNEL_SYMBOLS_NEVER);
 618         LCK_MTX_ASSERT(&mod_lock, LCK_MTX_ASSERT_OWNED);
 619
 620         if (MOD_SDT_DONE(ctl))
 621                 return;
 622
 623         if (MOD_IS_MACH_KERNEL(ctl)) {
 624                 __sdt_provide_module(arg, &g_sdt_kernctl);
 625
 626                 sdt_probedesc_t *sdpd = g_sdt_mach_module.sdt_probes;
 627                 while (sdpd) {
 628                         sdt_probedesc_t *this_sdpd = sdpd;
 629                         kmem_free((void *)sdpd->sdpd_name, strlen(sdpd->sdpd_name) + 1);
 630                         kmem_free((void *)sdpd->sdpd_func, strlen(sdpd->sdpd_func) + 1);
 631                         sdpd = sdpd->sdpd_next;
 632                         kmem_free((void *)this_sdpd, sizeof(sdt_probedesc_t));
 633                 }
 634                 g_sdt_mach_module.sdt_probes = NULL;
 635         } else {
 636                 /*
 637                  * APPLE NOTE:  sdt probes for kexts not yet implemented
 638                  */
 639         }
 640
 641         /* Need to mark this module as completed */
 642         ctl->mod_flags |= MODCTL_SDT_PROBES_PROVIDED;
 643 }