bsd/dev/dtrace/sdt.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25
  26 /* #pragma ident        "@(#)sdt.c      1.9     08/07/01 SMI" */
  27
  28 #ifdef KERNEL
  29 #ifndef _KERNEL
  30 #define _KERNEL /* Solaris vs. Darwin */
  31 #endif
  32 #endif
  33
  34 #include <sys/param.h>
  35 #include <sys/systm.h>
  36 #include <sys/errno.h>
  37 #include <sys/stat.h>
  38 #include <sys/ioctl.h>
  39 #include <sys/conf.h>
  40 #include <sys/fcntl.h>
  41 #include <miscfs/devfs/devfs.h>
  42
  43 #include <sys/dtrace.h>
  44 #include <sys/dtrace_impl.h>
  45
  46 #include <sys/dtrace_glue.h>
  47
  48 #include <sys/sdt_impl.h>
  49
  50 struct savearea_t; /* Used anonymously */
  51 typedef kern_return_t (*perfCallback)(int, struct savearea_t *, int, int);
  52
  53 #if defined (__ppc__) || defined (__ppc64__)
  54 extern perfCallback tempDTraceTrapHook, tempDTraceIntHook;
  55 extern kern_return_t fbt_perfCallback(int, struct savearea_t *, int, int);
  56 extern kern_return_t fbt_perfIntCallback(int, struct savearea_t *, int, int);
  57
  58 #define SDT_PATCHVAL    0x7c810808
  59 #define SDT_AFRAMES     6
  60 #elif defined(__i386__) || defined(__x86_64__)
  61 extern perfCallback tempDTraceTrapHook;
  62 extern kern_return_t fbt_perfCallback(int, struct savearea_t *, int, int);
  63
  64 #define SDT_PATCHVAL    0xf0
  65 #define SDT_AFRAMES             6
  66 #else
  67 #error Unknown architecture
  68 #endif
  69
  70 #define SDT_PROBETAB_SIZE       0x1000          /* 4k entries -- 16K total */
  71
  72 #if defined(__x86_64__)
  73 #define DTRACE_PROBE_PREFIX "_dtrace_probeDOLLAR"
  74 #else
  75 #define DTRACE_PROBE_PREFIX "_dtrace_probe$"
  76 #endif
  77
  78 static dev_info_t               *sdt_devi;
  79 static int                      sdt_verbose = 0;
  80 sdt_probe_t             **sdt_probetab;
  81 int                     sdt_probetab_size;
  82 int                     sdt_probetab_mask;
  83
  84 /*ARGSUSED*/
  85 static void
  86 __sdt_provide_module(void *arg, struct modctl *ctl)
  87 {
  88 #pragma unused(arg)
  89         struct module *mp = (struct module *)ctl->address;
  90         char *modname = ctl->mod_modname;
  91         sdt_probedesc_t *sdpd;
  92         sdt_probe_t *sdp, *old;
  93         sdt_provider_t *prov;
  94         int len;
  95
  96         /*
  97          * One for all, and all for one:  if we haven't yet registered all of
  98          * our providers, we'll refuse to provide anything.
  99          */
 100         for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
 101                 if (prov->sdtp_id == DTRACE_PROVNONE)
 102                         return;
 103         }
 104
 105         if (!mp || mp->sdt_nprobes != 0 || (sdpd = mp->sdt_probes) == NULL)
 106                 return;
 107
 108         for (sdpd = mp->sdt_probes; sdpd != NULL; sdpd = sdpd->sdpd_next) {
 109             const char *name = sdpd->sdpd_name, *func;
 110             char *nname;
 111                 int i, j;
 112                 dtrace_id_t id;
 113
 114                 for (prov = sdt_providers; prov->sdtp_prefix != NULL; prov++) {
 115                         const char *prefpart, *prefix = prov->sdtp_prefix;
 116
 117                         if ((prefpart = strstr(name, prefix))) {
 118                                 name = prefpart + strlen(prefix);
 119                                 break;
 120                         }
 121                 }
 122
 123                 nname = kmem_alloc(len = strlen(name) + 1, KM_SLEEP);
 124
 125                 for (i = 0, j = 0; name[j] != '\0'; i++) {
 126                         if (name[j] == '_' && name[j + 1] == '_') {
 127                                 nname[i] = '-';
 128                                 j += 2;
 129                         } else {
 130                                 nname[i] = name[j++];
 131                         }
 132                 }
 133
 134                 nname[i] = '\0';
 135
 136                 sdp = kmem_zalloc(sizeof (sdt_probe_t), KM_SLEEP);
 137                 sdp->sdp_loadcnt = ctl->mod_loadcnt;
 138                 sdp->sdp_ctl = ctl;
 139                 sdp->sdp_name = nname;
 140                 sdp->sdp_namelen = len;
 141                 sdp->sdp_provider = prov;
 142
 143                 func = sdpd->sdpd_func;
 144
 145                 if (func == NULL)
 146                         func = "<unknown>";
 147
 148                 /*
 149                  * We have our provider.  Now create the probe.
 150                  */
 151                 if ((id = dtrace_probe_lookup(prov->sdtp_id, modname,
 152                     func, nname)) != DTRACE_IDNONE) {
 153                         old = dtrace_probe_arg(prov->sdtp_id, id);
 154                         ASSERT(old != NULL);
 155
 156                         sdp->sdp_next = old->sdp_next;
 157                         sdp->sdp_id = id;
 158                         old->sdp_next = sdp;
 159                 } else {
 160                         sdp->sdp_id = dtrace_probe_create(prov->sdtp_id,
 161                             modname, func, nname, SDT_AFRAMES, sdp);
 162
 163                         mp->sdt_nprobes++;
 164                 }
 165
 166                 sdp->sdp_hashnext =
 167                     sdt_probetab[SDT_ADDR2NDX(sdpd->sdpd_offset)];
 168                 sdt_probetab[SDT_ADDR2NDX(sdpd->sdpd_offset)] = sdp;
 169
 170                 sdp->sdp_patchval = SDT_PATCHVAL;
 171                 sdp->sdp_patchpoint = (sdt_instr_t *)sdpd->sdpd_offset;
 172                 sdp->sdp_savedval = *sdp->sdp_patchpoint;
 173         }
 174 }
 175
 176 /*ARGSUSED*/
 177 static void
 178 sdt_destroy(void *arg, dtrace_id_t id, void *parg)
 179 {
 180 #pragma unused(arg,id)
 181         sdt_probe_t *sdp = parg, *old, *last, *hash;
 182         int ndx;
 183 #if !defined(__APPLE__)
 184         struct modctl *ctl = sdp->sdp_ctl;
 185
 186         if (ctl != NULL && ctl->mod_loadcnt == sdp->sdp_loadcnt) {
 187                 if ((ctl->mod_loadcnt == sdp->sdp_loadcnt &&
 188                     ctl->mod_loaded)) {
 189                         ((struct module *)(ctl->mod_mp))->sdt_nprobes--;
 190                 }
 191         }
 192 #endif /* __APPLE__ */
 193
 194         while (sdp != NULL) {
 195                 old = sdp;
 196
 197                 /*
 198                  * Now we need to remove this probe from the sdt_probetab.
 199                  */
 200                 ndx = SDT_ADDR2NDX(sdp->sdp_patchpoint);
 201                 last = NULL;
 202                 hash = sdt_probetab[ndx];
 203
 204                 while (hash != sdp) {
 205                         ASSERT(hash != NULL);
 206                         last = hash;
 207                         hash = hash->sdp_hashnext;
 208                 }
 209
 210                 if (last != NULL) {
 211                         last->sdp_hashnext = sdp->sdp_hashnext;
 212                 } else {
 213                         sdt_probetab[ndx] = sdp->sdp_hashnext;
 214                 }
 215
 216                 kmem_free(sdp->sdp_name, sdp->sdp_namelen);
 217                 sdp = sdp->sdp_next;
 218                 kmem_free(old, sizeof (sdt_probe_t));
 219         }
 220 }
 221
 222 /*ARGSUSED*/
 223 static void
 224 sdt_enable(void *arg, dtrace_id_t id, void *parg)
 225 {
 226 #pragma unused(arg,id)
 227         sdt_probe_t *sdp = parg;
 228         struct modctl *ctl = sdp->sdp_ctl;
 229
 230 #if !defined(__APPLE__)
 231         ctl->mod_nenabled++;
 232
 233         /*
 234          * If this module has disappeared since we discovered its probes,
 235          * refuse to enable it.
 236          */
 237         if (!ctl->mod_loaded) {
 238                 if (sdt_verbose) {
 239                         cmn_err(CE_NOTE, "sdt is failing for probe %s "
 240                             "(module %s unloaded)",
 241                             sdp->sdp_name, ctl->mod_modname);
 242                 }
 243                 goto err;
 244         }
 245
 246         /*
 247          * Now check that our modctl has the expected load count.  If it
 248          * doesn't, this module must have been unloaded and reloaded -- and
 249          * we're not going to touch it.
 250          */
 251         if (ctl->mod_loadcnt != sdp->sdp_loadcnt) {
 252                 if (sdt_verbose) {
 253                         cmn_err(CE_NOTE, "sdt is failing for probe %s "
 254                             "(module %s reloaded)",
 255                             sdp->sdp_name, ctl->mod_modname);
 256                 }
 257                 goto err;
 258         }
 259 #endif /* __APPLE__ */
 260
 261 #if defined (__ppc__) || defined (__ppc64__)
 262         dtrace_casptr(&tempDTraceIntHook, NULL, fbt_perfIntCallback);
 263         if (tempDTraceIntHook != (perfCallback)fbt_perfIntCallback) {
 264                 if (sdt_verbose) {
 265                         cmn_err(CE_NOTE, "sdt_enable is failing for probe %s "
 266                             "in module %s: tempDTraceIntHook already occupied.",
 267                             sdp->sdp_name, ctl->mod_modname);
 268                 }
 269                 return;
 270         }
 271 #endif
 272
 273         dtrace_casptr(&tempDTraceTrapHook, NULL, fbt_perfCallback);
 274         if (tempDTraceTrapHook != (perfCallback)fbt_perfCallback) {
 275                 if (sdt_verbose) {
 276                         cmn_err(CE_NOTE, "sdt_enable is failing for probe %s "
 277                             "in module %s: tempDTraceTrapHook already occupied.",
 278                             sdp->sdp_name, ctl->mod_modname);
 279                 }
 280                 return;
 281         }
 282
 283         while (sdp != NULL) {
 284                 (void)ml_nofault_copy( (vm_offset_t)&sdp->sdp_patchval, (vm_offset_t)sdp->sdp_patchpoint,
 285                                        (vm_size_t)sizeof(sdp->sdp_patchval));
 286                 sdp = sdp->sdp_next;
 287         }
 288 #if !defined(__APPLE__)
 289 err:
 290 #endif /* __APPLE__ */
 291         ;
 292 }
 293
 294 /*ARGSUSED*/
 295 static void
 296 sdt_disable(void *arg, dtrace_id_t id, void *parg)
 297 {
 298 #pragma unused(arg,id)
 299         sdt_probe_t *sdp = parg;
 300 #if !defined(__APPLE__)
 301         struct modctl *ctl = sdp->sdp_ctl;
 302
 303         ctl->mod_nenabled--;
 304
 305         if (!ctl->mod_loaded || ctl->mod_loadcnt != sdp->sdp_loadcnt)
 306                 goto err;
 307 #endif /* __APPLE__ */
 308
 309         while (sdp != NULL) {
 310                 (void)ml_nofault_copy( (vm_offset_t)&sdp->sdp_savedval, (vm_offset_t)sdp->sdp_patchpoint,
 311                                        (vm_size_t)sizeof(sdp->sdp_savedval));
 312                 sdp = sdp->sdp_next;
 313         }
 314
 315 #if !defined(__APPLE__)
 316 err:
 317 #endif /* __APPLE__ */
 318         ;
 319 }
 320
 321 static uint64_t
 322 sdt_getarg(void *arg, dtrace_id_t id, void *parg, int argno, int aframes)
 323 {
 324 #pragma unused(arg,id,parg)     /* __APPLE__ */
 325         return dtrace_getarg(argno, aframes);
 326 }
 327
 328 static dtrace_pops_t sdt_pops = {
 329         NULL,
 330         sdt_provide_module,
 331         sdt_enable,
 332         sdt_disable,
 333         NULL,
 334         NULL,
 335         sdt_getargdesc,
 336         sdt_getarg,
 337         NULL,
 338         sdt_destroy
 339 };
 340
 341 /*ARGSUSED*/
 342 static int
 343 sdt_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
 344 {
 345 #pragma unused(cmd)
 346         sdt_provider_t *prov;
 347
 348         if (ddi_create_minor_node(devi, "sdt", S_IFCHR,
 349             0, DDI_PSEUDO, 0) == DDI_FAILURE) {
 350                 cmn_err(CE_NOTE, "/dev/sdt couldn't create minor node");
 351                 ddi_remove_minor_node(devi, NULL);
 352                 return (DDI_FAILURE);
 353         }
 354
 355         ddi_report_dev(devi);
 356         sdt_devi = devi;
 357
 358         if (sdt_probetab_size == 0)
 359                 sdt_probetab_size = SDT_PROBETAB_SIZE;
 360
 361         sdt_probetab_mask = sdt_probetab_size - 1;
 362         sdt_probetab =
 363             kmem_zalloc(sdt_probetab_size * sizeof (sdt_probe_t *), KM_SLEEP);
 364         dtrace_invop_add(sdt_invop);
 365
 366         for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
 367                 if (dtrace_register(prov->sdtp_name, prov->sdtp_attr,
 368                     DTRACE_PRIV_KERNEL, NULL,
 369                     &sdt_pops, prov, &prov->sdtp_id) != 0) {
 370                         cmn_err(CE_WARN, "failed to register sdt provider %s",
 371                             prov->sdtp_name);
 372                 }
 373         }
 374
 375         return (DDI_SUCCESS);
 376 }
 377
 378 #if !defined(__APPLE__)
 379 /*ARGSUSED*/
 380 static int
 381 sdt_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
 382 {
 383         sdt_provider_t *prov;
 384
 385         switch (cmd) {
 386         case DDI_DETACH:
 387                 break;
 388
 389         case DDI_SUSPEND:
 390                 return (DDI_SUCCESS);
 391
 392         default:
 393                 return (DDI_FAILURE);
 394         }
 395
 396         for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
 397                 if (prov->sdtp_id != DTRACE_PROVNONE) {
 398                         if (dtrace_unregister(prov->sdtp_id) != 0)
 399                                 return (DDI_FAILURE);
 400
 401                         prov->sdtp_id = DTRACE_PROVNONE;
 402                 }
 403         }
 404
 405         dtrace_invop_remove(sdt_invop);
 406         kmem_free(sdt_probetab, sdt_probetab_size * sizeof (sdt_probe_t *));
 407
 408         return (DDI_SUCCESS);
 409 }
 410
 411 /*ARGSUSED*/
 412 static int
 413 sdt_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
 414 {
 415         int error;
 416
 417         switch (infocmd) {
 418         case DDI_INFO_DEVT2DEVINFO:
 419                 *result = (void *)sdt_devi;
 420                 error = DDI_SUCCESS;
 421                 break;
 422         case DDI_INFO_DEVT2INSTANCE:
 423                 *result = (void *)0;
 424                 error = DDI_SUCCESS;
 425                 break;
 426         default:
 427                 error = DDI_FAILURE;
 428         }
 429         return (error);
 430 }
 431
 432 /*ARGSUSED*/
 433 static int
 434 sdt_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
 435 {
 436         return (0);
 437 }
 438
 439 static struct cb_ops sdt_cb_ops = {
 440         sdt_open,               /* open */
 441         nodev,                  /* close */
 442         nulldev,                /* strategy */
 443         nulldev,                /* print */
 444         nodev,                  /* dump */
 445         nodev,                  /* read */
 446         nodev,                  /* write */
 447         nodev,                  /* ioctl */
 448         nodev,                  /* devmap */
 449         nodev,                  /* mmap */
 450         nodev,                  /* segmap */
 451         nochpoll,               /* poll */
 452         ddi_prop_op,            /* cb_prop_op */
 453         0,                      /* streamtab  */
 454         D_NEW | D_MP            /* Driver compatibility flag */
 455 };
 456
 457 static struct dev_ops sdt_ops = {
 458         DEVO_REV,               /* devo_rev, */
 459         0,                      /* refcnt  */
 460         sdt_info,               /* get_dev_info */
 461         nulldev,                /* identify */
 462         nulldev,                /* probe */
 463         sdt_attach,             /* attach */
 464         sdt_detach,             /* detach */
 465         nodev,                  /* reset */
 466         &sdt_cb_ops,            /* driver operations */
 467         NULL,                   /* bus operations */
 468         nodev                   /* dev power */
 469 };
 470
 471 /*
 472  * Module linkage information for the kernel.
 473  */
 474 static struct modldrv modldrv = {
 475         &mod_driverops,         /* module type (this is a pseudo driver) */
 476         "Statically Defined Tracing",   /* name of module */
 477         &sdt_ops,               /* driver ops */
 478 };
 479
 480 static struct modlinkage modlinkage = {
 481         MODREV_1,
 482         (void *)&modldrv,
 483         NULL
 484 };
 485
 486 int
 487 _init(void)
 488 {
 489         return (mod_install(&modlinkage));
 490 }
 491
 492 int
 493 _info(struct modinfo *modinfop)
 494 {
 495         return (mod_info(&modlinkage, modinfop));
 496 }
 497
 498 int
 499 _fini(void)
 500 {
 501         return (mod_remove(&modlinkage));
 502 }
 503 #else
 504 d_open_t _sdt_open;
 505
 506 int _sdt_open(dev_t dev, int flags, int devtype, struct proc *p)
 507 {
 508 #pragma unused(dev,flags,devtype,p)
 509         return 0;
 510 }
 511
 512 #define SDT_MAJOR  -24 /* let the kernel pick the device number */
 513
 514 /*
 515  * A struct describing which functions will get invoked for certain
 516  * actions.
 517  */
 518 static struct cdevsw sdt_cdevsw =
 519 {
 520         _sdt_open,              /* open */
 521         eno_opcl,                       /* close */
 522         eno_rdwrt,                      /* read */
 523         eno_rdwrt,                      /* write */
 524         eno_ioctl,                      /* ioctl */
 525         (stop_fcn_t *)nulldev, /* stop */
 526         (reset_fcn_t *)nulldev, /* reset */
 527         NULL,                           /* tty's */
 528         eno_select,                     /* select */
 529         eno_mmap,                       /* mmap */
 530         eno_strat,                      /* strategy */
 531         eno_getc,                       /* getc */
 532         eno_putc,                       /* putc */
 533         0                                       /* type */
 534 };
 535
 536 static int gSDTInited = 0;
 537 static struct modctl g_sdt_kernctl;
 538 static struct module g_sdt_mach_module;
 539
 540 #include <mach-o/nlist.h>
 541 #include <libkern/kernel_mach_header.h>
 542
 543 #if defined(__LP64__)
 544 #define KERNEL_MAGIC MH_MAGIC_64
 545 typedef struct nlist_64 kernel_nlist_t;
 546 #else
 547 #define KERNEL_MAGIC MH_MAGIC
 548 typedef struct nlist kernel_nlist_t;
 549 #endif
 550
 551 void sdt_init( void )
 552 {
 553         if (0 == gSDTInited)
 554         {
 555                 int majdevno = cdevsw_add(SDT_MAJOR, &sdt_cdevsw);
 556
 557                 if (majdevno < 0) {
 558                         printf("sdt_init: failed to allocate a major number!\n");
 559                         gSDTInited = 0;
 560                         return;
 561                 }
 562
 563                 if (KERNEL_MAGIC != _mh_execute_header.magic) {
 564                 g_sdt_kernctl.address = (vm_address_t)NULL;
 565                 g_sdt_kernctl.size = 0;
 566                 } else {
 567                 kernel_mach_header_t        *mh;
 568                 struct load_command         *cmd;
 569                 kernel_segment_command_t    *orig_ts = NULL, *orig_le = NULL;
 570                 struct symtab_command       *orig_st = NULL;
 571                 kernel_nlist_t              *sym = NULL;
 572                 char                        *strings;
 573                 unsigned int                i;
 574
 575                 g_sdt_mach_module.sdt_nprobes = 0;
 576                 g_sdt_mach_module.sdt_probes = NULL;
 577
 578                 g_sdt_kernctl.address = (vm_address_t)&g_sdt_mach_module;
 579                 g_sdt_kernctl.size = 0;
 580                 strncpy((char *)&(g_sdt_kernctl.mod_modname), "mach_kernel", KMOD_MAX_NAME);
 581
 582                 mh = &_mh_execute_header;
 583                 cmd = (struct load_command*) &mh[1];
 584                 for (i = 0; i < mh->ncmds; i++) {
 585                         if (cmd->cmd == LC_SEGMENT_KERNEL) {
 586                         kernel_segment_command_t *orig_sg = (kernel_segment_command_t *) cmd;
 587
 588                         if (LIT_STRNEQL(orig_sg->segname, SEG_TEXT))
 589                                 orig_ts = orig_sg;
 590                         else if (LIT_STRNEQL(orig_sg->segname, SEG_LINKEDIT))
 591                                 orig_le = orig_sg;
 592                         else if (LIT_STRNEQL(orig_sg->segname, ""))
 593                                 orig_ts = orig_sg; /* kexts have a single unnamed segment */
 594                         }
 595                         else if (cmd->cmd == LC_SYMTAB)
 596                         orig_st = (struct symtab_command *) cmd;
 597
 598                         cmd = (struct load_command *) ((uintptr_t) cmd + cmd->cmdsize);
 599                 }
 600
 601                 if ((orig_ts == NULL) || (orig_st == NULL) || (orig_le == NULL))
 602                         return;
 603
 604                 sym = (kernel_nlist_t *)(orig_le->vmaddr + orig_st->symoff - orig_le->fileoff);
 605                 strings = (char *)(orig_le->vmaddr + orig_st->stroff - orig_le->fileoff);
 606
 607                 for (i = 0; i < orig_st->nsyms; i++) {
 608                         uint8_t n_type = sym[i].n_type & (N_TYPE | N_EXT);
 609                         char *name = strings + sym[i].n_un.n_strx;
 610                                 const char *prev_name;
 611                                 unsigned long best;
 612                                 unsigned int j;
 613
 614                         /* Check that the symbol is a global and that it has a name. */
 615                         if (((N_SECT | N_EXT) != n_type && (N_ABS | N_EXT) != n_type))
 616                         continue;
 617
 618                         if (0 == sym[i].n_un.n_strx) /* iff a null, "", name. */
 619                         continue;
 620
 621                         /* Lop off omnipresent leading underscore. */
 622                         if (*name == '_')
 623                         name += 1;
 624
 625                                 if (strstr(name, DTRACE_PROBE_PREFIX)) {
 626                                         sdt_probedesc_t *sdpd = kmem_alloc(sizeof(sdt_probedesc_t), KM_SLEEP);
 627                                         int len = strlen(name) + 1;
 628
 629                                         sdpd->sdpd_name = kmem_alloc(len, KM_SLEEP);
 630                                         strncpy(sdpd->sdpd_name, name, len); /* NUL termination is ensured. */
 631
 632                                         prev_name = "<unknown>";
 633                                         best = 0;
 634
 635                                         /* Avoid shadow build warnings */
 636                                         for (j = 0; j < orig_st->nsyms; j++) {
 637                                                 uint8_t jn_type = sym[j].n_type & (N_TYPE | N_EXT);
 638                                                 char *jname = strings + sym[j].n_un.n_strx;
 639
 640                                                 if (((N_SECT | N_EXT) != jn_type && (N_ABS | N_EXT) != jn_type))
 641                                                         continue;
 642
 643                                                 if (0 == sym[j].n_un.n_strx) /* iff a null, "", name. */
 644                                                         continue;
 645
 646                                                 if (*jname == '_')
 647                                                         jname += 1;
 648                                                 if (strstr(jname, DTRACE_PROBE_PREFIX))
 649                                                         continue;
 650
 651                                                 if (*(unsigned long *)sym[i].n_value <= (unsigned long)sym[j].n_value)
 652                                                         continue;
 653
 654                                                 if ((unsigned long)sym[j].n_value > best) {
 655                                                         best = (unsigned long)sym[j].n_value;
 656                                                         prev_name = jname;
 657                                                 }
 658                                         }
 659
 660                                         sdpd->sdpd_func = kmem_alloc((len = strlen(prev_name) + 1), KM_SLEEP);
 661                                         strncpy(sdpd->sdpd_func, prev_name, len); /* NUL termination is ensured. */
 662
 663                                         sdpd->sdpd_offset = *(unsigned long *)sym[i].n_value;
 664
 665                                         sdpd->sdpd_next = g_sdt_mach_module.sdt_probes;
 666                                         g_sdt_mach_module.sdt_probes = sdpd;
 667                                 } else {
 668                                         prev_name = name;
 669                                 }
 670                         }
 671                 }
 672
 673                 sdt_attach( (dev_info_t *)(uintptr_t)majdevno, DDI_ATTACH );
 674
 675                 gSDTInited = 1;
 676         } else
 677                 panic("sdt_init: called twice!\n");
 678 }
 679
 680 #undef SDT_MAJOR
 681
 682 /*ARGSUSED*/
 683 void
 684 sdt_provide_module(void *arg, struct modctl *ctl)
 685 {
 686 #pragma unused(ctl)
 687 #pragma unused(arg)
 688     __sdt_provide_module(arg, &g_sdt_kernctl);
 689
 690         sdt_probedesc_t *sdpd = g_sdt_mach_module.sdt_probes;
 691         while (sdpd) {
 692                 sdt_probedesc_t *this_sdpd = sdpd;
 693                 kmem_free((void *)sdpd->sdpd_name, strlen(sdpd->sdpd_name) + 1);
 694                 kmem_free((void *)sdpd->sdpd_func, strlen(sdpd->sdpd_func) + 1);
 695                 sdpd = sdpd->sdpd_next;
 696                 kmem_free((void *)this_sdpd, sizeof(sdt_probedesc_t));
 697         }
 698         g_sdt_mach_module.sdt_probes = NULL;
 699 }
 700
 701 #endif /* __APPLE__ */