4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
26 /* #pragma ident "@(#)sdt.c 1.9 08/07/01 SMI" */
30 #define _KERNEL /* Solaris vs. Darwin */
34 #include <sys/param.h>
35 #include <sys/systm.h>
36 #include <sys/errno.h>
38 #include <sys/ioctl.h>
40 #include <sys/fcntl.h>
41 #include <miscfs/devfs/devfs.h>
44 #include <sys/dtrace.h>
45 #include <sys/dtrace_impl.h>
47 #include <sys/dtrace_glue.h>
49 #include <sys/sdt_impl.h>
50 extern int dtrace_kernel_symbol_mode
;
52 /* #include <machine/trap.h */
53 struct savearea_t
; /* Used anonymously */
55 #if defined(__x86_64__)
56 typedef kern_return_t (*perfCallback
)(int, struct savearea_t
*, uintptr_t *, int);
57 extern perfCallback tempDTraceTrapHook
;
58 extern kern_return_t
fbt_perfCallback(int, struct savearea_t
*, uintptr_t *, int);
59 #define SDT_PATCHVAL 0xf0
62 #error Unknown architecture
65 #define SDT_PROBETAB_SIZE 0x1000 /* 4k entries -- 16K total */
67 #define DTRACE_PROBE_PREFIX "_dtrace_probe$"
69 static dev_info_t
*sdt_devi
;
70 static int sdt_verbose
= 0;
71 sdt_probe_t
**sdt_probetab
;
72 int sdt_probetab_size
;
73 int sdt_probetab_mask
;
77 __sdt_provide_module(void *arg
, struct modctl
*ctl
)
80 struct module *mp
= (struct module *)ctl
->mod_address
;
81 char *modname
= ctl
->mod_modname
;
82 sdt_probedesc_t
*sdpd
;
83 sdt_probe_t
*sdp
, *old
;
88 * One for all, and all for one: if we haven't yet registered all of
89 * our providers, we'll refuse to provide anything.
91 for (prov
= sdt_providers
; prov
->sdtp_name
!= NULL
; prov
++) {
92 if (prov
->sdtp_id
== DTRACE_PROVNONE
)
96 if (!mp
|| mp
->sdt_nprobes
!= 0 || (sdpd
= mp
->sdt_probes
) == NULL
)
99 for (sdpd
= mp
->sdt_probes
; sdpd
!= NULL
; sdpd
= sdpd
->sdpd_next
) {
100 const char *name
= sdpd
->sdpd_name
, *func
;
105 for (prov
= sdt_providers
; prov
->sdtp_prefix
!= NULL
; prov
++) {
106 const char *prefpart
, *prefix
= prov
->sdtp_prefix
;
108 if ((prefpart
= strstr(name
, prefix
))) {
109 name
= prefpart
+ strlen(prefix
);
114 nname
= kmem_alloc(len
= strlen(name
) + 1, KM_SLEEP
);
116 for (i
= 0, j
= 0; name
[j
] != '\0'; i
++) {
117 if (name
[j
] == '_' && name
[j
+ 1] == '_') {
121 nname
[i
] = name
[j
++];
127 sdp
= kmem_zalloc(sizeof (sdt_probe_t
), KM_SLEEP
);
128 sdp
->sdp_loadcnt
= ctl
->mod_loadcnt
;
130 sdp
->sdp_name
= nname
;
131 sdp
->sdp_namelen
= len
;
132 sdp
->sdp_provider
= prov
;
134 func
= sdpd
->sdpd_func
;
140 * We have our provider. Now create the probe.
142 if ((id
= dtrace_probe_lookup(prov
->sdtp_id
, modname
,
143 func
, nname
)) != DTRACE_IDNONE
) {
144 old
= dtrace_probe_arg(prov
->sdtp_id
, id
);
147 sdp
->sdp_next
= old
->sdp_next
;
151 sdp
->sdp_id
= dtrace_probe_create(prov
->sdtp_id
,
152 modname
, func
, nname
, SDT_AFRAMES
, sdp
);
158 printf ("__sdt_provide_module: sdpd=0x%p sdp=0x%p name=%s, id=%d\n", sdpd
, sdp
, nname
, sdp
->sdp_id
);
162 sdt_probetab
[SDT_ADDR2NDX(sdpd
->sdpd_offset
)];
163 sdt_probetab
[SDT_ADDR2NDX(sdpd
->sdpd_offset
)] = sdp
;
165 sdp
->sdp_patchval
= SDT_PATCHVAL
;
166 sdp
->sdp_patchpoint
= (sdt_instr_t
*)sdpd
->sdpd_offset
;
167 sdp
->sdp_savedval
= *sdp
->sdp_patchpoint
;
173 sdt_destroy(void *arg
, dtrace_id_t id
, void *parg
)
175 #pragma unused(arg,id)
176 sdt_probe_t
*sdp
= parg
, *old
, *last
, *hash
;
179 #if !defined(__APPLE__)
181 * APPLE NOTE: sdt probes for kexts not yet implemented
183 struct modctl
*ctl
= sdp
->sdp_ctl
;
185 if (ctl
!= NULL
&& ctl
->mod_loadcnt
== sdp
->sdp_loadcnt
) {
186 if ((ctl
->mod_loadcnt
== sdp
->sdp_loadcnt
&&
188 ((struct module *)(ctl
->mod_mp
))->sdt_nprobes
--;
191 #endif /* __APPLE__ */
193 while (sdp
!= NULL
) {
197 * Now we need to remove this probe from the sdt_probetab.
199 ndx
= SDT_ADDR2NDX(sdp
->sdp_patchpoint
);
201 hash
= sdt_probetab
[ndx
];
203 while (hash
!= sdp
) {
204 ASSERT(hash
!= NULL
);
206 hash
= hash
->sdp_hashnext
;
210 last
->sdp_hashnext
= sdp
->sdp_hashnext
;
212 sdt_probetab
[ndx
] = sdp
->sdp_hashnext
;
215 kmem_free(sdp
->sdp_name
, sdp
->sdp_namelen
);
217 kmem_free(old
, sizeof (sdt_probe_t
));
223 sdt_enable(void *arg
, dtrace_id_t id
, void *parg
)
225 #pragma unused(arg,id)
226 sdt_probe_t
*sdp
= parg
;
227 struct modctl
*ctl
= sdp
->sdp_ctl
;
232 * If this module has disappeared since we discovered its probes,
233 * refuse to enable it.
235 if (!ctl
->mod_loaded
) {
237 cmn_err(CE_NOTE
, "sdt is failing for probe %s "
238 "(module %s unloaded)",
239 sdp
->sdp_name
, ctl
->mod_modname
);
245 * Now check that our modctl has the expected load count. If it
246 * doesn't, this module must have been unloaded and reloaded -- and
247 * we're not going to touch it.
249 if (ctl
->mod_loadcnt
!= sdp
->sdp_loadcnt
) {
251 cmn_err(CE_NOTE
, "sdt is failing for probe %s "
252 "(module %s reloaded)",
253 sdp
->sdp_name
, ctl
->mod_modname
);
258 dtrace_casptr(&tempDTraceTrapHook
, NULL
, fbt_perfCallback
);
259 if (tempDTraceTrapHook
!= (perfCallback
)fbt_perfCallback
) {
261 cmn_err(CE_NOTE
, "sdt_enable is failing for probe %s "
262 "in module %s: tempDTraceTrapHook already occupied.",
263 sdp
->sdp_name
, ctl
->mod_modname
);
268 while (sdp
!= NULL
) {
269 (void)ml_nofault_copy( (vm_offset_t
)&sdp
->sdp_patchval
, (vm_offset_t
)sdp
->sdp_patchpoint
,
270 (vm_size_t
)sizeof(sdp
->sdp_patchval
));
280 sdt_disable(void *arg
, dtrace_id_t id
, void *parg
)
282 #pragma unused(arg,id)
283 sdt_probe_t
*sdp
= parg
;
284 struct modctl
*ctl
= sdp
->sdp_ctl
;
288 if (!ctl
->mod_loaded
|| ctl
->mod_loadcnt
!= sdp
->sdp_loadcnt
)
291 while (sdp
!= NULL
) {
292 (void)ml_nofault_copy( (vm_offset_t
)&sdp
->sdp_savedval
, (vm_offset_t
)sdp
->sdp_patchpoint
,
293 (vm_size_t
)sizeof(sdp
->sdp_savedval
));
301 static dtrace_pops_t sdt_pops
= {
316 sdt_attach(dev_info_t
*devi
, ddi_attach_cmd_t cmd
)
319 sdt_provider_t
*prov
;
321 if (ddi_create_minor_node(devi
, "sdt", S_IFCHR
,
322 0, DDI_PSEUDO
, 0) == DDI_FAILURE
) {
323 cmn_err(CE_NOTE
, "/dev/sdt couldn't create minor node");
324 ddi_remove_minor_node(devi
, NULL
);
325 return (DDI_FAILURE
);
328 ddi_report_dev(devi
);
331 if (sdt_probetab_size
== 0)
332 sdt_probetab_size
= SDT_PROBETAB_SIZE
;
334 sdt_probetab_mask
= sdt_probetab_size
- 1;
336 kmem_zalloc(sdt_probetab_size
* sizeof (sdt_probe_t
*), KM_SLEEP
);
337 dtrace_invop_add(sdt_invop
);
339 for (prov
= sdt_providers
; prov
->sdtp_name
!= NULL
; prov
++) {
340 if (dtrace_register(prov
->sdtp_name
, prov
->sdtp_attr
,
341 DTRACE_PRIV_KERNEL
, NULL
,
342 &sdt_pops
, prov
, &prov
->sdtp_id
) != 0) {
343 cmn_err(CE_WARN
, "failed to register sdt provider %s",
348 return (DDI_SUCCESS
);
352 * APPLE NOTE: sdt_detach not implemented
354 #if !defined(__APPLE__)
357 sdt_detach(dev_info_t
*dip
, ddi_detach_cmd_t cmd
)
359 sdt_provider_t
*prov
;
366 return (DDI_SUCCESS
);
369 return (DDI_FAILURE
);
372 for (prov
= sdt_providers
; prov
->sdtp_name
!= NULL
; prov
++) {
373 if (prov
->sdtp_id
!= DTRACE_PROVNONE
) {
374 if (dtrace_unregister(prov
->sdtp_id
) != 0)
375 return (DDI_FAILURE
);
377 prov
->sdtp_id
= DTRACE_PROVNONE
;
381 dtrace_invop_remove(sdt_invop
);
382 kmem_free(sdt_probetab
, sdt_probetab_size
* sizeof (sdt_probe_t
*));
384 return (DDI_SUCCESS
);
386 #endif /* __APPLE__ */
390 int _sdt_open(dev_t dev
, int flags
, int devtype
, struct proc
*p
)
392 #pragma unused(dev,flags,devtype,p)
396 #define SDT_MAJOR -24 /* let the kernel pick the device number */
399 * A struct describing which functions will get invoked for certain
402 static struct cdevsw sdt_cdevsw
=
404 _sdt_open
, /* open */
405 eno_opcl
, /* close */
406 eno_rdwrt
, /* read */
407 eno_rdwrt
, /* write */
408 eno_ioctl
, /* ioctl */
409 (stop_fcn_t
*)nulldev
, /* stop */
410 (reset_fcn_t
*)nulldev
, /* reset */
412 eno_select
, /* select */
414 eno_strat
, /* strategy */
420 static int gSDTInited
= 0;
421 static struct modctl g_sdt_kernctl
;
422 static struct module g_sdt_mach_module
;
424 #include <mach-o/nlist.h>
425 #include <libkern/kernel_mach_header.h>
427 void sdt_init( void )
431 int majdevno
= cdevsw_add(SDT_MAJOR
, &sdt_cdevsw
);
434 printf("sdt_init: failed to allocate a major number!\n");
439 if (dtrace_fbt_probes_restricted()) {
443 if (MH_MAGIC_KERNEL
!= _mh_execute_header
.magic
) {
444 g_sdt_kernctl
.mod_address
= (vm_address_t
)NULL
;
445 g_sdt_kernctl
.mod_size
= 0;
447 kernel_mach_header_t
*mh
;
448 struct load_command
*cmd
;
449 kernel_segment_command_t
*orig_ts
= NULL
, *orig_le
= NULL
;
450 struct symtab_command
*orig_st
= NULL
;
451 kernel_nlist_t
*sym
= NULL
;
455 g_sdt_mach_module
.sdt_nprobes
= 0;
456 g_sdt_mach_module
.sdt_probes
= NULL
;
458 g_sdt_kernctl
.mod_address
= (vm_address_t
)&g_sdt_mach_module
;
459 g_sdt_kernctl
.mod_size
= 0;
460 strncpy((char *)&(g_sdt_kernctl
.mod_modname
), "mach_kernel", KMOD_MAX_NAME
);
462 g_sdt_kernctl
.mod_next
= NULL
;
463 g_sdt_kernctl
.mod_stale
= NULL
;
464 g_sdt_kernctl
.mod_id
= 0;
465 g_sdt_kernctl
.mod_loadcnt
= 1;
466 g_sdt_kernctl
.mod_loaded
= 1;
467 g_sdt_kernctl
.mod_flags
= 0;
468 g_sdt_kernctl
.mod_nenabled
= 0;
470 mh
= &_mh_execute_header
;
471 cmd
= (struct load_command
*) &mh
[1];
472 for (i
= 0; i
< mh
->ncmds
; i
++) {
473 if (cmd
->cmd
== LC_SEGMENT_KERNEL
) {
474 kernel_segment_command_t
*orig_sg
= (kernel_segment_command_t
*) cmd
;
476 if (LIT_STRNEQL(orig_sg
->segname
, SEG_TEXT
))
478 else if (LIT_STRNEQL(orig_sg
->segname
, SEG_LINKEDIT
))
480 else if (LIT_STRNEQL(orig_sg
->segname
, ""))
481 orig_ts
= orig_sg
; /* kexts have a single unnamed segment */
483 else if (cmd
->cmd
== LC_SYMTAB
)
484 orig_st
= (struct symtab_command
*) cmd
;
486 cmd
= (struct load_command
*) ((uintptr_t) cmd
+ cmd
->cmdsize
);
489 if ((orig_ts
== NULL
) || (orig_st
== NULL
) || (orig_le
== NULL
))
492 sym
= (kernel_nlist_t
*)(orig_le
->vmaddr
+ orig_st
->symoff
- orig_le
->fileoff
);
493 strings
= (char *)(orig_le
->vmaddr
+ orig_st
->stroff
- orig_le
->fileoff
);
495 for (i
= 0; i
< orig_st
->nsyms
; i
++) {
496 uint8_t n_type
= sym
[i
].n_type
& (N_TYPE
| N_EXT
);
497 char *name
= strings
+ sym
[i
].n_un
.n_strx
;
498 const char *prev_name
;
502 /* Check that the symbol is a global and that it has a name. */
503 if (((N_SECT
| N_EXT
) != n_type
&& (N_ABS
| N_EXT
) != n_type
))
506 if (0 == sym
[i
].n_un
.n_strx
) /* iff a null, "", name. */
509 /* Lop off omnipresent leading underscore. */
513 if (strncmp(name
, DTRACE_PROBE_PREFIX
, sizeof(DTRACE_PROBE_PREFIX
) - 1) == 0) {
514 sdt_probedesc_t
*sdpd
= kmem_alloc(sizeof(sdt_probedesc_t
), KM_SLEEP
);
515 int len
= strlen(name
) + 1;
517 sdpd
->sdpd_name
= kmem_alloc(len
, KM_SLEEP
);
518 strncpy(sdpd
->sdpd_name
, name
, len
); /* NUL termination is ensured. */
520 prev_name
= "<unknown>";
524 * Find the symbol immediately preceding the sdt probe site just discovered,
525 * that symbol names the function containing the sdt probe.
527 for (j
= 0; j
< orig_st
->nsyms
; j
++) {
528 uint8_t jn_type
= sym
[j
].n_type
& (N_TYPE
| N_EXT
);
529 char *jname
= strings
+ sym
[j
].n_un
.n_strx
;
531 if (((N_SECT
| N_EXT
) != jn_type
&& (N_ABS
| N_EXT
) != jn_type
))
534 if (0 == sym
[j
].n_un
.n_strx
) /* iff a null, "", name. */
540 if (*(unsigned long *)sym
[i
].n_value
<= (unsigned long)sym
[j
].n_value
)
543 if ((unsigned long)sym
[j
].n_value
> best
) {
544 best
= (unsigned long)sym
[j
].n_value
;
549 sdpd
->sdpd_func
= kmem_alloc((len
= strlen(prev_name
) + 1), KM_SLEEP
);
550 strncpy(sdpd
->sdpd_func
, prev_name
, len
); /* NUL termination is ensured. */
552 sdpd
->sdpd_offset
= *(unsigned long *)sym
[i
].n_value
;
555 printf("sdt_init: sdpd_offset=0x%lx, n_value=0x%lx, name=%s\n",
556 sdpd
->sdpd_offset
, *(unsigned long *)sym
[i
].n_value
, name
);
559 sdpd
->sdpd_next
= g_sdt_mach_module
.sdt_probes
;
560 g_sdt_mach_module
.sdt_probes
= sdpd
;
567 sdt_attach( (dev_info_t
*)(uintptr_t)majdevno
, DDI_ATTACH
);
571 panic("sdt_init: called twice!\n");
578 sdt_provide_module(void *arg
, struct modctl
*ctl
)
582 ASSERT(dtrace_kernel_symbol_mode
!= DTRACE_KERNEL_SYMBOLS_NEVER
);
583 lck_mtx_assert(&mod_lock
, LCK_MTX_ASSERT_OWNED
);
585 if (MOD_SDT_DONE(ctl
))
588 if (MOD_IS_MACH_KERNEL(ctl
)) {
589 __sdt_provide_module(arg
, &g_sdt_kernctl
);
591 sdt_probedesc_t
*sdpd
= g_sdt_mach_module
.sdt_probes
;
593 sdt_probedesc_t
*this_sdpd
= sdpd
;
594 kmem_free((void *)sdpd
->sdpd_name
, strlen(sdpd
->sdpd_name
) + 1);
595 kmem_free((void *)sdpd
->sdpd_func
, strlen(sdpd
->sdpd_func
) + 1);
596 sdpd
= sdpd
->sdpd_next
;
597 kmem_free((void *)this_sdpd
, sizeof(sdt_probedesc_t
));
599 g_sdt_mach_module
.sdt_probes
= NULL
;
602 * APPLE NOTE: sdt probes for kexts not yet implemented
606 /* Need to mark this module as completed */
607 ctl
->mod_flags
|= MODCTL_SDT_PROBES_PROVIDED
;