4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
26 /* #pragma ident "@(#)lockstat.c 1.12 08/01/16 SMI" */
31 #define _KERNEL /* Solaris vs. Darwin */
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/errno.h>
39 #include <sys/ioctl.h>
41 #include <sys/fcntl.h>
42 #include <miscfs/devfs/devfs.h>
44 #include <sys/dtrace.h>
45 #include <sys/dtrace_impl.h>
47 #include <sys/dtrace_glue.h>
49 #include <sys/lockstat.h>
51 #include <kern/lock_stat.h>
53 #include <kern/processor.h>
55 #define membar_producer dtrace_membar_producer
57 #define PROBE_ARGS0(a, b, c, d, e) "\000"
58 #define PROBE_ARGS1(a, b, c, d, e) a "\000"
59 #define PROBE_ARGS2(a, b, c, d, e) a "\000" b "\000"
60 #define PROBE_ARGS3(a, b, c, d, e) a "\000" b "\000" c "\000"
61 #define PROBE_ARGS4(a, b, c, d, e) a "\000" b "\000" c "\000" d "\000"
62 #define PROBE_ARGS5(a, b, c, d, e) a "\000" b "\000" c "\000" d "\000" e "\000"
63 #define PROBE_ARGS_(a, b, c, d, e, n, ...) PROBE_ARGS##n(a, b, c, d, e)
64 #define PROBE_ARGS(...) PROBE_ARGS_(__VA_ARGS__, 5, 4, 3, 2, 1, 0)
66 #define LOCKSTAT_PROBE(func, name, probe, ...) \
67 {func, name, probe, DTRACE_IDNONE, PROBE_ARGS(__VA_ARGS__)}
70 * Hot patch values, x86
72 #if defined(__x86_64__)
75 #define LOCKSTAT_AFRAMES 1
76 #elif defined(__arm__)
77 #define NOP 0xE1A00000
78 #define BXLR 0xE12FFF1E
79 #define LOCKSTAT_AFRAMES 2
80 #elif defined(__arm64__)
81 #define NOP 0xD503201F
82 #define RET 0xD65f03c0
83 #define LOCKSTAT_AFRAMES 2
85 #error "not ported to this architecture"
88 typedef struct lockstat_probe
{
96 lockstat_probe_t lockstat_probes
[] =
98 #if defined(__x86_64__)
99 /* Only provide implemented probes for each architecture */
100 LOCKSTAT_PROBE(LS_LCK_MTX_LOCK
, LSA_ACQUIRE
, LS_LCK_MTX_LOCK_ACQUIRE
, "lck_mtx_t"),
101 LOCKSTAT_PROBE(LS_LCK_MTX_LOCK
, LSA_SPIN
, LS_LCK_MTX_LOCK_SPIN
, "lck_mtx_t", "uint64_t"),
102 LOCKSTAT_PROBE(LS_LCK_MTX_LOCK
, LSA_BLOCK
, LS_LCK_MTX_LOCK_BLOCK
, "lck_mtx_t", "uint64_t"),
103 LOCKSTAT_PROBE(LS_LCK_MTX_TRY_LOCK
, LSA_ACQUIRE
, LS_LCK_MTX_TRY_LOCK_ACQUIRE
, "lck_mtx_t"),
104 LOCKSTAT_PROBE(LS_LCK_MTX_TRY_SPIN_LOCK
, LSA_ACQUIRE
, LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE
, "lck_mtx_t"),
105 LOCKSTAT_PROBE(LS_LCK_MTX_UNLOCK
, LSA_RELEASE
, LS_LCK_MTX_UNLOCK_RELEASE
, "lck_mtx_t"),
106 LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK
, LSA_ACQUIRE
, LS_LCK_MTX_EXT_LOCK_ACQUIRE
, "lck_mtx_t"),
107 LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK
, LSA_SPIN
, LS_LCK_MTX_EXT_LOCK_SPIN
, "lck_mtx_t", "uint64_t"),
108 LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK
, LSA_BLOCK
, LS_LCK_MTX_EXT_LOCK_BLOCK
, "lck_mtx_t", "uint64_t"),
109 // LOCKSTAT_PROBE(LS_LCK_MTX_EXT_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_EXT_LOCK_ACQUIRE)
110 LOCKSTAT_PROBE(LS_LCK_MTX_EXT_UNLOCK
, LSA_RELEASE
, LS_LCK_MTX_EXT_UNLOCK_RELEASE
, "lck_mtx_t"),
111 LOCKSTAT_PROBE(LS_LCK_MTX_LOCK_SPIN_LOCK
, LSA_ACQUIRE
, LS_LCK_MTX_LOCK_SPIN_ACQUIRE
, "lck_mtx_t"),
112 // TODO: This should not be a uint64_t !
113 LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED
, LSR_ACQUIRE
, LS_LCK_RW_LOCK_SHARED_ACQUIRE
, "lck_rw_t", "uint64_t"),
114 LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED
, LSR_BLOCK
, LS_LCK_RW_LOCK_SHARED_BLOCK
, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
115 LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED
, LSR_SPIN
, LS_LCK_RW_LOCK_SHARED_SPIN
, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
116 // TODO: This should NOT be a uint64_t
117 LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL
, LSR_ACQUIRE
, LS_LCK_RW_LOCK_EXCL_ACQUIRE
, "lck_rw_t", "uint64_t"),
118 LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL
, LSR_BLOCK
, LS_LCK_RW_LOCK_EXCL_BLOCK
, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
119 LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL
, LSR_SPIN
, LS_LCK_RW_LOCK_EXCL_SPIN
, "lck_rw_t", "uint64_t", "int"),
120 LOCKSTAT_PROBE(LS_LCK_RW_DONE
, LSR_RELEASE
, LS_LCK_RW_DONE_RELEASE
, "lck_rw_t", "_Bool"),
121 // TODO : This should NOT be a uint64_t
122 LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_SHARED
, LSR_ACQUIRE
, LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE
, "lck_rw_t", "uint64_t"),
124 LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_EXCL
, LSR_ACQUIRE
, LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE
, "lck_rw_t", "uint64_t"),
125 LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL
, LSR_UPGRADE
, LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE
, "lck_rw_t", "_Bool"),
126 LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL
, LSR_SPIN
, LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN
, "lck_rw_t", "uint64_t"),
127 LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL
, LSR_BLOCK
, LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK
, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
128 LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL_TO_SHARED
, LSR_DOWNGRADE
, LS_LCK_RW_LOCK_EXCL_TO_SHARED_DOWNGRADE
, "lck_rw_t"),
129 //TODO : Separate the probes for the hw_bit from the probe for the normal hw locks
130 LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK
, LSS_ACQUIRE
, LS_LCK_SPIN_LOCK_ACQUIRE
, "hw_lock_t"),
131 LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK
, LSS_SPIN
, LS_LCK_SPIN_LOCK_SPIN
, "hw_lock_t", "uint64_t", "uint64_t"),
132 LOCKSTAT_PROBE(LS_LCK_SPIN_UNLOCK
, LSS_RELEASE
, LS_LCK_SPIN_UNLOCK_RELEASE
, "hw_lock_t"),
133 #elif defined(__arm__) || defined(__arm64__)
134 LOCKSTAT_PROBE(LS_LCK_MTX_LOCK
, LSA_ACQUIRE
, LS_LCK_MTX_LOCK_ACQUIRE
, "lck_mtx_t"),
135 // LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_SPIN, LS_LCK_MTX_LOCK_SPIN, "lck_mtx_t", "uint64_t"),
136 LOCKSTAT_PROBE(LS_LCK_MTX_LOCK
, LSA_BLOCK
, LS_LCK_MTX_LOCK_BLOCK
, "lck_mtx_t", "uint64_t"),
137 LOCKSTAT_PROBE(LS_LCK_MTX_TRY_LOCK
, LSA_ACQUIRE
, LS_LCK_MTX_TRY_LOCK_ACQUIRE
, "lck_mtx_t"),
138 // LOCKSTAT_PROBE(LS_LCK_MTX_TRY_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE, "lck_mtx_t"),
139 LOCKSTAT_PROBE(LS_LCK_MTX_UNLOCK
, LSA_RELEASE
, LS_LCK_MTX_UNLOCK_RELEASE
, "lck_mtx_t"),
140 LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK
, LSA_ACQUIRE
, LS_LCK_MTX_EXT_LOCK_ACQUIRE
, "lck_mtx_t"),
141 // LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_SPIN, LS_LCK_MTX_EXT_LOCK_SPIN, "lck_mtx_t", "uint64_t"),
142 LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK
, LSA_BLOCK
, LS_LCK_MTX_EXT_LOCK_BLOCK
, "lck_mtx_t", "uint64_t"),
143 // LOCKSTAT_PROBE(LS_LCK_MTX_EXT_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_EXT_LOCK_ACQUIRE)
144 // LOCKSTAT_PROBE(LS_LCK_MTX_EXT_UNLOCK, LSA_RELEASE, LS_LCK_MTX_EXT_UNLOCK_RELEASE, "lck_mtx_t"),
145 // LOCKSTAT_PROBE(LS_LCK_MTX_LOCK_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_SPIN_ACQUIRE, "lck_mtx_t"),
146 LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED
, LSR_ACQUIRE
, LS_LCK_RW_LOCK_SHARED_ACQUIRE
, "lck_rw_t", "uint64_t"),
147 LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED
, LSR_BLOCK
, LS_LCK_RW_LOCK_SHARED_BLOCK
, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
148 LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED
, LSR_SPIN
, LS_LCK_RW_LOCK_SHARED_SPIN
, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
149 LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL
, LSR_ACQUIRE
, LS_LCK_RW_LOCK_EXCL_ACQUIRE
, "lck_rw_t", "uint64_t"),
150 LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL
, LSR_BLOCK
, LS_LCK_RW_LOCK_EXCL_BLOCK
, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
151 LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL
, LSR_SPIN
, LS_LCK_RW_LOCK_EXCL_SPIN
, "lck_rw_t", "uint64_t", "int"),
152 LOCKSTAT_PROBE(LS_LCK_RW_DONE
, LSR_RELEASE
, LS_LCK_RW_DONE_RELEASE
, "lck_rw_t", "_Bool"),
153 // TODO : This should NOT be a uint64_t
154 LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_SHARED
, LSR_ACQUIRE
, LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE
, "lck_rw_t", "uint64_t"),
156 LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_EXCL
, LSR_ACQUIRE
, LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE
, "lck_rw_t", "uint64_t"),
157 LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL
, LSR_UPGRADE
, LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE
, "lck_rw_t", "_Bool"),
158 LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL
, LSR_SPIN
, LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN
, "lck_rw_t", "uint64_t"),
159 LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL
, LSR_BLOCK
, LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK
, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
160 LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL_TO_SHARED
, LSR_DOWNGRADE
, LS_LCK_RW_LOCK_EXCL_TO_SHARED_DOWNGRADE
, "lck_rw_t"),
161 //TODO : Separate the probes for the hw_bit from the probe for the normal hw locks
162 LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK
, LSS_ACQUIRE
, LS_LCK_SPIN_LOCK_ACQUIRE
, "hw_lock_t"),
163 LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK
, LSS_SPIN
, LS_LCK_SPIN_LOCK_SPIN
, "hw_lock_t", "uint64_t", "uint64_t"),
164 LOCKSTAT_PROBE(LS_LCK_SPIN_UNLOCK
, LSS_RELEASE
, LS_LCK_SPIN_UNLOCK_RELEASE
, "hw_lock_t"),
166 /* Interlock measurements would be nice, but later */
169 LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL_TO_SHARED
, LSA_ILK_SPIN
, LS_LCK_RW_LOCK_EXCL_TO_SHARED_ILK_SPIN
),
170 LOCKSTAT_PROBE(LS_LCK_MTX_LOCK
, LSA_ILK_SPIN
, LS_LCK_MTX_LOCK_ILK_SPIN
),
171 LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK
, LSA_ILK_SPIN
, LS_LCK_MTX_EXT_LOCK_ILK_SPIN
),
172 LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_EXCL
, LSA_ILK_SPIN
, LS_LCK_RW_TRY_LOCK_EXCL_ILK_SPIN
),
173 LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_SHARED
, LSA_SPIN
, LS_LCK_RW_TRY_LOCK_SHARED_SPIN
)
177 NULL
, NULL
, 0, 0, NULL
181 dtrace_id_t lockstat_probemap
[LS_NPROBES
];
182 void (*lockstat_probe
)(dtrace_id_t
, uint64_t, uint64_t,
183 uint64_t, uint64_t, uint64_t);
185 static dtrace_provider_id_t lockstat_id
;
189 lockstat_enable(void *arg
, dtrace_id_t id
, void *parg
)
191 #pragma unused(arg) /* __APPLE__ */
193 lockstat_probe_t
*probe
= parg
;
195 ASSERT(!lockstat_probemap
[probe
->lsp_probe
]);
197 lockstat_probemap
[probe
->lsp_probe
] = id
;
206 lockstat_disable(void *arg
, dtrace_id_t id
, void *parg
)
208 #pragma unused(arg, id) /* __APPLE__ */
210 lockstat_probe_t
*probe
= parg
;
213 ASSERT(lockstat_probemap
[probe
->lsp_probe
]);
215 lockstat_probemap
[probe
->lsp_probe
] = 0;
219 * See if we have any probes left enabled.
221 for (i
= 0; i
< LS_NPROBES
; i
++) {
222 if (lockstat_probemap
[i
]) {
224 * This probe is still enabled. We don't need to deal
225 * with waiting for all threads to be out of the
226 * lockstat critical sections; just return.
235 lockstat_provide(void *arg
, const dtrace_probedesc_t
*desc
)
237 #pragma unused(arg, desc) /* __APPLE__ */
241 for (i
= 0; lockstat_probes
[i
].lsp_func
!= NULL
; i
++) {
242 lockstat_probe_t
*probe
= &lockstat_probes
[i
];
244 if (dtrace_probe_lookup(lockstat_id
, "mach_kernel",
245 probe
->lsp_func
, probe
->lsp_name
) != 0) {
249 ASSERT(!probe
->lsp_id
);
250 probe
->lsp_id
= dtrace_probe_create(lockstat_id
,
251 "mach_kernel", probe
->lsp_func
, probe
->lsp_name
,
252 LOCKSTAT_AFRAMES
, probe
);
259 lockstat_destroy(void *arg
, dtrace_id_t id
, void *parg
)
261 #pragma unused(arg, id) /* __APPLE__ */
263 lockstat_probe_t
*probe
= parg
;
265 ASSERT(!lockstat_probemap
[probe
->lsp_probe
]);
270 lockstat_getargdesc(void *arg
, dtrace_id_t id
, void *parg
, dtrace_argdesc_t
*desc
)
272 #pragma unused(arg, id)
273 lockstat_probe_t
*probe
= parg
;
274 const char* argdesc
= probe
->lsp_args
;
277 desc
->dtargd_native
[0] = '\0';
278 desc
->dtargd_xlate
[0] = '\0';
280 while (argdesc
[0] != '\0') {
281 if (narg
== desc
->dtargd_ndx
) {
282 strlcpy(desc
->dtargd_native
, argdesc
, DTRACE_ARGTYPELEN
);
285 argdesc
+= strlen(argdesc
) + 1;
289 desc
->dtargd_ndx
= DTRACE_ARGNONE
;
292 static dtrace_pattr_t lockstat_attr
= {
293 { DTRACE_STABILITY_EVOLVING
, DTRACE_STABILITY_EVOLVING
, DTRACE_CLASS_COMMON
},
294 { DTRACE_STABILITY_PRIVATE
, DTRACE_STABILITY_PRIVATE
, DTRACE_CLASS_UNKNOWN
},
295 { DTRACE_STABILITY_PRIVATE
, DTRACE_STABILITY_PRIVATE
, DTRACE_CLASS_UNKNOWN
},
296 { DTRACE_STABILITY_EVOLVING
, DTRACE_STABILITY_EVOLVING
, DTRACE_CLASS_COMMON
},
297 { DTRACE_STABILITY_EVOLVING
, DTRACE_STABILITY_EVOLVING
, DTRACE_CLASS_COMMON
},
300 static dtrace_pops_t lockstat_pops
= {
301 .dtps_provide
= lockstat_provide
,
302 .dtps_provide_module
= NULL
,
303 .dtps_enable
= lockstat_enable
,
304 .dtps_disable
= lockstat_disable
,
305 .dtps_suspend
= NULL
,
307 .dtps_getargdesc
= lockstat_getargdesc
,
308 .dtps_getargval
= NULL
,
309 .dtps_usermode
= NULL
,
310 .dtps_destroy
= lockstat_destroy
314 lockstat_attach(dev_info_t
*devi
)
316 if (ddi_create_minor_node(devi
, "lockstat", S_IFCHR
, 0,
317 DDI_PSEUDO
, 0) == DDI_FAILURE
||
318 dtrace_register("lockstat", &lockstat_attr
, DTRACE_PRIV_KERNEL
,
319 NULL
, &lockstat_pops
, NULL
, &lockstat_id
) != 0) {
320 ddi_remove_minor_node(devi
, NULL
);
324 lockstat_probe
= dtrace_probe
;
330 d_open_t _lockstat_open
;
333 _lockstat_open(dev_t dev
, int flags
, int devtype
, struct proc
*p
)
335 #pragma unused(dev,flags,devtype,p)
339 #define LOCKSTAT_MAJOR -24 /* let the kernel pick the device number */
342 * A struct describing which functions will get invoked for certain
345 static struct cdevsw lockstat_cdevsw
=
347 _lockstat_open
, /* open */
348 eno_opcl
, /* close */
349 eno_rdwrt
, /* read */
350 eno_rdwrt
, /* write */
351 eno_ioctl
, /* ioctl */
352 (stop_fcn_t
*)nulldev
, /* stop */
353 (reset_fcn_t
*)nulldev
, /* reset */
355 eno_select
, /* select */
357 eno_strat
, /* strategy */
363 void lockstat_init( void );
366 lockstat_init( void )
368 int majdevno
= cdevsw_add(LOCKSTAT_MAJOR
, &lockstat_cdevsw
);
371 printf("lockstat_init: failed to allocate a major number!\n");
375 lockstat_attach((dev_info_t
*)(uintptr_t)majdevno
);
377 #undef LOCKSTAT_MAJOR