bsd/dev/dtrace/lockstat.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25
  26 /* #pragma ident        "@(#)lockstat.c 1.12    08/01/16 SMI" */
  27
  28
  29 #ifdef KERNEL
  30 #ifndef _KERNEL
  31 #define _KERNEL /* Solaris vs. Darwin */
  32 #endif
  33 #endif
  34
  35 #include <sys/param.h>
  36 #include <sys/systm.h>
  37 #include <sys/errno.h>
  38 #include <sys/stat.h>
  39 #include <sys/ioctl.h>
  40 #include <sys/conf.h>
  41 #include <sys/fcntl.h>
  42 #include <miscfs/devfs/devfs.h>
  43
  44 #include <sys/dtrace.h>
  45 #include <sys/dtrace_impl.h>
  46
  47 #include <sys/dtrace_glue.h>
  48
  49 #include <sys/lockstat.h>
  50
  51 #include <kern/processor.h>
  52
  53 #define membar_producer dtrace_membar_producer
  54
  55 #define PROBE_ARGS0(a, b, c, d, e) "\000"
  56 #define PROBE_ARGS1(a, b, c, d, e) a "\000"
  57 #define PROBE_ARGS2(a, b, c, d, e) a "\000" b "\000"
  58 #define PROBE_ARGS3(a, b, c, d, e) a "\000" b "\000" c "\000"
  59 #define PROBE_ARGS4(a, b, c, d, e) a "\000" b "\000" c "\000" d "\000"
  60 #define PROBE_ARGS5(a, b, c, d, e) a "\000" b "\000" c "\000" d "\000" e "\000"
  61 #define PROBE_ARGS_(a, b, c, d, e, n, ...) PROBE_ARGS##n(a, b, c, d, e)
  62 #define PROBE_ARGS(...) PROBE_ARGS_(__VA_ARGS__, 5, 4, 3, 2, 1, 0)
  63
  64 #define LOCKSTAT_PROBE(func, name, probe, ...) \
  65         {func, name, probe, DTRACE_IDNONE, PROBE_ARGS(__VA_ARGS__)}
  66
  67 /*
  68  * Hot patch values, x86
  69  */
  70 #if defined(__x86_64__)
  71 #define NOP     0x90
  72 #define RET     0xc3
  73 #define LOCKSTAT_AFRAMES 1
  74 #elif   defined(__arm__)
  75 #define NOP     0xE1A00000
  76 #define BXLR    0xE12FFF1E
  77 #define LOCKSTAT_AFRAMES 2
  78 #elif   defined(__arm64__)
  79 #define NOP     0xD503201F
  80 #define RET     0xD65f03c0
  81 #define LOCKSTAT_AFRAMES 2
  82 #else
  83 #error "not ported to this architecture"
  84 #endif
  85
  86 typedef struct lockstat_probe {
  87         const char      *lsp_func;
  88         const char      *lsp_name;
  89         int             lsp_probe;
  90         dtrace_id_t     lsp_id;
  91         const char      *lsp_args;
  92 } lockstat_probe_t;
  93
  94 lockstat_probe_t lockstat_probes[] =
  95 {
  96 #if defined(__x86_64__)
  97         /* Only provide implemented probes for each architecture  */
  98         LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_ACQUIRE, "lck_mtx_t"),
  99         LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_SPIN, LS_LCK_MTX_LOCK_SPIN, "lck_mtx_t", "uint64_t"),
 100         LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_BLOCK, LS_LCK_MTX_LOCK_BLOCK, "lck_mtx_t", "uint64_t"),
 101         LOCKSTAT_PROBE(LS_LCK_MTX_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_LOCK_ACQUIRE, "lck_mtx_t"),
 102         LOCKSTAT_PROBE(LS_LCK_MTX_TRY_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE, "lck_mtx_t"),
 103         LOCKSTAT_PROBE(LS_LCK_MTX_UNLOCK, LSA_RELEASE, LS_LCK_MTX_UNLOCK_RELEASE, "lck_mtx_t"),
 104         LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_ACQUIRE, LS_LCK_MTX_EXT_LOCK_ACQUIRE, "lck_mtx_t"),
 105         LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_SPIN, LS_LCK_MTX_EXT_LOCK_SPIN, "lck_mtx_t", "uint64_t"),
 106         LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_BLOCK, LS_LCK_MTX_EXT_LOCK_BLOCK, "lck_mtx_t", "uint64_t"),
 107 //      LOCKSTAT_PROBE(LS_LCK_MTX_EXT_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_EXT_LOCK_ACQUIRE)
 108         LOCKSTAT_PROBE(LS_LCK_MTX_EXT_UNLOCK, LSA_RELEASE, LS_LCK_MTX_EXT_UNLOCK_RELEASE, "lck_mtx_t"),
 109         LOCKSTAT_PROBE(LS_LCK_MTX_LOCK_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_SPIN_ACQUIRE, "lck_mtx_t"),
 110         // TODO: This should not be a uint64_t !
 111         LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_LOCK_SHARED_ACQUIRE, "lck_rw_t", "uint64_t"),
 112         LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
 113         LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_SPIN, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
 114         // TODO: This should NOT be a uint64_t
 115         LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_LOCK_EXCL_ACQUIRE, "lck_rw_t", "uint64_t"),
 116         LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_EXCL_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
 117         LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_EXCL_SPIN, "lck_rw_t", "uint64_t", "int"),
 118         LOCKSTAT_PROBE(LS_LCK_RW_DONE, LSR_RELEASE, LS_LCK_RW_DONE_RELEASE, "lck_rw_t", "_Bool"),
 119         // TODO : This should NOT be a uint64_t
 120         LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, "lck_rw_t", "uint64_t"),
 121         // See above
 122         LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, "lck_rw_t", "uint64_t"),
 123         LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_UPGRADE, LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, "lck_rw_t", "_Bool"),
 124         LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN, "lck_rw_t", "uint64_t"),
 125         LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
 126         LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL_TO_SHARED, LSR_DOWNGRADE, LS_LCK_RW_LOCK_EXCL_TO_SHARED_DOWNGRADE, "lck_rw_t"),
 127         //TODO : Separate the probes for the hw_bit from the probe for the normal hw locks
 128         LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK, LSS_ACQUIRE, LS_LCK_SPIN_LOCK_ACQUIRE, "hw_lock_t"),
 129         LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK, LSS_SPIN, LS_LCK_SPIN_LOCK_SPIN, "hw_lock_t", "uint64_t", "uint64_t"),
 130         LOCKSTAT_PROBE(LS_LCK_SPIN_UNLOCK, LSS_RELEASE, LS_LCK_SPIN_UNLOCK_RELEASE, "hw_lock_t"),
 131 #elif defined(__arm__) || defined(__arm64__)
 132         LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_ACQUIRE, "lck_mtx_t"),
 133 //      LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_SPIN, LS_LCK_MTX_LOCK_SPIN, "lck_mtx_t", "uint64_t"),
 134         LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_BLOCK, LS_LCK_MTX_LOCK_BLOCK, "lck_mtx_t", "uint64_t"),
 135         LOCKSTAT_PROBE(LS_LCK_MTX_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_LOCK_ACQUIRE, "lck_mtx_t"),
 136 //      LOCKSTAT_PROBE(LS_LCK_MTX_TRY_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE, "lck_mtx_t"),
 137         LOCKSTAT_PROBE(LS_LCK_MTX_UNLOCK, LSA_RELEASE, LS_LCK_MTX_UNLOCK_RELEASE, "lck_mtx_t"),
 138         LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_ACQUIRE, LS_LCK_MTX_EXT_LOCK_ACQUIRE, "lck_mtx_t"),
 139 //      LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_SPIN, LS_LCK_MTX_EXT_LOCK_SPIN, "lck_mtx_t", "uint64_t"),
 140         LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_BLOCK, LS_LCK_MTX_EXT_LOCK_BLOCK, "lck_mtx_t", "uint64_t"),
 141 //      LOCKSTAT_PROBE(LS_LCK_MTX_EXT_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_EXT_LOCK_ACQUIRE)
 142 //      LOCKSTAT_PROBE(LS_LCK_MTX_EXT_UNLOCK, LSA_RELEASE, LS_LCK_MTX_EXT_UNLOCK_RELEASE, "lck_mtx_t"),
 143 //      LOCKSTAT_PROBE(LS_LCK_MTX_LOCK_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_SPIN_ACQUIRE, "lck_mtx_t"),
 144         LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_LOCK_SHARED_ACQUIRE, "lck_rw_t", "uint64_t"),
 145         LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
 146         LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_SPIN, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
 147         LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_LOCK_EXCL_ACQUIRE, "lck_rw_t", "uint64_t"),
 148         LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_EXCL_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
 149         LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_EXCL_SPIN, "lck_rw_t", "uint64_t", "int"),
 150         LOCKSTAT_PROBE(LS_LCK_RW_DONE, LSR_RELEASE, LS_LCK_RW_DONE_RELEASE, "lck_rw_t", "_Bool"),
 151         // TODO : This should NOT be a uint64_t
 152         LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, "lck_rw_t", "uint64_t"),
 153         // See above
 154         LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, "lck_rw_t", "uint64_t"),
 155         LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_UPGRADE, LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, "lck_rw_t", "_Bool"),
 156         LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN, "lck_rw_t", "uint64_t"),
 157         LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
 158         LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL_TO_SHARED, LSR_DOWNGRADE, LS_LCK_RW_LOCK_EXCL_TO_SHARED_DOWNGRADE, "lck_rw_t"),
 159         //TODO : Separate the probes for the hw_bit from the probe for the normal hw locks
 160         LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK, LSS_ACQUIRE, LS_LCK_SPIN_LOCK_ACQUIRE, "hw_lock_t"),
 161         LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK, LSS_SPIN, LS_LCK_SPIN_LOCK_SPIN, "hw_lock_t", "uint64_t", "uint64_t"),
 162         LOCKSTAT_PROBE(LS_LCK_SPIN_UNLOCK, LSS_RELEASE, LS_LCK_SPIN_UNLOCK_RELEASE, "hw_lock_t"),
 163 #endif
 164         /* Interlock measurements would be nice, but later */
 165
 166 #ifdef  LATER
 167         LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL_TO_SHARED, LSA_ILK_SPIN, LS_LCK_RW_LOCK_EXCL_TO_SHARED_ILK_SPIN),
 168         LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_ILK_SPIN, LS_LCK_MTX_LOCK_ILK_SPIN),
 169         LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_ILK_SPIN, LS_LCK_MTX_EXT_LOCK_ILK_SPIN),
 170         LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_EXCL, LSA_ILK_SPIN, LS_LCK_RW_TRY_LOCK_EXCL_ILK_SPIN),
 171         LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_SHARED, LSA_SPIN, LS_LCK_RW_TRY_LOCK_SHARED_SPIN)
 172 #endif
 173
 174         { NULL, NULL, 0, 0, NULL}
 175 };
 176
 177 dtrace_id_t lockstat_probemap[LS_NPROBES];
 178
 179 #if CONFIG_DTRACE
 180 #if defined(__x86_64__)
 181 extern void lck_mtx_lock_lockstat_patch_point(void);
 182 extern void lck_mtx_try_lock_lockstat_patch_point(void);
 183 extern void lck_mtx_try_lock_spin_lockstat_patch_point(void);
 184 extern void lck_mtx_unlock_lockstat_patch_point(void);
 185 extern void lck_mtx_lock_ext_lockstat_patch_point(void);
 186 extern void lck_mtx_ext_unlock_lockstat_patch_point(void);
 187 extern void lck_mtx_lock_spin_lockstat_patch_point(void);
 188 #endif
 189 #if defined (__arm__)
 190
 191 #endif
 192
 193 #if defined (__arm64__)
 194
 195 #endif
 196 #endif /* CONFIG_DTRACE */
 197
 198 typedef struct lockstat_assembly_probe {
 199         int lsap_probe;
 200         vm_offset_t * lsap_patch_point;
 201 } lockstat_assembly_probe_t;
 202
 203
 204         lockstat_assembly_probe_t assembly_probes[] =
 205         {
 206 #if CONFIG_DTRACE
 207 #if defined(__x86_64__)
 208                 /*
 209                  * On x86 these points are better done via hot patches, which ensure
 210                  * there is zero overhead when not in use.  On x86 these patch points
 211                  * are swapped between the return instruction and a no-op, with the
 212                  * Dtrace call following the return.
 213                  */
 214                 { LS_LCK_MTX_LOCK_ACQUIRE,              (vm_offset_t *) lck_mtx_lock_lockstat_patch_point },
 215                 { LS_LCK_MTX_TRY_LOCK_ACQUIRE,          (vm_offset_t *) lck_mtx_try_lock_lockstat_patch_point },
 216                 { LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE,     (vm_offset_t *) lck_mtx_try_lock_spin_lockstat_patch_point },
 217                 { LS_LCK_MTX_UNLOCK_RELEASE,            (vm_offset_t *) lck_mtx_unlock_lockstat_patch_point },
 218                 { LS_LCK_MTX_EXT_LOCK_ACQUIRE,          (vm_offset_t *) lck_mtx_lock_ext_lockstat_patch_point },
 219                 { LS_LCK_MTX_EXT_UNLOCK_RELEASE,        (vm_offset_t *) lck_mtx_ext_unlock_lockstat_patch_point },
 220                 { LS_LCK_MTX_LOCK_SPIN_ACQUIRE,         (vm_offset_t *) lck_mtx_lock_spin_lockstat_patch_point },
 221 #endif
 222                 /* No assembly patch points for ARM */
 223 #endif /* CONFIG_DTRACE */
 224                 { LS_LCK_INVALID, NULL }
 225 };
 226
 227
 228 /*
 229  * APPLE NOTE:
 230  * Hot patch is used to manipulate probe points by swapping between
 231  * no-op and return instructions.
 232  * The active flag indicates whether the probe point will turn on or off.
 233  *      on == plant a NOP and thus fall through to the probe call
 234  *     off == plant a RET and thus avoid the probe call completely
 235  * The ls_probe identifies which probe we will patch.
 236  */
 237 static
 238 void lockstat_hot_patch(boolean_t active, int ls_probe)
 239 {
 240 #pragma unused(active)
 241         int i;
 242
 243         /*
 244          * Loop through entire table, in case there are
 245          * multiple patch points per probe.
 246          */
 247         for (i = 0; assembly_probes[i].lsap_patch_point; i++) {
 248                 if (ls_probe == assembly_probes[i].lsap_probe)
 249 #if defined(__x86_64__)
 250                 {
 251                         uint8_t instr;
 252                         instr = (active ? NOP : RET );
 253                         (void) ml_nofault_copy( (vm_offset_t)&instr, *(assembly_probes[i].lsap_patch_point),
 254                                                                 sizeof(instr));
 255                 }
 256 #elif defined (__arm__)
 257                 {
 258                         uint32_t instr;
 259                         instr = (active ? NOP : BXLR );
 260                         (void) ml_nofault_copy( (vm_offset_t)&instr, *(assembly_probes[i].lsap_patch_point),
 261                                                                 sizeof(instr));
 262                 }
 263 #elif defined (__arm64__)
 264                 {
 265                         uint32_t instr;
 266                         instr = (active ? NOP : RET );
 267                         (void) ml_nofault_copy( (vm_offset_t)&instr, *(assembly_probes[i].lsap_patch_point),
 268                                                                 sizeof(instr));
 269                 }
 270 #endif
 271         } /* for */
 272 }
 273
 274 void (*lockstat_probe)(dtrace_id_t, uint64_t, uint64_t,
 275                                     uint64_t, uint64_t, uint64_t);
 276
 277
 278 /*
 279  * APPLE NOTE:
 280  * This wrapper is used only by assembler hot patched probes.
 281  */
 282 void
 283 lockstat_probe_wrapper(int probe, uintptr_t lp, int rwflag)
 284 {
 285         dtrace_id_t id;
 286         id = lockstat_probemap[probe];
 287         if (id != 0)
 288         {
 289                 (*lockstat_probe)(id, (uintptr_t)lp, (uint64_t)rwflag, 0,0,0);
 290         }
 291 }
 292
 293 static dev_info_t       *lockstat_devi; /* saved in xxattach() for xxinfo() */
 294 static dtrace_provider_id_t lockstat_id;
 295
 296 /*ARGSUSED*/
 297 static int
 298 lockstat_enable(void *arg, dtrace_id_t id, void *parg)
 299 {
 300 #pragma unused(arg) /* __APPLE__ */
 301
 302         lockstat_probe_t *probe = parg;
 303
 304         ASSERT(!lockstat_probemap[probe->lsp_probe]);
 305
 306         lockstat_probemap[probe->lsp_probe] = id;
 307         membar_producer();
 308
 309         lockstat_hot_patch(TRUE, probe->lsp_probe);
 310         membar_producer();
 311         return(0);
 312
 313 }
 314
 315 /*ARGSUSED*/
 316 static void
 317 lockstat_disable(void *arg, dtrace_id_t id, void *parg)
 318 {
 319 #pragma unused(arg, id) /* __APPLE__ */
 320
 321         lockstat_probe_t *probe = parg;
 322         int i;
 323
 324         ASSERT(lockstat_probemap[probe->lsp_probe]);
 325
 326         lockstat_probemap[probe->lsp_probe] = 0;
 327         lockstat_hot_patch(FALSE, probe->lsp_probe);
 328         membar_producer();
 329
 330         /*
 331          * See if we have any probes left enabled.
 332          */
 333         for (i = 0; i < LS_NPROBES; i++) {
 334                 if (lockstat_probemap[i]) {
 335                         /*
 336                          * This probe is still enabled.  We don't need to deal
 337                          * with waiting for all threads to be out of the
 338                          * lockstat critical sections; just return.
 339                          */
 340                         return;
 341                 }
 342         }
 343
 344 }
 345
 346 /*ARGSUSED*/
 347 static void
 348 lockstat_provide(void *arg, const dtrace_probedesc_t *desc)
 349 {
 350 #pragma unused(arg, desc) /* __APPLE__ */
 351
 352         int i = 0;
 353
 354         for (i = 0; lockstat_probes[i].lsp_func != NULL; i++) {
 355                 lockstat_probe_t *probe = &lockstat_probes[i];
 356
 357                 if (dtrace_probe_lookup(lockstat_id, "mach_kernel",
 358                     probe->lsp_func, probe->lsp_name) != 0)
 359                         continue;
 360
 361                 ASSERT(!probe->lsp_id);
 362                 probe->lsp_id = dtrace_probe_create(lockstat_id,
 363                     "mach_kernel", probe->lsp_func, probe->lsp_name,
 364                     LOCKSTAT_AFRAMES, probe);
 365         }
 366 }
 367
 368
 369 /*ARGSUSED*/
 370 static void
 371 lockstat_destroy(void *arg, dtrace_id_t id, void *parg)
 372 {
 373 #pragma unused(arg, id) /* __APPLE__ */
 374
 375         lockstat_probe_t *probe = parg;
 376
 377         ASSERT(!lockstat_probemap[probe->lsp_probe]);
 378         probe->lsp_id = 0;
 379 }
 380
 381 static void
 382 lockstat_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc)
 383 {
 384 #pragma unused(arg, id)
 385         lockstat_probe_t *probe = parg;
 386         const char* argdesc = probe->lsp_args;
 387         int narg = 0;
 388
 389         desc->dtargd_native[0] = '\0';
 390         desc->dtargd_xlate[0] = '\0';
 391
 392         while(argdesc[0] != '\0') {
 393                 if (narg == desc->dtargd_ndx) {
 394                         strlcpy(desc->dtargd_native, argdesc, DTRACE_ARGTYPELEN);
 395                         return;
 396                 }
 397                 argdesc += strlen(argdesc) + 1;
 398                 narg++;
 399         }
 400
 401         desc->dtargd_ndx = DTRACE_ARGNONE;
 402 }
 403
 404 static dtrace_pattr_t lockstat_attr = {
 405 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
 406 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
 407 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
 408 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
 409 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
 410 };
 411
 412 static dtrace_pops_t lockstat_pops = {
 413         lockstat_provide,
 414         NULL,
 415         lockstat_enable,
 416         lockstat_disable,
 417         NULL,
 418         NULL,
 419         lockstat_getargdesc,
 420         NULL,
 421         NULL,
 422         lockstat_destroy
 423 };
 424
 425 static int
 426 lockstat_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
 427 {
 428         switch (cmd) {
 429         case DDI_ATTACH:
 430                 break;
 431         case DDI_RESUME:
 432                 return (DDI_SUCCESS);
 433         default:
 434                 return (DDI_FAILURE);
 435         }
 436
 437         if (ddi_create_minor_node(devi, "lockstat", S_IFCHR, 0,
 438             DDI_PSEUDO, 0) == DDI_FAILURE ||
 439             dtrace_register("lockstat", &lockstat_attr, DTRACE_PRIV_KERNEL,
 440             NULL, &lockstat_pops, NULL, &lockstat_id) != 0) {
 441                 ddi_remove_minor_node(devi, NULL);
 442                 return (DDI_FAILURE);
 443         }
 444
 445         lockstat_probe = dtrace_probe;
 446         membar_producer();
 447
 448         ddi_report_dev(devi);
 449         lockstat_devi = devi;
 450         return (DDI_SUCCESS);
 451 }
 452
 453 d_open_t _lockstat_open;
 454
 455 int _lockstat_open(dev_t dev, int flags, int devtype, struct proc *p)
 456 {
 457 #pragma unused(dev,flags,devtype,p)
 458         return 0;
 459 }
 460
 461 #define LOCKSTAT_MAJOR  -24 /* let the kernel pick the device number */
 462
 463 /*
 464  * A struct describing which functions will get invoked for certain
 465  * actions.
 466  */
 467 static struct cdevsw lockstat_cdevsw =
 468 {
 469         _lockstat_open,         /* open */
 470         eno_opcl,                       /* close */
 471         eno_rdwrt,                      /* read */
 472         eno_rdwrt,                      /* write */
 473         eno_ioctl,                      /* ioctl */
 474         (stop_fcn_t *)nulldev, /* stop */
 475         (reset_fcn_t *)nulldev, /* reset */
 476         NULL,                           /* tty's */
 477         eno_select,                     /* select */
 478         eno_mmap,                       /* mmap */
 479         eno_strat,                      /* strategy */
 480         eno_getc,                       /* getc */
 481         eno_putc,                       /* putc */
 482         0                                       /* type */
 483 };
 484
 485 static int gLockstatInited = 0;
 486
 487 void lockstat_init( void );
 488
 489 void lockstat_init( void )
 490 {
 491         if (0 == gLockstatInited)
 492         {
 493                 int majdevno = cdevsw_add(LOCKSTAT_MAJOR, &lockstat_cdevsw);
 494
 495                 if (majdevno < 0) {
 496                         printf("lockstat_init: failed to allocate a major number!\n");
 497                         gLockstatInited = 0;
 498                         return;
 499                 }
 500
 501                 lockstat_attach( (dev_info_t    *)(uintptr_t)majdevno, DDI_ATTACH );
 502                 gLockstatInited = 1;
 503         } else
 504                 panic("lockstat_init: called twice!\n");
 505 }
 506 #undef LOCKSTAT_MAJOR