bsd/dev/dtrace/lockstat.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25
  26 /* #pragma ident        "@(#)lockstat.c 1.11    06/03/24 SMI" */
  27
  28
  29 #ifdef KERNEL
  30 #ifndef _KERNEL
  31 #define _KERNEL /* Solaris vs. Darwin */
  32 #endif
  33 #endif
  34
  35 #include <sys/param.h>
  36 #include <sys/systm.h>
  37 #include <sys/errno.h>
  38 #include <sys/stat.h>
  39 #include <sys/ioctl.h>
  40 #include <sys/conf.h>
  41 #include <sys/fcntl.h>
  42 #include <miscfs/devfs/devfs.h>
  43
  44 #include <sys/dtrace.h>
  45 #include <sys/dtrace_impl.h>
  46
  47 #include <sys/dtrace_glue.h>
  48
  49 #include <sys/lockstat.h>
  50
  51 #include <kern/processor.h>
  52
  53 #define membar_producer dtrace_membar_producer
  54
  55 /*
  56  * Hot patch values, x86
  57  */
  58 #ifdef  __i386__
  59 #define NOP     0x90
  60 #define RET     0xc3
  61 #define LOCKSTAT_AFRAMES 1
  62 #elif   __ppc__
  63 #define NOP     0x60000000
  64 #define RET     0x4e800020      /* blr */
  65 #define LOCKSTAT_AFRAMES 2
  66 #else
  67 #error "not ported to this architecture"
  68 #endif
  69
  70
  71 typedef struct lockstat_probe {
  72         const char      *lsp_func;
  73         const char      *lsp_name;
  74         int             lsp_probe;
  75         dtrace_id_t     lsp_id;
  76 } lockstat_probe_t;
  77
  78 lockstat_probe_t lockstat_probes[] =
  79 {
  80 #ifdef __i386__
  81         /* Not implemented yet on PPC... */
  82         { LS_LCK_MTX_LOCK,      LSA_ACQUIRE,    LS_LCK_MTX_LOCK_ACQUIRE, DTRACE_IDNONE },
  83         { LS_LCK_MTX_LOCK,      LSA_SPIN,       LS_LCK_MTX_LOCK_SPIN, DTRACE_IDNONE },
  84         { LS_LCK_MTX_TRY_LOCK,  LSA_ACQUIRE,    LS_LCK_MTX_TRY_LOCK_ACQUIRE, DTRACE_IDNONE },
  85         { LS_LCK_MTX_TRY_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE, DTRACE_IDNONE },
  86         { LS_LCK_MTX_UNLOCK,    LSA_RELEASE,    LS_LCK_MTX_UNLOCK_RELEASE, DTRACE_IDNONE },
  87         { LS_LCK_MTX_EXT_LOCK,  LSA_ACQUIRE,    LS_LCK_MTX_EXT_LOCK_ACQUIRE, DTRACE_IDNONE },
  88         { LS_LCK_MTX_EXT_LOCK,  LSA_SPIN,       LS_LCK_MTX_EXT_LOCK_SPIN, DTRACE_IDNONE },
  89         { LS_LCK_MTX_EXT_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_EXT_LOCK_ACQUIRE, DTRACE_IDNONE },
  90         { LS_LCK_MTX_UNLOCK,    LSA_RELEASE,    LS_LCK_MTX_EXT_UNLOCK_RELEASE, DTRACE_IDNONE },
  91         { LS_MUTEX_LOCK,        LSA_ACQUIRE,    LS_MUTEX_LOCK_ACQUIRE, DTRACE_IDNONE },
  92         { LS_MUTEX_UNLOCK,      LSA_RELEASE,    LS_MUTEX_UNLOCK_RELEASE, DTRACE_IDNONE },
  93         { LS_MUTEX_TRY_LOCK,    LSA_ACQUIRE,    LS_MUTEX_TRY_LOCK_ACQUIRE, DTRACE_IDNONE },
  94         { LS_MUTEX_TRY_SPIN,    LSA_ACQUIRE,    LS_MUTEX_TRY_SPIN_ACQUIRE, DTRACE_IDNONE },
  95         { LS_MUTEX_LOCK_SPIN,   LSA_ACQUIRE,    LS_MUTEX_LOCK_SPIN_ACQUIRE, DTRACE_IDNONE },
  96 #endif
  97         { LS_LCK_MTX_LOCK,      LSA_BLOCK,      LS_LCK_MTX_LOCK_BLOCK, DTRACE_IDNONE },
  98         { LS_LCK_MTX_EXT_LOCK,  LSA_BLOCK,      LS_LCK_MTX_EXT_LOCK_BLOCK, DTRACE_IDNONE },
  99
 100         { LS_MUTEX_CONVERT_SPIN,        LSA_ACQUIRE,    LS_MUTEX_CONVERT_SPIN_ACQUIRE, DTRACE_IDNONE },
 101
 102         { LS_LCK_RW_LOCK_SHARED,        LSR_ACQUIRE,    LS_LCK_RW_LOCK_SHARED_ACQUIRE, DTRACE_IDNONE },
 103         { LS_LCK_RW_LOCK_SHARED,        LSR_BLOCK,      LS_LCK_RW_LOCK_SHARED_BLOCK, DTRACE_IDNONE },
 104         { LS_LCK_RW_LOCK_SHARED,        LSR_SPIN,       LS_LCK_RW_LOCK_SHARED_SPIN, DTRACE_IDNONE },
 105         { LS_LCK_RW_LOCK_EXCL,          LSR_ACQUIRE,    LS_LCK_RW_LOCK_EXCL_ACQUIRE, DTRACE_IDNONE },
 106         { LS_LCK_RW_LOCK_EXCL,          LSR_BLOCK,      LS_LCK_RW_LOCK_EXCL_BLOCK, DTRACE_IDNONE },
 107         { LS_LCK_RW_LOCK_EXCL,          LSR_SPIN,       LS_LCK_RW_LOCK_EXCL_SPIN, DTRACE_IDNONE },
 108         { LS_LCK_RW_DONE,               LSR_RELEASE,    LS_LCK_RW_DONE_RELEASE, DTRACE_IDNONE },
 109         { LS_LCK_RW_TRY_LOCK_SHARED,    LSR_ACQUIRE,    LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, DTRACE_IDNONE },
 110         { LS_LCK_RW_TRY_LOCK_EXCL,      LSR_ACQUIRE,    LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, DTRACE_IDNONE },
 111         { LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_UPGRADE,   LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, DTRACE_IDNONE },
 112         { LS_LCK_RW_LOCK_SHARED_TO_EXCL,        LSR_BLOCK,      LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK, DTRACE_IDNONE },
 113         { LS_LCK_RW_LOCK_SHARED_TO_EXCL,        LSR_SPIN,       LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN, DTRACE_IDNONE },
 114         { LS_LCK_RW_LOCK_EXCL_TO_SHARED,        LSR_DOWNGRADE,  LS_LCK_RW_LOCK_EXCL_TO_SHARED_DOWNGRADE, DTRACE_IDNONE },
 115
 116
 117 #ifdef  LATER
 118         /* Interlock and spinlock measurements would be nice, but later */
 119         { LS_LCK_SPIN_LOCK,     LSS_ACQUIRE,    LS_LCK_SPIN_LOCK_ACQUIRE, DTRACE_IDNONE },
 120         { LS_LCK_SPIN_LOCK,     LSS_SPIN,       LS_LCK_SPIN_LOCK_SPIN, DTRACE_IDNONE },
 121         { LS_LCK_SPIN_UNLOCK,   LSS_RELEASE,    LS_LCK_SPIN_UNLOCK_RELEASE, DTRACE_IDNONE },
 122
 123         { LS_LCK_RW_LOCK_EXCL_TO_SHARED,        LSA_ILK_SPIN,   LS_LCK_RW_LOCK_EXCL_TO_SHARED_ILK_SPIN, DTRACE_IDNONE },
 124         { LS_LCK_MTX_LOCK,      LSA_ILK_SPIN,   LS_LCK_MTX_LOCK_ILK_SPIN, DTRACE_IDNONE },
 125         { LS_LCK_MTX_EXT_LOCK,  LSA_ILK_SPIN,   LS_LCK_MTX_EXT_LOCK_ILK_SPIN, DTRACE_IDNONE },
 126         { LS_LCK_RW_TRY_LOCK_EXCL,      LSA_ILK_SPIN,   LS_LCK_RW_TRY_LOCK_EXCL_ILK_SPIN, DTRACE_IDNONE },
 127         { LS_LCK_RW_TRY_LOCK_SHARED,    LSA_SPIN,       LS_LCK_RW_TRY_LOCK_SHARED_SPIN, DTRACE_IDNONE },
 128 #endif
 129
 130         { NULL }
 131 };
 132
 133 dtrace_id_t lockstat_probemap[LS_NPROBES];
 134
 135 extern void lck_mtx_lock_lockstat_patch_point();
 136 extern void lck_mtx_try_lock_lockstat_patch_point();
 137 extern void lck_mtx_try_lock_spin_lockstat_patch_point();
 138 extern void lck_mtx_unlock_lockstat_patch_point();
 139 extern void lck_mtx_unlock2_lockstat_patch_point();
 140 extern void mutex_lock_lockstat_patch_point();
 141 extern void mutex_unlock_lockstat_patch_point();
 142 extern void mutex_unlock2_lockstat_patch_point();
 143 extern void mutex_try_lockstat_patch_point();
 144 extern void mutex_try_spin_lockstat_patch_point();
 145 extern void mutex_lock_spin_lockstat_patch_point();
 146 extern void mutex_convert_spin_lockstat_patch_point();
 147 extern void lck_rw_done_lockstat_patch_point();
 148 extern void lck_rw_lock_shared_lockstat_patch_point();
 149 extern void lck_mtx_lock_ext_lockstat_patch_point();
 150 extern void lck_mtx_ext_unlock_lockstat_patch_point();
 151
 152 vm_offset_t *assembly_probes[] = {
 153 #if     defined(__i386__)
 154         /*
 155          * On x86 these points are better done via hot patches, which ensure
 156          * there is zero overhead when not in use.  On x86 these patch points
 157          * are swapped between the return instruction and a no-op, with the
 158          * Dtrace call following the return.
 159          */
 160         (vm_offset_t *) lck_mtx_lock_lockstat_patch_point,
 161         (vm_offset_t *) lck_mtx_try_lock_lockstat_patch_point,
 162         (vm_offset_t *) lck_mtx_try_lock_spin_lockstat_patch_point,
 163         (vm_offset_t *) lck_mtx_unlock_lockstat_patch_point,
 164         (vm_offset_t *) lck_mtx_unlock2_lockstat_patch_point,
 165         (vm_offset_t *) lck_rw_lock_shared_lockstat_patch_point,
 166         (vm_offset_t *) lck_rw_done_lockstat_patch_point,
 167         (vm_offset_t *) lck_mtx_lock_ext_lockstat_patch_point,
 168         (vm_offset_t *) lck_mtx_ext_unlock_lockstat_patch_point,
 169         (vm_offset_t *) mutex_lock_lockstat_patch_point,
 170         (vm_offset_t *) mutex_try_spin_lockstat_patch_point,
 171         (vm_offset_t *) mutex_try_lockstat_patch_point,
 172         (vm_offset_t *) mutex_unlock_lockstat_patch_point,
 173         (vm_offset_t *) mutex_unlock2_lockstat_patch_point,
 174         (vm_offset_t *) mutex_lock_spin_lockstat_patch_point,
 175         (vm_offset_t *) mutex_convert_spin_lockstat_patch_point,
 176 #endif
 177         (vm_offset_t *) lck_mtx_unlock_lockstat_patch_point,
 178         NULL
 179 };
 180 /*
 181  * Hot patch switches back and forth the probe points between NOP and RET.
 182  * The argument indicates whether the probe point is on or off.
 183  */
 184 void lockstat_hot_patch(boolean_t active)
 185 {
 186 #pragma unused(active)
 187         int i;
 188
 189
 190         for (i = 0; assembly_probes[i]; i++) {
 191 #ifdef __i386__
 192                 uint8_t instr;
 193                 instr = (active ? NOP : RET );
 194                 (void) ml_nofault_copy( (vm_offset_t)&instr, *(assembly_probes[i]),
 195                                                                 sizeof(instr));
 196 #endif
 197 #ifdef __ppc__
 198                 uint32_t instr;
 199                 instr = (active ? NOP : RET );
 200                 (void) ml_nofault_copy( (vm_offset_t)&instr, *(assembly_probes[i]), sizeof(instr));
 201 #endif
 202         }
 203 }
 204
 205
 206
 207 void (*lockstat_probe)(dtrace_id_t, uint64_t, uint64_t,
 208                                     uint64_t, uint64_t, uint64_t);
 209
 210 /*
 211  * An initial value for lockstat_probe. See lockstat_attach(). Think safety.
 212  */
 213 static void
 214 lockstat_stub(dtrace_id_t id, uint64_t arg0, uint64_t arg1,
 215                                     uint64_t arg2, uint64_t arg3, uint64_t arg4)
 216 {
 217 #pragma unused(id,arg0,arg1,arg2,arg3,arg4)
 218 }
 219
 220
 221 static dev_info_t       *lockstat_devi; /* saved in xxattach() for xxinfo() */
 222 static dtrace_provider_id_t lockstat_id;
 223
 224 /*ARGSUSED*/
 225 static void
 226 lockstat_enable(void *arg, dtrace_id_t id, void *parg)
 227 {
 228 #pragma unused(arg)
 229         lockstat_probe_t *probe = parg;
 230
 231         ASSERT(!lockstat_probemap[probe->lsp_probe]);
 232
 233         lockstat_probemap[probe->lsp_probe] = id;
 234         membar_producer();
 235
 236         lockstat_probe = dtrace_probe;
 237         membar_producer();
 238
 239         lockstat_hot_patch(TRUE);
 240         membar_producer();
 241
 242 }
 243
 244 /*ARGSUSED*/
 245 static void
 246 lockstat_disable(void *arg, dtrace_id_t id, void *parg)
 247 {
 248 #pragma unused(arg,id)
 249         lockstat_probe_t *probe = parg;
 250         int i;
 251
 252         ASSERT(lockstat_probemap[probe->lsp_probe]);
 253
 254         lockstat_probemap[probe->lsp_probe] = 0;
 255         lockstat_hot_patch(FALSE);
 256         membar_producer();
 257
 258         /*
 259          * See if we have any probes left enabled.
 260          */
 261         for (i = 0; i < LS_NPROBES; i++) {
 262                 if (lockstat_probemap[i]) {
 263                         /*
 264                          * This probe is still enabled.  We don't need to deal
 265                          * with waiting for all threads to be out of the
 266                          * lockstat critical sections; just return.
 267                          */
 268                         return;
 269                 }
 270         }
 271
 272 }
 273
 274 /*ARGSUSED*/
 275 static void
 276 lockstat_provide(void *arg, const dtrace_probedesc_t *desc)
 277 {
 278 #pragma unused(arg,desc)
 279         int i = 0;
 280
 281         for (i = 0; lockstat_probes[i].lsp_func != NULL; i++) {
 282                 lockstat_probe_t *probe = &lockstat_probes[i];
 283
 284                 if (dtrace_probe_lookup(lockstat_id, "mach_kernel",
 285                     probe->lsp_func, probe->lsp_name) != 0)
 286                         continue;
 287
 288                 ASSERT(!probe->lsp_id);
 289                 probe->lsp_id = dtrace_probe_create(lockstat_id,
 290                     "mach_kernel", probe->lsp_func, probe->lsp_name,
 291                     LOCKSTAT_AFRAMES, probe);
 292         }
 293 }
 294
 295
 296 /*ARGSUSED*/
 297 static void
 298 lockstat_destroy(void *arg, dtrace_id_t id, void *parg)
 299 {
 300 #pragma unused(arg,id)
 301         lockstat_probe_t *probe = parg;
 302
 303         ASSERT(!lockstat_probemap[probe->lsp_probe]);
 304         probe->lsp_id = 0;
 305 }
 306
 307 static dtrace_pattr_t lockstat_attr = {
 308 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
 309 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
 310 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
 311 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
 312 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
 313 };
 314
 315 static dtrace_pops_t lockstat_pops = {
 316         lockstat_provide,
 317         NULL,
 318         lockstat_enable,
 319         lockstat_disable,
 320         NULL,
 321         NULL,
 322         NULL,
 323         NULL,
 324         NULL,
 325         lockstat_destroy
 326 };
 327
 328 static int
 329 lockstat_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
 330 {
 331         switch (cmd) {
 332         case DDI_ATTACH:
 333                 break;
 334         case DDI_RESUME:
 335                 return (DDI_SUCCESS);
 336         default:
 337                 return (DDI_FAILURE);
 338         }
 339
 340         if (ddi_create_minor_node(devi, "lockstat", S_IFCHR, 0,
 341             DDI_PSEUDO, 0) == DDI_FAILURE ||
 342             dtrace_register("lockstat", &lockstat_attr, DTRACE_PRIV_KERNEL,
 343             NULL, &lockstat_pops, NULL, &lockstat_id) != 0) {
 344                 ddi_remove_minor_node(devi, NULL);
 345                 return (DDI_FAILURE);
 346         }
 347
 348         ddi_report_dev(devi);
 349         lockstat_devi = devi;
 350         lockstat_probe = lockstat_stub;
 351         return (DDI_SUCCESS);
 352 }
 353
 354 d_open_t _lockstat_open;
 355
 356 int _lockstat_open(dev_t dev, int flags, int devtype, struct proc *p)
 357 {
 358 #pragma unused(dev,flags,devtype,p)
 359         return 0;
 360 }
 361
 362 #define LOCKSTAT_MAJOR  -24 /* let the kernel pick the device number */
 363
 364 /*
 365  * A struct describing which functions will get invoked for certain
 366  * actions.
 367  */
 368 static struct cdevsw lockstat_cdevsw =
 369 {
 370         _lockstat_open,         /* open */
 371         eno_opcl,                       /* close */
 372         eno_rdwrt,                      /* read */
 373         eno_rdwrt,                      /* write */
 374         eno_ioctl,                      /* ioctl */
 375         (stop_fcn_t *)nulldev, /* stop */
 376         (reset_fcn_t *)nulldev, /* reset */
 377         NULL,                           /* tty's */
 378         eno_select,                     /* select */
 379         eno_mmap,                       /* mmap */
 380         eno_strat,                      /* strategy */
 381         eno_getc,                       /* getc */
 382         eno_putc,                       /* putc */
 383         0                                       /* type */
 384 };
 385
 386 static int gLockstatInited = 0;
 387
 388 void lockstat_init( void );
 389
 390 void lockstat_init( void )
 391 {
 392         if (0 == gLockstatInited)
 393         {
 394                 int majdevno = cdevsw_add(LOCKSTAT_MAJOR, &lockstat_cdevsw);
 395
 396                 if (majdevno < 0) {
 397                         printf("lockstat_init: failed to allocate a major number!\n");
 398                         gLockstatInited = 0;
 399                         return;
 400                 }
 401
 402                 lockstat_attach( (dev_info_t    *)majdevno, DDI_ATTACH );
 403                 gLockstatInited = 1;
 404         } else
 405                 panic("lockstat_init: called twice!\n");
 406 }
 407 #undef LOCKSTAT_MAJOR