bsd/dev/dtrace/lockprof.c

   1 /*
   2  * Copyright (c) 2019 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 #include <sys/dtrace.h>
  29 #include <sys/dtrace_impl.h>
  30 #include <kern/lock_group.h>
  31 #include <kern/lock_stat.h>
  32
  33 #if LOCK_STATS
  34
  35 #define SPIN_HELD 0
  36 #define SPIN_MISS 1
  37 #define SPIN_SPIN 2
  38
  39 #define SPIN_HELD_PREFIX "spin-held-"
  40 #define SPIN_MISS_PREFIX "spin-miss-"
  41 #define SPIN_SPIN_PREFIX "spin-spin-"
  42
  43 #define LOCKGROUPSTAT_AFRAMES 1
  44 #define LOCKGROUPSTAT_LEN 64
  45
  46 static dtrace_provider_id_t lockprof_id;
  47
  48 decl_lck_mtx_data(extern, lck_grp_lock);
  49 extern queue_head_t lck_grp_queue;
  50 extern unsigned int lck_grp_cnt;
  51
  52 #define LOCKPROF_MAX 10000 /* maximum number of lockprof probes */
  53 static uint32_t lockprof_count; /* current number of lockprof probes */
  54
  55 static const struct {
  56         int kind;
  57         const char *prefix;
  58         bool time_event;
  59 } events[] = {
  60         {SPIN_HELD, SPIN_HELD_PREFIX, false},
  61         {SPIN_MISS, SPIN_MISS_PREFIX, false},
  62         {SPIN_SPIN, SPIN_SPIN_PREFIX, true},
  63         {0, NULL, false}
  64 };
  65
  66 const static int hold_defaults[] = {
  67         100, 1000
  68 };
  69
  70 const static struct {
  71         unsigned int time;
  72         const char *suffix;
  73         uint64_t mult;
  74 } cont_defaults[] = {
  75         {100, "ms", NANOSEC / MILLISEC}
  76 };
  77
  78 typedef struct lockprof_probe {
  79         int lockprof_kind;
  80         dtrace_id_t lockprof_id;
  81         uint64_t lockprof_limit;
  82         lck_grp_t *lockprof_grp;
  83 } lockprof_probe_t;
  84
  85 void
  86 lockprof_invoke(lck_grp_t *grp, lck_grp_stat_t *stat, uint64_t val)
  87 {
  88         dtrace_probe(stat->lgs_probeid, (uintptr_t)grp, val, 0, 0, 0);
  89 }
  90
  91 static void
  92 probe_create(int kind, const char *suffix, const char *grp_name, uint64_t count, uint64_t mult)
  93 {
  94         char name[LOCKGROUPSTAT_LEN];
  95         lck_mtx_lock(&lck_grp_lock);
  96         lck_grp_t *grp = (lck_grp_t*)queue_first(&lck_grp_queue);
  97         uint64_t limit = count * mult;
  98
  99         if (events[kind].time_event) {
 100                 nanoseconds_to_absolutetime(limit, &limit);
 101         }
 102
 103         for (unsigned int i = 0; i < lck_grp_cnt; i++, grp = (lck_grp_t*)queue_next((queue_entry_t)grp)) {
 104                 if (!grp_name || grp_name[0] == '\0' || strcmp(grp_name, grp->lck_grp_name) == 0) {
 105                         snprintf(name, sizeof(name), "%s%llu%s", events[kind].prefix, count, suffix ?: "");
 106
 107                         if (dtrace_probe_lookup(lockprof_id, grp->lck_grp_name, NULL, name) != 0) {
 108                                 continue;
 109                         }
 110                         if (lockprof_count >= LOCKPROF_MAX) {
 111                                 break;
 112                         }
 113
 114                         lockprof_probe_t *probe = kmem_zalloc(sizeof(lockprof_probe_t), KM_SLEEP);
 115                         probe->lockprof_kind = kind;
 116                         probe->lockprof_limit = limit;
 117                         probe->lockprof_grp = grp;
 118
 119                         probe->lockprof_id = dtrace_probe_create(lockprof_id, grp->lck_grp_name, NULL, name,
 120                             LOCKGROUPSTAT_AFRAMES, probe);
 121
 122                         lockprof_count++;
 123                 }
 124         }
 125         lck_mtx_unlock(&lck_grp_lock);
 126 }
 127
 128 static void
 129 lockprof_provide(void *arg, const dtrace_probedesc_t *desc)
 130 {
 131 #pragma unused(arg)
 132         size_t event_id, i, len;
 133
 134         if (desc == NULL) {
 135                 for (i = 0; i < sizeof(hold_defaults) / sizeof(hold_defaults[0]); i++) {
 136                         probe_create(SPIN_HELD, NULL, NULL, hold_defaults[i], 1);
 137                         probe_create(SPIN_MISS, NULL, NULL, hold_defaults[i], 1);
 138                 }
 139                 for (i = 0; i < sizeof(cont_defaults) / sizeof(cont_defaults[0]); i++) {
 140                         probe_create(SPIN_SPIN, cont_defaults[i].suffix, NULL, cont_defaults[i].time, cont_defaults[i].mult);
 141                 }
 142                 return;
 143         }
 144
 145         const char *name, *suffix = NULL;
 146         hrtime_t val = 0, mult = 1;
 147
 148         const struct {
 149                 const char *name;
 150                 hrtime_t mult;
 151         } suffixes[] = {
 152                 { "us", NANOSEC / MICROSEC },
 153                 { "usec", NANOSEC / MICROSEC },
 154                 { "ms", NANOSEC / MILLISEC },
 155                 { "msec", NANOSEC / MILLISEC },
 156                 { "s", NANOSEC / SEC },
 157                 { "sec", NANOSEC / SEC },
 158                 { NULL, 0 }
 159         };
 160
 161         name = desc->dtpd_name;
 162
 163         for (event_id = 0; events[event_id].prefix != NULL; event_id++) {
 164                 len = strlen(events[event_id].prefix);
 165
 166                 if (strncmp(name, events[event_id].prefix, len) != 0) {
 167                         continue;
 168                 }
 169                 break;
 170         }
 171
 172         if (events[event_id].prefix == NULL) {
 173                 return;
 174         }
 175
 176
 177         /*
 178          * We need to start before any time suffix.
 179          */
 180         for (i = strlen(name); i >= len; i--) {
 181                 if (name[i] >= '0' && name[i] <= '9') {
 182                         break;
 183                 }
 184                 suffix = &name[i];
 185         }
 186
 187         /*
 188          * Now determine the numerical value present in the probe name.
 189          */
 190         for (uint64_t m = 1; i >= len; i--) {
 191                 if (name[i] < '0' || name[i] > '9') {
 192                         return;
 193                 }
 194
 195                 val += (name[i] - '0') * m;
 196                 m *= (hrtime_t)10;
 197         }
 198
 199         if (val == 0) {
 200                 return;
 201         }
 202
 203         if (events[event_id].time_event) {
 204                 for (i = 0, mult = 0; suffixes[i].name != NULL; i++) {
 205                         if (strncasecmp(suffixes[i].name, suffix, strlen(suffixes[i].name) + 1) == 0) {
 206                                 mult = suffixes[i].mult;
 207                                 break;
 208                         }
 209                 }
 210                 if (suffixes[i].name == NULL) {
 211                         return;
 212                 }
 213         } else if (*suffix != '\0') {
 214                 return;
 215         }
 216
 217         probe_create(events[event_id].kind, suffix, desc->dtpd_mod, val, mult);
 218 }
 219
 220
 221 static lck_grp_stat_t*
 222 lockprof_stat(lck_grp_t *grp, int kind)
 223 {
 224         switch (kind) {
 225         case SPIN_HELD:
 226                 return &grp->lck_grp_stats.lgss_spin_held;
 227         case SPIN_MISS:
 228                 return &grp->lck_grp_stats.lgss_spin_miss;
 229         case SPIN_SPIN:
 230                 return &grp->lck_grp_stats.lgss_spin_spin;
 231         default:
 232                 return NULL;
 233         }
 234 }
 235
 236 static int
 237 lockprof_enable(void *arg, dtrace_id_t id, void *parg)
 238 {
 239 #pragma unused(arg, id, parg)
 240         lockprof_probe_t *probe = (lockprof_probe_t*)parg;
 241         lck_grp_t *grp = probe->lockprof_grp;
 242         lck_grp_stat_t *stat;
 243
 244         if (grp == NULL) {
 245                 return -1;
 246         }
 247
 248         if ((stat = lockprof_stat(grp, probe->lockprof_kind)) == NULL) {
 249                 return -1;
 250         }
 251
 252         /*
 253          * lockprof_enable/disable are called with
 254          * dtrace_lock held
 255          */
 256         if (stat->lgs_limit != 0) {
 257                 return -1;
 258         }
 259
 260         stat->lgs_limit = probe->lockprof_limit;
 261         stat->lgs_enablings++;
 262         stat->lgs_probeid = probe->lockprof_id;
 263
 264         return 0;
 265 }
 266
 267 static void
 268 lockprof_disable(void *arg, dtrace_id_t id, void *parg)
 269 {
 270 #pragma unused(arg, id)
 271         lockprof_probe_t *probe = (lockprof_probe_t*)parg;
 272         lck_grp_t *grp = probe->lockprof_grp;
 273         lck_grp_stat_t *stat;
 274
 275         if (grp == NULL) {
 276                 return;
 277         }
 278
 279         if ((stat = lockprof_stat(grp, probe->lockprof_kind)) == NULL) {
 280                 return;
 281         }
 282
 283         if (stat->lgs_limit == 0 || stat->lgs_enablings == 0) {
 284                 return;
 285         }
 286
 287         stat->lgs_limit = 0;
 288         stat->lgs_enablings--;
 289         stat->lgs_probeid = 0;
 290 }
 291
 292 static void
 293 lockprof_destroy(void *arg, dtrace_id_t id, void *parg)
 294 {
 295 #pragma unused(arg, id)
 296         lockprof_probe_t *probe = (lockprof_probe_t*)parg;
 297         kmem_free(probe, sizeof(lockprof_probe_t));
 298         lockprof_count--;
 299 }
 300
 301 static void
 302 lockprof_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc)
 303 {
 304 #pragma unused(arg, id, parg)
 305         const char *argdesc = NULL;
 306         switch (desc->dtargd_ndx) {
 307         case 0:
 308                 argdesc = "lck_grp_t*";
 309                 break;
 310         case 1:
 311                 argdesc = "uint64_t";
 312                 break;
 313         }
 314
 315         if (argdesc) {
 316                 strlcpy(desc->dtargd_native, argdesc, DTRACE_ARGTYPELEN);
 317         } else {
 318                 desc->dtargd_ndx = DTRACE_ARGNONE;
 319         }
 320 }
 321 static dtrace_pattr_t lockprof_attr = {
 322         { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
 323         { DTRACE_STABILITY_UNSTABLE, DTRACE_STABILITY_UNSTABLE, DTRACE_CLASS_UNKNOWN },
 324         { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
 325         { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
 326         { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
 327 };
 328
 329 static dtrace_pops_t lockprof_pops = {
 330         .dtps_provide =         lockprof_provide,
 331         .dtps_provide_module =  NULL,
 332         .dtps_enable =          lockprof_enable,
 333         .dtps_disable =         lockprof_disable,
 334         .dtps_suspend =         NULL,
 335         .dtps_resume =          NULL,
 336         .dtps_getargdesc =      lockprof_getargdesc,
 337         .dtps_getargval =       NULL,
 338         .dtps_usermode =        NULL,
 339         .dtps_destroy =         lockprof_destroy
 340 };
 341 #endif /* LOCK_STATS */
 342 void lockprof_init(void);
 343 void
 344 lockprof_init(void)
 345 {
 346 #if LOCK_STATS
 347         dtrace_register("lockprof", &lockprof_attr,
 348             DTRACE_PRIV_KERNEL, NULL,
 349             &lockprof_pops, NULL, &lockprof_id);
 350 #endif /* LOCK_STATS */
 351 }