bsd/kern/kern_kpc.c

   1 /*
   2  * Copyright (c) 2012 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <kern/debug.h>
  30 #include <kern/kalloc.h>
  31 #include <sys/param.h>
  32 #include <sys/mman.h>
  33 #include <sys/stat.h>
  34 #include <sys/sysctl.h>
  35 #include <libkern/libkern.h>
  36 #include <kern/assert.h>
  37
  38 #include <kern/kpc.h>
  39 #include <sys/ktrace.h>
  40
  41 #include <pexpert/pexpert.h>
  42 #include <kperf/kperf.h>
  43
  44 /* Various sysctl requests */
  45 #define REQ_CLASSES              (1)
  46 #define REQ_COUNTING             (2)
  47 #define REQ_THREAD_COUNTING      (3)
  48 #define REQ_CONFIG_COUNT         (4)
  49 #define REQ_COUNTER_COUNT        (5)
  50 #define REQ_THREAD_COUNTERS      (6)
  51 #define REQ_COUNTERS             (7)
  52 #define REQ_SHADOW_COUNTERS      (8)
  53 #define REQ_CONFIG               (9)
  54 #define REQ_PERIOD              (10)
  55 #define REQ_ACTIONID            (11)
  56 #define REQ_SW_INC              (14)
  57 #define REQ_PMU_VERSION         (15)
  58
  59 /* Type-munging casts */
  60 typedef int (*getint_t)(void);
  61 typedef int (*setint_t)(int);
  62
  63 static int kpc_initted = 0;
  64
  65 static lck_grp_attr_t *sysctl_lckgrp_attr = NULL;
  66 static lck_grp_t *sysctl_lckgrp = NULL;
  67 static lck_mtx_t sysctl_lock;
  68
  69 /*
  70  * Another element is needed to hold the CPU number when getting counter values.
  71  */
  72 #define KPC_MAX_BUF_LEN (KPC_MAX_COUNTERS_COPIED + 1)
  73
  74 typedef int (*setget_func_t)(int);
  75
  76 void
  77 kpc_init(void)
  78 {
  79         sysctl_lckgrp_attr = lck_grp_attr_alloc_init();
  80         sysctl_lckgrp = lck_grp_alloc_init("kpc", sysctl_lckgrp_attr);
  81         lck_mtx_init(&sysctl_lock, sysctl_lckgrp, LCK_ATTR_NULL);
  82
  83         kpc_arch_init();
  84
  85         kpc_initted = 1;
  86 }
  87
  88 static uint64_t *
  89 kpc_get_bigarray(uint32_t *size_out)
  90 {
  91         static uint64_t *bigarray = NULL;
  92
  93         LCK_MTX_ASSERT(&sysctl_lock, LCK_MTX_ASSERT_OWNED);
  94
  95         uint32_t size = kpc_get_counterbuf_size() + sizeof(uint64_t);
  96         *size_out = size;
  97
  98         if (bigarray) {
  99                 return bigarray;
 100         }
 101
 102         /*
 103          * Another element is needed to hold the CPU number when getting counter
 104          * values.
 105          */
 106         bigarray = kheap_alloc_tag(KHEAP_DATA_BUFFERS, size,
 107             Z_WAITOK, VM_KERN_MEMORY_DIAG);
 108         assert(bigarray != NULL);
 109         return bigarray;
 110 }
 111
 112 /* abstract sysctl handlers */
 113 static int
 114 sysctl_get_int( struct sysctl_oid *oidp, struct sysctl_req *req,
 115     uint32_t value )
 116 {
 117         int error = 0;
 118
 119         /* copy out the old value */
 120         error = sysctl_handle_int(oidp, &value, 0, req);
 121
 122         return error;
 123 }
 124
 125 static int
 126 sysctl_set_int( struct sysctl_req *req, int (*set_func)(int))
 127 {
 128         int error = 0;
 129         int value = 0;
 130
 131         error = SYSCTL_IN( req, &value, sizeof(value));
 132         if (error) {
 133                 return error;
 134         }
 135
 136         error = set_func( value );
 137
 138         return error;
 139 }
 140
 141 static int
 142 sysctl_getset_int( struct sysctl_oid *oidp, struct sysctl_req *req,
 143     int (*get_func)(void), int (*set_func)(int))
 144 {
 145         int error = 0;
 146         uint32_t value = 0;
 147
 148         /* get the old value and process it */
 149         value = get_func();
 150
 151         /* copy out the old value, get the new value */
 152         error = sysctl_handle_int(oidp, &value, 0, req);
 153         if (error || !req->newptr) {
 154                 return error;
 155         }
 156
 157         /* if that worked, and we're writing... */
 158         error = set_func( value );
 159
 160         return error;
 161 }
 162
 163
 164 static int
 165 sysctl_setget_int( struct sysctl_req *req,
 166     int (*setget_func)(int))
 167 {
 168         int error = 0;
 169         int value = 0;
 170
 171         error = SYSCTL_IN( req, &value, sizeof(value));
 172         if (error) {
 173                 return error;
 174         }
 175
 176         value = setget_func(value);
 177
 178         error = SYSCTL_OUT( req, &value, sizeof(value));
 179
 180         return error;
 181 }
 182
 183 static int
 184 sysctl_kpc_get_counters(uint32_t counters,
 185     uint32_t *size, void *buf)
 186 {
 187         uint64_t *ctr_buf = (uint64_t*)buf;
 188         int curcpu;
 189         uint32_t count;
 190
 191         count = kpc_get_cpu_counters(counters & KPC_ALL_CPUS,
 192             counters,
 193             &curcpu, &ctr_buf[1]);
 194         if (!count) {
 195                 return EINVAL;
 196         }
 197
 198         ctr_buf[0] = curcpu;
 199
 200         *size = (count + 1) * sizeof(uint64_t);
 201
 202         return 0;
 203 }
 204
 205 static int
 206 sysctl_kpc_get_shadow_counters(uint32_t counters,
 207     uint32_t *size, void *buf)
 208 {
 209         uint64_t *ctr_buf = (uint64_t*)buf;
 210         int curcpu;
 211         uint32_t count;
 212
 213         count = kpc_get_shadow_counters(counters & KPC_ALL_CPUS,
 214             counters,
 215             &curcpu, &ctr_buf[1]);
 216
 217         if (!count) {
 218                 return EINVAL;
 219         }
 220
 221         ctr_buf[0] = curcpu;
 222
 223         *size = (count + 1) * sizeof(uint64_t);
 224
 225         return 0;
 226 }
 227
 228 static int
 229 sysctl_kpc_get_thread_counters(uint32_t tid,
 230     uint32_t *size, void *buf)
 231 {
 232         uint32_t count = *size / sizeof(uint64_t);
 233         int r;
 234
 235         if (tid != 0) {
 236                 return EINVAL;
 237         }
 238
 239         r = kpc_get_curthread_counters(&count, buf);
 240         if (!r) {
 241                 *size = count * sizeof(uint64_t);
 242         }
 243
 244         return r;
 245 }
 246
 247 static int
 248 sysctl_kpc_get_config(uint32_t classes, void* buf)
 249 {
 250         return kpc_get_config( classes, buf );
 251 }
 252
 253 static int
 254 sysctl_kpc_set_config(uint32_t classes, void* buf)
 255 {
 256         /* userspace cannot reconfigure the power class */
 257         if (classes & KPC_CLASS_POWER_MASK) {
 258                 return EPERM;
 259         }
 260         return kpc_set_config( classes, buf);
 261 }
 262
 263 static int
 264 sysctl_kpc_get_period(uint32_t classes, void* buf)
 265 {
 266         return kpc_get_period( classes, buf );
 267 }
 268
 269 static int
 270 sysctl_kpc_set_period(uint32_t classes, void* buf)
 271 {
 272         /* userspace cannot reconfigure the power class */
 273         if (classes & KPC_CLASS_POWER_MASK) {
 274                 return EPERM;
 275         }
 276         return kpc_set_period( classes, buf);
 277 }
 278
 279 static int
 280 sysctl_kpc_get_actionid(uint32_t classes, void* buf)
 281 {
 282         return kpc_get_actionid( classes, buf );
 283 }
 284
 285 static int
 286 sysctl_kpc_set_actionid(uint32_t classes, void* buf)
 287 {
 288         return kpc_set_actionid( classes, buf);
 289 }
 290
 291
 292 static int
 293 sysctl_get_bigarray(struct sysctl_req *req,
 294     int (*get_fn)(uint32_t, uint32_t*, void*))
 295 {
 296         uint32_t bufsize = 0;
 297         uint64_t *buf = kpc_get_bigarray(&bufsize);
 298         uint32_t arg = 0;
 299
 300         /* get the argument */
 301         int error = SYSCTL_IN(req, &arg, sizeof(arg));
 302         if (error) {
 303                 return error;
 304         }
 305
 306         error = get_fn(arg, &bufsize, buf);
 307         if (!error) {
 308                 error = SYSCTL_OUT(req, buf, bufsize);
 309         }
 310
 311         return error;
 312 }
 313
 314 /* given a config word, how many bytes does it take? */
 315 static int
 316 sysctl_config_size( uint32_t config )
 317 {
 318         return kpc_get_config_count(config) * sizeof(kpc_config_t);
 319 }
 320
 321 static int
 322 sysctl_counter_size( uint32_t classes )
 323 {
 324         return kpc_get_counter_count(classes) * sizeof(uint64_t);
 325 }
 326
 327 static int
 328 sysctl_actionid_size( uint32_t classes )
 329 {
 330         return kpc_get_counter_count(classes) * sizeof(int32_t);
 331 }
 332
 333 static int
 334 sysctl_getset_bigarray(struct sysctl_req *req, int (*size_fn)(uint32_t arg),
 335     int (*get_fn)(uint32_t, void*), int (*set_fn)(uint32_t, void*))
 336 {
 337         int error = 0;
 338         uint64_t arg;
 339
 340         uint32_t bufsize = 0;
 341         uint64_t *buf = kpc_get_bigarray(&bufsize);
 342
 343         /* get the config word */
 344         error = SYSCTL_IN(req, &arg, sizeof(arg));
 345         if (error) {
 346                 return error;
 347         }
 348
 349         /* Determine the size of registers to modify. */
 350         uint32_t regsize = size_fn((uint32_t)arg);
 351         if (regsize == 0 || regsize > bufsize) {
 352                 return EINVAL;
 353         }
 354
 355         /* if writing */
 356         if (req->newptr) {
 357                 /* copy the rest -- SYSCTL_IN knows the copyin should be shifted */
 358                 error = SYSCTL_IN(req, buf, regsize);
 359
 360                 /* SYSCTL_IN failure means only need to read */
 361                 if (!error) {
 362                         error = set_fn((uint32_t)arg, buf);
 363                         if (error) {
 364                                 return error;
 365                         }
 366                 }
 367         }
 368
 369         /* if reading */
 370         if (req->oldptr) {
 371                 error = get_fn((uint32_t)arg, buf);
 372                 if (error) {
 373                         return error;
 374                 }
 375
 376                 error = SYSCTL_OUT(req, buf, regsize);
 377         }
 378
 379         return error;
 380 }
 381
 382 static int
 383 kpc_sysctl SYSCTL_HANDLER_ARGS
 384 {
 385         int ret;
 386
 387         // __unused struct sysctl_oid *unused_oidp = oidp;
 388         (void)arg2;
 389
 390         if (!kpc_initted) {
 391                 panic("kpc_init not called");
 392         }
 393
 394         if (!kpc_supported) {
 395                 return ENOTSUP;
 396         }
 397
 398         ktrace_lock();
 399
 400         // Most sysctls require an access check, but a few are public.
 401         switch ((uintptr_t) arg1) {
 402         case REQ_CLASSES:
 403         case REQ_CONFIG_COUNT:
 404         case REQ_COUNTER_COUNT:
 405                 // These read-only sysctls are public.
 406                 break;
 407
 408         default:
 409                 // Require kperf access to read or write anything else.
 410                 // This is either root or the blessed pid.
 411                 if ((ret = ktrace_read_check())) {
 412                         ktrace_unlock();
 413                         return ret;
 414                 }
 415                 break;
 416         }
 417
 418         ktrace_unlock();
 419
 420         lck_mtx_lock(&sysctl_lock);
 421
 422         /* which request */
 423         switch ((uintptr_t) arg1) {
 424         case REQ_CLASSES:
 425                 ret = sysctl_get_int( oidp, req,
 426                     kpc_get_classes());
 427                 break;
 428         case REQ_COUNTING:
 429                 ret = sysctl_getset_int( oidp, req,
 430                     (getint_t)kpc_get_running,
 431                     (setint_t)kpc_set_running );
 432                 break;
 433         case REQ_THREAD_COUNTING:
 434                 ret = sysctl_getset_int( oidp, req,
 435                     (getint_t)kpc_get_thread_counting,
 436                     (setint_t)kpc_set_thread_counting );
 437                 break;
 438
 439         case REQ_CONFIG_COUNT:
 440                 ret = sysctl_setget_int( req,
 441                     (setget_func_t)kpc_get_config_count );
 442                 break;
 443
 444         case REQ_COUNTER_COUNT:
 445                 ret = sysctl_setget_int( req,
 446                     (setget_func_t)kpc_get_counter_count );
 447                 break;
 448
 449
 450         case REQ_THREAD_COUNTERS:
 451                 ret = sysctl_get_bigarray( req, sysctl_kpc_get_thread_counters );
 452                 break;
 453
 454         case REQ_COUNTERS:
 455                 ret = sysctl_get_bigarray( req, sysctl_kpc_get_counters );
 456                 break;
 457
 458         case REQ_SHADOW_COUNTERS:
 459                 ret = sysctl_get_bigarray( req, sysctl_kpc_get_shadow_counters );
 460                 break;
 461
 462         case REQ_CONFIG:
 463                 ret = sysctl_getset_bigarray( req,
 464                     sysctl_config_size,
 465                     sysctl_kpc_get_config,
 466                     sysctl_kpc_set_config );
 467                 break;
 468
 469         case REQ_PERIOD:
 470                 ret = sysctl_getset_bigarray( req,
 471                     sysctl_counter_size,
 472                     sysctl_kpc_get_period,
 473                     sysctl_kpc_set_period );
 474                 break;
 475
 476         case REQ_ACTIONID:
 477                 ret = sysctl_getset_bigarray( req,
 478                     sysctl_actionid_size,
 479                     sysctl_kpc_get_actionid,
 480                     sysctl_kpc_set_actionid );
 481                 break;
 482
 483
 484         case REQ_SW_INC:
 485                 ret = sysctl_set_int( req, (setget_func_t)kpc_set_sw_inc );
 486                 break;
 487
 488         case REQ_PMU_VERSION:
 489                 ret = sysctl_get_int(oidp, req, kpc_get_pmu_version());
 490                 break;
 491
 492         default:
 493                 ret = ENOENT;
 494                 break;
 495         }
 496
 497         lck_mtx_unlock(&sysctl_lock);
 498
 499         return ret;
 500 }
 501
 502
 503 /***  sysctl definitions  ***/
 504
 505 /* root kperf node */
 506 SYSCTL_NODE(, OID_AUTO, kpc, CTLFLAG_RW | CTLFLAG_LOCKED, 0,
 507     "kpc");
 508
 509 /* values */
 510 SYSCTL_PROC(_kpc, OID_AUTO, classes,
 511     CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
 512     (void*)REQ_CLASSES,
 513     sizeof(int), kpc_sysctl, "I", "Available classes");
 514
 515 SYSCTL_PROC(_kpc, OID_AUTO, counting,
 516     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
 517     (void*)REQ_COUNTING,
 518     sizeof(int), kpc_sysctl, "I", "PMCs counting");
 519
 520 SYSCTL_PROC(_kpc, OID_AUTO, thread_counting,
 521     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
 522     (void*)REQ_THREAD_COUNTING,
 523     sizeof(int), kpc_sysctl, "I", "Thread accumulation");
 524
 525 SYSCTL_PROC(_kpc, OID_AUTO, pmu_version,
 526     CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
 527     (void *)REQ_PMU_VERSION,
 528     sizeof(int), kpc_sysctl, "I", "PMU version for hardware");
 529
 530 /* faux values */
 531 SYSCTL_PROC(_kpc, OID_AUTO, config_count,
 532     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
 533     (void*)REQ_CONFIG_COUNT,
 534     sizeof(int), kpc_sysctl, "S", "Config count");
 535
 536 SYSCTL_PROC(_kpc, OID_AUTO, counter_count,
 537     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
 538     (void*)REQ_COUNTER_COUNT,
 539     sizeof(int), kpc_sysctl, "S", "Counter count");
 540
 541 SYSCTL_PROC(_kpc, OID_AUTO, sw_inc,
 542     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
 543     (void*)REQ_SW_INC,
 544     sizeof(int), kpc_sysctl, "S", "Software increment");
 545
 546 /* arrays */
 547 SYSCTL_PROC(_kpc, OID_AUTO, thread_counters,
 548     CTLFLAG_RD | CTLFLAG_WR | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
 549     (void*)REQ_THREAD_COUNTERS,
 550     sizeof(uint64_t), kpc_sysctl,
 551     "QU", "Current thread counters");
 552
 553 SYSCTL_PROC(_kpc, OID_AUTO, counters,
 554     CTLFLAG_RD | CTLFLAG_WR | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
 555     (void*)REQ_COUNTERS,
 556     sizeof(uint64_t), kpc_sysctl,
 557     "QU", "Current counters");
 558
 559 SYSCTL_PROC(_kpc, OID_AUTO, shadow_counters,
 560     CTLFLAG_RD | CTLFLAG_WR | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
 561     (void*)REQ_SHADOW_COUNTERS,
 562     sizeof(uint64_t), kpc_sysctl,
 563     "QU", "Current shadow counters");
 564
 565 SYSCTL_PROC(_kpc, OID_AUTO, config,
 566     CTLFLAG_RD | CTLFLAG_WR | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
 567     (void*)REQ_CONFIG,
 568     sizeof(uint64_t), kpc_sysctl,
 569     "QU", "Set counter configs");
 570
 571 SYSCTL_PROC(_kpc, OID_AUTO, period,
 572     CTLFLAG_RD | CTLFLAG_WR | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
 573     (void*)REQ_PERIOD,
 574     sizeof(uint64_t), kpc_sysctl,
 575     "QU", "Set counter periods");
 576
 577 SYSCTL_PROC(_kpc, OID_AUTO, actionid,
 578     CTLFLAG_RD | CTLFLAG_WR | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
 579     (void*)REQ_ACTIONID,
 580     sizeof(uint32_t), kpc_sysctl,
 581     "QU", "Set counter actionids");
 582
 583