bsd/kern/kern_kpc.c

   1 /*
   2  * Copyright (c) 2012 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <kern/debug.h>
  30 #include <kern/kalloc.h>
  31 #include <sys/param.h>
  32 #include <sys/mman.h>
  33 #include <sys/stat.h>
  34 #include <sys/sysctl.h>
  35 #include <libkern/libkern.h>
  36 #include <kern/assert.h>
  37
  38 #include <kern/kpc.h>
  39 #include <sys/ktrace.h>
  40
  41 #include <pexpert/pexpert.h>
  42 #include <kperf/kperf.h>
  43
  44 /* Various sysctl requests */
  45 #define REQ_CLASSES              (1)
  46 #define REQ_COUNTING             (2)
  47 #define REQ_THREAD_COUNTING      (3)
  48 #define REQ_CONFIG_COUNT         (4)
  49 #define REQ_COUNTER_COUNT        (5)
  50 #define REQ_THREAD_COUNTERS      (6)
  51 #define REQ_COUNTERS             (7)
  52 #define REQ_SHADOW_COUNTERS      (8)
  53 #define REQ_CONFIG               (9)
  54 #define REQ_PERIOD              (10)
  55 #define REQ_ACTIONID            (11)
  56 #define REQ_SW_INC              (14)
  57 #define REQ_PMU_VERSION         (15)
  58
  59 /* Type-munging casts */
  60 typedef int (*getint_t)(void);
  61 typedef int (*setint_t)(int);
  62
  63 static int kpc_initted = 0;
  64
  65 static lck_grp_attr_t *sysctl_lckgrp_attr = NULL;
  66 static lck_grp_t *sysctl_lckgrp = NULL;
  67 static lck_mtx_t sysctl_lock;
  68
  69 /*
  70  * Another element is needed to hold the CPU number when getting counter values.
  71  */
  72 #define KPC_MAX_BUF_LEN (KPC_MAX_COUNTERS_COPIED + 1)
  73
  74 typedef int (*setget_func_t)(int);
  75
  76 void
  77 kpc_init(void)
  78 {
  79         sysctl_lckgrp_attr = lck_grp_attr_alloc_init();
  80         sysctl_lckgrp = lck_grp_alloc_init("kpc", sysctl_lckgrp_attr);
  81         lck_mtx_init(&sysctl_lock, sysctl_lckgrp, LCK_ATTR_NULL);
  82
  83         kpc_arch_init();
  84         kpc_common_init();
  85         kpc_thread_init();
  86
  87         kpc_initted = 1;
  88 }
  89
  90 static uint64_t *
  91 kpc_get_bigarray(uint32_t *size_out)
  92 {
  93         static uint64_t *bigarray = NULL;
  94
  95         LCK_MTX_ASSERT(&sysctl_lock, LCK_MTX_ASSERT_OWNED);
  96
  97         uint32_t size = kpc_get_counterbuf_size() + sizeof(uint64_t);
  98         *size_out = size;
  99
 100         if (bigarray) {
 101                 return bigarray;
 102         }
 103
 104         /*
 105          * Another element is needed to hold the CPU number when getting counter
 106          * values.
 107          */
 108         bigarray = kalloc_tag(size, VM_KERN_MEMORY_DIAG);
 109         assert(bigarray != NULL);
 110         return bigarray;
 111 }
 112
 113 /* abstract sysctl handlers */
 114 static int
 115 sysctl_get_int( struct sysctl_oid *oidp, struct sysctl_req *req,
 116     uint32_t value )
 117 {
 118         int error = 0;
 119
 120         /* copy out the old value */
 121         error = sysctl_handle_int(oidp, &value, 0, req);
 122
 123         return error;
 124 }
 125
 126 static int
 127 sysctl_set_int( struct sysctl_req *req, int (*set_func)(int))
 128 {
 129         int error = 0;
 130         int value = 0;
 131
 132         error = SYSCTL_IN( req, &value, sizeof(value));
 133         if (error) {
 134                 return error;
 135         }
 136
 137         error = set_func( value );
 138
 139         return error;
 140 }
 141
 142 static int
 143 sysctl_getset_int( struct sysctl_oid *oidp, struct sysctl_req *req,
 144     int (*get_func)(void), int (*set_func)(int))
 145 {
 146         int error = 0;
 147         uint32_t value = 0;
 148
 149         /* get the old value and process it */
 150         value = get_func();
 151
 152         /* copy out the old value, get the new value */
 153         error = sysctl_handle_int(oidp, &value, 0, req);
 154         if (error || !req->newptr) {
 155                 return error;
 156         }
 157
 158         /* if that worked, and we're writing... */
 159         error = set_func( value );
 160
 161         return error;
 162 }
 163
 164
 165 static int
 166 sysctl_setget_int( struct sysctl_req *req,
 167     int (*setget_func)(int))
 168 {
 169         int error = 0;
 170         int value = 0;
 171
 172         error = SYSCTL_IN( req, &value, sizeof(value));
 173         if (error) {
 174                 return error;
 175         }
 176
 177         value = setget_func(value);
 178
 179         error = SYSCTL_OUT( req, &value, sizeof(value));
 180
 181         return error;
 182 }
 183
 184 static int
 185 sysctl_kpc_get_counters(uint32_t counters,
 186     uint32_t *size, void *buf)
 187 {
 188         uint64_t *ctr_buf = (uint64_t*)buf;
 189         int curcpu;
 190         uint32_t count;
 191
 192         count = kpc_get_cpu_counters(counters & KPC_ALL_CPUS,
 193             counters,
 194             &curcpu, &ctr_buf[1]);
 195         if (!count) {
 196                 return EINVAL;
 197         }
 198
 199         ctr_buf[0] = curcpu;
 200
 201         *size = (count + 1) * sizeof(uint64_t);
 202
 203         return 0;
 204 }
 205
 206 static int
 207 sysctl_kpc_get_shadow_counters(uint32_t counters,
 208     uint32_t *size, void *buf)
 209 {
 210         uint64_t *ctr_buf = (uint64_t*)buf;
 211         int curcpu;
 212         uint32_t count;
 213
 214         count = kpc_get_shadow_counters(counters & KPC_ALL_CPUS,
 215             counters,
 216             &curcpu, &ctr_buf[1]);
 217
 218         if (!count) {
 219                 return EINVAL;
 220         }
 221
 222         ctr_buf[0] = curcpu;
 223
 224         *size = (count + 1) * sizeof(uint64_t);
 225
 226         return 0;
 227 }
 228
 229 static int
 230 sysctl_kpc_get_thread_counters(uint32_t tid,
 231     uint32_t *size, void *buf)
 232 {
 233         uint32_t count = *size / sizeof(uint64_t);
 234         int r;
 235
 236         if (tid != 0) {
 237                 return EINVAL;
 238         }
 239
 240         r = kpc_get_curthread_counters(&count, buf);
 241         if (!r) {
 242                 *size = count * sizeof(uint64_t);
 243         }
 244
 245         return r;
 246 }
 247
 248 static int
 249 sysctl_kpc_get_config(uint32_t classes, void* buf)
 250 {
 251         return kpc_get_config( classes, buf );
 252 }
 253
 254 static int
 255 sysctl_kpc_set_config(uint32_t classes, void* buf)
 256 {
 257         /* userspace cannot reconfigure the power class */
 258         if (classes & KPC_CLASS_POWER_MASK) {
 259                 return EPERM;
 260         }
 261         return kpc_set_config( classes, buf);
 262 }
 263
 264 static int
 265 sysctl_kpc_get_period(uint32_t classes, void* buf)
 266 {
 267         return kpc_get_period( classes, buf );
 268 }
 269
 270 static int
 271 sysctl_kpc_set_period(uint32_t classes, void* buf)
 272 {
 273         /* userspace cannot reconfigure the power class */
 274         if (classes & KPC_CLASS_POWER_MASK) {
 275                 return EPERM;
 276         }
 277         return kpc_set_period( classes, buf);
 278 }
 279
 280 static int
 281 sysctl_kpc_get_actionid(uint32_t classes, void* buf)
 282 {
 283         return kpc_get_actionid( classes, buf );
 284 }
 285
 286 static int
 287 sysctl_kpc_set_actionid(uint32_t classes, void* buf)
 288 {
 289         return kpc_set_actionid( classes, buf);
 290 }
 291
 292
 293 static int
 294 sysctl_get_bigarray(struct sysctl_req *req,
 295     int (*get_fn)(uint32_t, uint32_t*, void*))
 296 {
 297         uint32_t bufsize = 0;
 298         uint64_t *buf = kpc_get_bigarray(&bufsize);
 299         uint32_t arg = 0;
 300
 301         /* get the argument */
 302         int error = SYSCTL_IN(req, &arg, sizeof(arg));
 303         if (error) {
 304                 return error;
 305         }
 306
 307         error = get_fn(arg, &bufsize, buf);
 308         if (!error) {
 309                 error = SYSCTL_OUT(req, buf, bufsize);
 310         }
 311
 312         return error;
 313 }
 314
 315 /* given a config word, how many bytes does it take? */
 316 static int
 317 sysctl_config_size( uint32_t config )
 318 {
 319         return kpc_get_config_count(config) * sizeof(kpc_config_t);
 320 }
 321
 322 static int
 323 sysctl_counter_size( uint32_t classes )
 324 {
 325         return kpc_get_counter_count(classes) * sizeof(uint64_t);
 326 }
 327
 328 static int
 329 sysctl_actionid_size( uint32_t classes )
 330 {
 331         return kpc_get_counter_count(classes) * sizeof(int32_t);
 332 }
 333
 334 static int
 335 sysctl_getset_bigarray(struct sysctl_req *req, int (*size_fn)(uint32_t arg),
 336     int (*get_fn)(uint32_t, void*), int (*set_fn)(uint32_t, void*))
 337 {
 338         int error = 0;
 339         uint64_t arg;
 340
 341         uint32_t bufsize = 0;
 342         uint64_t *buf = kpc_get_bigarray(&bufsize);
 343
 344         /* get the config word */
 345         error = SYSCTL_IN(req, &arg, sizeof(arg));
 346         if (error) {
 347                 return error;
 348         }
 349
 350         /* Determine the size of registers to modify. */
 351         uint32_t regsize = size_fn((uint32_t)arg);
 352         if (regsize == 0 || regsize > bufsize) {
 353                 return EINVAL;
 354         }
 355
 356         /* if writing */
 357         if (req->newptr) {
 358                 /* copy the rest -- SYSCTL_IN knows the copyin should be shifted */
 359                 error = SYSCTL_IN(req, buf, regsize);
 360
 361                 /* SYSCTL_IN failure means only need to read */
 362                 if (!error) {
 363                         error = set_fn((uint32_t)arg, buf);
 364                         if (error) {
 365                                 return error;
 366                         }
 367                 }
 368         }
 369
 370         /* if reading */
 371         if (req->oldptr) {
 372                 error = get_fn((uint32_t)arg, buf);
 373                 if (error) {
 374                         return error;
 375                 }
 376
 377                 error = SYSCTL_OUT(req, buf, regsize);
 378         }
 379
 380         return error;
 381 }
 382
 383 static int
 384 kpc_sysctl SYSCTL_HANDLER_ARGS
 385 {
 386         int ret;
 387
 388         // __unused struct sysctl_oid *unused_oidp = oidp;
 389         (void)arg2;
 390
 391         if (!kpc_initted) {
 392                 panic("kpc_init not called");
 393         }
 394
 395         if (!kpc_supported) {
 396                 return ENOTSUP;
 397         }
 398
 399         ktrace_lock();
 400
 401         // Most sysctls require an access check, but a few are public.
 402         switch ((uintptr_t) arg1) {
 403         case REQ_CLASSES:
 404         case REQ_CONFIG_COUNT:
 405         case REQ_COUNTER_COUNT:
 406                 // These read-only sysctls are public.
 407                 break;
 408
 409         default:
 410                 // Require kperf access to read or write anything else.
 411                 // This is either root or the blessed pid.
 412                 if ((ret = ktrace_read_check())) {
 413                         ktrace_unlock();
 414                         return ret;
 415                 }
 416                 break;
 417         }
 418
 419         ktrace_unlock();
 420
 421         lck_mtx_lock(&sysctl_lock);
 422
 423         /* which request */
 424         switch ((uintptr_t) arg1) {
 425         case REQ_CLASSES:
 426                 ret = sysctl_get_int( oidp, req,
 427                     kpc_get_classes());
 428                 break;
 429         case REQ_COUNTING:
 430                 ret = sysctl_getset_int( oidp, req,
 431                     (getint_t)kpc_get_running,
 432                     (setint_t)kpc_set_running );
 433                 break;
 434         case REQ_THREAD_COUNTING:
 435                 ret = sysctl_getset_int( oidp, req,
 436                     (getint_t)kpc_get_thread_counting,
 437                     (setint_t)kpc_set_thread_counting );
 438                 break;
 439
 440         case REQ_CONFIG_COUNT:
 441                 ret = sysctl_setget_int( req,
 442                     (setget_func_t)kpc_get_config_count );
 443                 break;
 444
 445         case REQ_COUNTER_COUNT:
 446                 ret = sysctl_setget_int( req,
 447                     (setget_func_t)kpc_get_counter_count );
 448                 break;
 449
 450
 451         case REQ_THREAD_COUNTERS:
 452                 ret = sysctl_get_bigarray( req, sysctl_kpc_get_thread_counters );
 453                 break;
 454
 455         case REQ_COUNTERS:
 456                 ret = sysctl_get_bigarray( req, sysctl_kpc_get_counters );
 457                 break;
 458
 459         case REQ_SHADOW_COUNTERS:
 460                 ret = sysctl_get_bigarray( req, sysctl_kpc_get_shadow_counters );
 461                 break;
 462
 463         case REQ_CONFIG:
 464                 ret = sysctl_getset_bigarray( req,
 465                     sysctl_config_size,
 466                     sysctl_kpc_get_config,
 467                     sysctl_kpc_set_config );
 468                 break;
 469
 470         case REQ_PERIOD:
 471                 ret = sysctl_getset_bigarray( req,
 472                     sysctl_counter_size,
 473                     sysctl_kpc_get_period,
 474                     sysctl_kpc_set_period );
 475                 break;
 476
 477         case REQ_ACTIONID:
 478                 ret = sysctl_getset_bigarray( req,
 479                     sysctl_actionid_size,
 480                     sysctl_kpc_get_actionid,
 481                     sysctl_kpc_set_actionid );
 482                 break;
 483
 484
 485         case REQ_SW_INC:
 486                 ret = sysctl_set_int( req, (setget_func_t)kpc_set_sw_inc );
 487                 break;
 488
 489         case REQ_PMU_VERSION:
 490                 ret = sysctl_get_int(oidp, req, kpc_get_pmu_version());
 491                 break;
 492
 493         default:
 494                 ret = ENOENT;
 495                 break;
 496         }
 497
 498         lck_mtx_unlock(&sysctl_lock);
 499
 500         return ret;
 501 }
 502
 503
 504 /***  sysctl definitions  ***/
 505
 506 /* root kperf node */
 507 SYSCTL_NODE(, OID_AUTO, kpc, CTLFLAG_RW | CTLFLAG_LOCKED, 0,
 508     "kpc");
 509
 510 /* values */
 511 SYSCTL_PROC(_kpc, OID_AUTO, classes,
 512     CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
 513     (void*)REQ_CLASSES,
 514     sizeof(int), kpc_sysctl, "I", "Available classes");
 515
 516 SYSCTL_PROC(_kpc, OID_AUTO, counting,
 517     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
 518     (void*)REQ_COUNTING,
 519     sizeof(int), kpc_sysctl, "I", "PMCs counting");
 520
 521 SYSCTL_PROC(_kpc, OID_AUTO, thread_counting,
 522     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
 523     (void*)REQ_THREAD_COUNTING,
 524     sizeof(int), kpc_sysctl, "I", "Thread accumulation");
 525
 526 SYSCTL_PROC(_kpc, OID_AUTO, pmu_version,
 527     CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
 528     (void *)REQ_PMU_VERSION,
 529     sizeof(int), kpc_sysctl, "I", "PMU version for hardware");
 530
 531 /* faux values */
 532 SYSCTL_PROC(_kpc, OID_AUTO, config_count,
 533     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
 534     (void*)REQ_CONFIG_COUNT,
 535     sizeof(int), kpc_sysctl, "S", "Config count");
 536
 537 SYSCTL_PROC(_kpc, OID_AUTO, counter_count,
 538     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
 539     (void*)REQ_COUNTER_COUNT,
 540     sizeof(int), kpc_sysctl, "S", "Counter count");
 541
 542 SYSCTL_PROC(_kpc, OID_AUTO, sw_inc,
 543     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
 544     (void*)REQ_SW_INC,
 545     sizeof(int), kpc_sysctl, "S", "Software increment");
 546
 547 /* arrays */
 548 SYSCTL_PROC(_kpc, OID_AUTO, thread_counters,
 549     CTLFLAG_RD | CTLFLAG_WR | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
 550     (void*)REQ_THREAD_COUNTERS,
 551     sizeof(uint64_t), kpc_sysctl,
 552     "QU", "Current thread counters");
 553
 554 SYSCTL_PROC(_kpc, OID_AUTO, counters,
 555     CTLFLAG_RD | CTLFLAG_WR | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
 556     (void*)REQ_COUNTERS,
 557     sizeof(uint64_t), kpc_sysctl,
 558     "QU", "Current counters");
 559
 560 SYSCTL_PROC(_kpc, OID_AUTO, shadow_counters,
 561     CTLFLAG_RD | CTLFLAG_WR | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
 562     (void*)REQ_SHADOW_COUNTERS,
 563     sizeof(uint64_t), kpc_sysctl,
 564     "QU", "Current shadow counters");
 565
 566 SYSCTL_PROC(_kpc, OID_AUTO, config,
 567     CTLFLAG_RD | CTLFLAG_WR | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
 568     (void*)REQ_CONFIG,
 569     sizeof(uint64_t), kpc_sysctl,
 570     "QU", "Set counter configs");
 571
 572 SYSCTL_PROC(_kpc, OID_AUTO, period,
 573     CTLFLAG_RD | CTLFLAG_WR | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
 574     (void*)REQ_PERIOD,
 575     sizeof(uint64_t), kpc_sysctl,
 576     "QU", "Set counter periods");
 577
 578 SYSCTL_PROC(_kpc, OID_AUTO, actionid,
 579     CTLFLAG_RD | CTLFLAG_WR | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
 580     (void*)REQ_ACTIONID,
 581     sizeof(uint32_t), kpc_sysctl,
 582     "QU", "Set counter actionids");
 583
 584