bsd/kern/kern_kpc.c

   1 /*
   2  * Copyright (c) 2012 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <kern/debug.h>
  30 #include <kern/kalloc.h>
  31 #include <sys/param.h>
  32 #include <sys/mman.h>
  33 #include <sys/stat.h>
  34 #include <sys/sysctl.h>
  35 #include <libkern/libkern.h>
  36 #include <kern/assert.h>
  37
  38 #include <kern/kpc.h>
  39 #include <sys/ktrace.h>
  40
  41 #include <pexpert/pexpert.h>
  42 #include <kperf/kperf.h>
  43
  44 /* Various sysctl requests */
  45 #define REQ_CLASSES              (1)
  46 #define REQ_COUNTING             (2)
  47 #define REQ_THREAD_COUNTING      (3)
  48 #define REQ_CONFIG_COUNT         (4)
  49 #define REQ_COUNTER_COUNT        (5)
  50 #define REQ_THREAD_COUNTERS      (6)
  51 #define REQ_COUNTERS             (7)
  52 #define REQ_SHADOW_COUNTERS      (8)
  53 #define REQ_CONFIG               (9)
  54 #define REQ_PERIOD              (10)
  55 #define REQ_ACTIONID            (11)
  56 #define REQ_SW_INC              (14)
  57 #define REQ_PMU_VERSION         (15)
  58
  59 /* Type-munging casts */
  60 typedef int (*getint_t)(void);
  61 typedef int (*setint_t)(int);
  62
  63 static int kpc_initted = 0;
  64
  65 static lck_grp_attr_t *sysctl_lckgrp_attr = NULL;
  66 static lck_grp_t *sysctl_lckgrp = NULL;
  67 static lck_mtx_t sysctl_lock;
  68
  69 #if defined(__x86_64__)
  70 /* 18 cores, 7 counters each */
  71 #define KPC_MAX_COUNTERS_COPIED (18 * 7)
  72 #elif defined(__arm64__)
  73 #include <pexpert/arm64/board_config.h>
  74 #if defined(CPU_COUNT)
  75 #define KPC_MAX_COUNTERS_COPIED (CPU_COUNT * 10)
  76 #else /* defined(CPU_COUNT) */
  77 #define KPC_MAX_COUNTERS_COPIED (2 * 10)
  78 #endif /* !defined(CPU_COUNT) */
  79 #elif defined(__arm__)
  80 #define KPC_MAX_COUNTERS_COPIED (16)
  81 #else /* !defined(__arm__) && !defined(__arm64__) && !defined(__x86_64__) */
  82 #error "unknown architecture for kpc buffer sizes"
  83 #endif /* !defined(__arm__) && !defined(__arm64__) && !defined(__x86_64__) */
  84
  85 static_assert((KPC_MAX_COUNTERS_COPIED * sizeof(uint64_t)) < 1024,
  86                 "kpc's stack could grow too large");
  87
  88 typedef int (*setget_func_t)(int);
  89
  90 void
  91 kpc_init(void)
  92 {
  93         sysctl_lckgrp_attr = lck_grp_attr_alloc_init();
  94         sysctl_lckgrp = lck_grp_alloc_init("kpc", sysctl_lckgrp_attr);
  95         lck_mtx_init(&sysctl_lock, sysctl_lckgrp, LCK_ATTR_NULL);
  96
  97         kpc_arch_init();
  98         kpc_common_init();
  99         kpc_thread_init();
 100
 101         kpc_initted = 1;
 102 }
 103
 104 /* abstract sysctl handlers */
 105 static int
 106 sysctl_get_int( struct sysctl_oid *oidp, struct sysctl_req *req,
 107                 uint32_t value )
 108 {
 109         int error = 0;
 110
 111         /* copy out the old value */
 112         error = sysctl_handle_int(oidp, &value, 0, req);
 113
 114         return error;
 115 }
 116
 117 static int
 118 sysctl_set_int( struct sysctl_req *req, int (*set_func)(int))
 119 {
 120         int error = 0;
 121         int value = 0;
 122
 123         error = SYSCTL_IN( req, &value, sizeof(value) );
 124         if( error )
 125                 return error;
 126
 127         error = set_func( value );
 128
 129         return error;
 130 }
 131
 132 static int
 133 sysctl_getset_int( struct sysctl_oid *oidp, struct sysctl_req *req,
 134                    int (*get_func)(void), int (*set_func)(int) )
 135 {
 136         int error = 0;
 137         uint32_t value = 0;
 138
 139         /* get the old value and process it */
 140         value = get_func();
 141
 142         /* copy out the old value, get the new value */
 143         error = sysctl_handle_int(oidp, &value, 0, req);
 144         if (error || !req->newptr)
 145                 return (error);
 146
 147         /* if that worked, and we're writing... */
 148         error = set_func( value );
 149
 150         return error;
 151 }
 152
 153
 154 static int
 155 sysctl_setget_int( struct sysctl_req *req,
 156                    int (*setget_func)(int) )
 157 {
 158         int error = 0;
 159         int value = 0;
 160
 161         error = SYSCTL_IN( req, &value, sizeof(value) );
 162         if( error )
 163                 return error;
 164
 165         value = setget_func(value);
 166
 167         error = SYSCTL_OUT( req, &value, sizeof(value) );
 168
 169         return error;
 170 }
 171
 172 static int
 173 sysctl_kpc_get_counters(uint32_t counters,
 174                       uint32_t *size, void *buf)
 175 {
 176         uint64_t *ctr_buf = (uint64_t*)buf;
 177         int curcpu;
 178         uint32_t count;
 179
 180         count = kpc_get_cpu_counters(counters & KPC_ALL_CPUS,
 181                                      counters,
 182                                      &curcpu, &ctr_buf[1]);
 183         if (!count)
 184                 return EINVAL;
 185
 186         ctr_buf[0] = curcpu;
 187
 188         *size = (count+1) * sizeof(uint64_t);
 189
 190         return 0;
 191 }
 192
 193 static int
 194 sysctl_kpc_get_shadow_counters(uint32_t counters,
 195                       uint32_t *size, void *buf)
 196 {
 197         uint64_t *ctr_buf = (uint64_t*)buf;
 198         int curcpu;
 199         uint32_t count;
 200
 201         count = kpc_get_shadow_counters(counters & KPC_ALL_CPUS,
 202                                         counters,
 203                                         &curcpu, &ctr_buf[1]);
 204
 205         if (!count)
 206                 return EINVAL;
 207
 208         ctr_buf[0] = curcpu;
 209
 210         *size = (count+1) * sizeof(uint64_t);
 211
 212         return 0;
 213 }
 214
 215 static int
 216 sysctl_kpc_get_thread_counters(uint32_t tid,
 217                              uint32_t *size, void *buf)
 218 {
 219         uint32_t count = *size / sizeof(uint64_t);
 220         int r;
 221
 222         if( tid != 0 )
 223                 return EINVAL;
 224
 225         r = kpc_get_curthread_counters(&count, buf);
 226         if( !r )
 227                 *size = count * sizeof(uint64_t);
 228
 229         return r;
 230 }
 231
 232 static int
 233 sysctl_kpc_get_config(uint32_t classes, void* buf)
 234 {
 235         return kpc_get_config( classes, buf );
 236 }
 237
 238 static int
 239 sysctl_kpc_set_config(uint32_t classes, void* buf)
 240 {
 241         /* userspace cannot reconfigure the power class */
 242         if (classes & KPC_CLASS_POWER_MASK)
 243                 return (EPERM);
 244         return kpc_set_config( classes, buf);
 245 }
 246
 247 static int
 248 sysctl_kpc_get_period(uint32_t classes, void* buf)
 249 {
 250         return kpc_get_period( classes, buf );
 251 }
 252
 253 static int
 254 sysctl_kpc_set_period(uint32_t classes, void* buf)
 255 {
 256         /* userspace cannot reconfigure the power class */
 257         if (classes & KPC_CLASS_POWER_MASK)
 258                 return (EPERM);
 259         return kpc_set_period( classes, buf);
 260 }
 261
 262 static int
 263 sysctl_kpc_get_actionid(uint32_t classes, void* buf)
 264 {
 265         return kpc_get_actionid( classes, buf );
 266 }
 267
 268 static int
 269 sysctl_kpc_set_actionid(uint32_t classes, void* buf)
 270 {
 271         return kpc_set_actionid( classes, buf);
 272 }
 273
 274
 275 static int
 276 sysctl_get_bigarray(struct sysctl_req *req,
 277                 int (*get_fn)(uint32_t, uint32_t*, void*))
 278 {
 279         uint64_t buf[KPC_MAX_COUNTERS_COPIED] = {};
 280         uint32_t bufsize = sizeof(buf);
 281         uint32_t arg = 0;
 282
 283         /* get the argument */
 284         int error = SYSCTL_IN(req, &arg, sizeof(arg));
 285         if (error) {
 286                 return error;
 287         }
 288
 289         error = get_fn(arg, &bufsize, &buf);
 290         if (!error) {
 291                 error = SYSCTL_OUT(req, &buf, bufsize);
 292         }
 293
 294         return error;
 295 }
 296
 297 /* given a config word, how many bytes does it take? */
 298 static int
 299 sysctl_config_size( uint32_t config )
 300 {
 301         return kpc_get_config_count(config) * sizeof(kpc_config_t);
 302 }
 303
 304 static int
 305 sysctl_counter_size( uint32_t classes )
 306 {
 307         return kpc_get_counter_count(classes) * sizeof(uint64_t);
 308 }
 309
 310 static int
 311 sysctl_actionid_size( uint32_t classes )
 312 {
 313         return kpc_get_counter_count(classes) * sizeof(int32_t);
 314 }
 315
 316 static int
 317 sysctl_getset_bigarray(struct sysctl_req *req, int (*size_fn)(uint32_t arg),
 318                 int (*get_fn)(uint32_t, void*), int (*set_fn)(uint32_t, void*))
 319 {
 320         int error = 0;
 321         uint64_t buf[KPC_MAX_COUNTERS_COPIED] = {};
 322         uint32_t bufsize = sizeof(buf);
 323         uint64_t arg;
 324
 325         /* get the config word */
 326         error = SYSCTL_IN(req, &arg, sizeof(arg));
 327         if (error) {
 328                 return error;
 329         }
 330
 331         /* Determine the size of registers to modify. */
 332         uint32_t regsize = size_fn((uint32_t)arg);
 333         if (regsize == 0 || regsize > bufsize) {
 334                 return EINVAL;
 335         }
 336
 337         /* if writing */
 338         if (req->newptr) {
 339                 /* copy the rest -- SYSCTL_IN knows the copyin should be shifted */
 340                 error = SYSCTL_IN(req, &buf, regsize);
 341
 342                 /* SYSCTL_IN failure means only need to read */
 343                 if (!error) {
 344                         error = set_fn((uint32_t)arg, &buf);
 345                         if (error) {
 346                                 return error;
 347                         }
 348                 }
 349         }
 350
 351         /* if reading */
 352         if (req->oldptr) {
 353                 error = get_fn((uint32_t)arg, &buf);
 354                 if (error) {
 355                         return error;
 356                 }
 357
 358                 error = SYSCTL_OUT(req, &buf, regsize);
 359         }
 360
 361         return error;
 362 }
 363
 364 static int
 365 kpc_sysctl SYSCTL_HANDLER_ARGS
 366 {
 367         int ret;
 368
 369         // __unused struct sysctl_oid *unused_oidp = oidp;
 370         (void)arg2;
 371
 372         if( !kpc_initted )
 373                 panic("kpc_init not called");
 374
 375         ktrace_lock();
 376
 377         // Most sysctls require an access check, but a few are public.
 378         switch( (uintptr_t) arg1 ) {
 379         case REQ_CLASSES:
 380         case REQ_CONFIG_COUNT:
 381         case REQ_COUNTER_COUNT:
 382                 // These read-only sysctls are public.
 383                 break;
 384
 385         default:
 386                 // Require kperf access to read or write anything else.
 387                 // This is either root or the blessed pid.
 388                 if ((ret = ktrace_read_check())) {
 389                         ktrace_unlock();
 390                         return ret;
 391                 }
 392                 break;
 393         }
 394
 395         ktrace_unlock();
 396
 397         lck_mtx_lock(&sysctl_lock);
 398
 399         /* which request */
 400         switch( (uintptr_t) arg1 )
 401         {
 402         case REQ_CLASSES:
 403                 ret = sysctl_get_int( oidp, req,
 404                                        kpc_get_classes() );
 405                 break;
 406         case REQ_COUNTING:
 407                 ret = sysctl_getset_int( oidp, req,
 408                                           (getint_t)kpc_get_running,
 409                                           (setint_t)kpc_set_running );
 410                 break;
 411         case REQ_THREAD_COUNTING:
 412                 ret = sysctl_getset_int( oidp, req,
 413                                           (getint_t)kpc_get_thread_counting,
 414                                           (setint_t)kpc_set_thread_counting );
 415                 break;
 416
 417         case REQ_CONFIG_COUNT:
 418                 ret = sysctl_setget_int( req,
 419                                           (setget_func_t)kpc_get_config_count );
 420                 break;
 421
 422         case REQ_COUNTER_COUNT:
 423                 ret = sysctl_setget_int( req,
 424                                           (setget_func_t)kpc_get_counter_count );
 425                 break;
 426
 427
 428         case REQ_THREAD_COUNTERS:
 429                 ret = sysctl_get_bigarray( req, sysctl_kpc_get_thread_counters );
 430                 break;
 431
 432         case REQ_COUNTERS:
 433                 ret = sysctl_get_bigarray( req, sysctl_kpc_get_counters );
 434                 break;
 435
 436         case REQ_SHADOW_COUNTERS:
 437                 ret = sysctl_get_bigarray( req, sysctl_kpc_get_shadow_counters );
 438                 break;
 439
 440         case REQ_CONFIG:
 441                 ret = sysctl_getset_bigarray( req,
 442                                                sysctl_config_size,
 443                                                sysctl_kpc_get_config,
 444                                                sysctl_kpc_set_config );
 445                 break;
 446
 447         case REQ_PERIOD:
 448                 ret = sysctl_getset_bigarray( req,
 449                                                sysctl_counter_size,
 450                                                sysctl_kpc_get_period,
 451                                                sysctl_kpc_set_period );
 452                 break;
 453
 454         case REQ_ACTIONID:
 455                 ret = sysctl_getset_bigarray( req,
 456                                                sysctl_actionid_size,
 457                                                sysctl_kpc_get_actionid,
 458                                                sysctl_kpc_set_actionid );
 459                 break;
 460
 461
 462         case REQ_SW_INC:
 463                 ret = sysctl_set_int( req, (setget_func_t)kpc_set_sw_inc );
 464                 break;
 465
 466         case REQ_PMU_VERSION:
 467                 ret = sysctl_get_int(oidp, req, kpc_get_pmu_version());
 468                 break;
 469
 470         default:
 471                 ret = ENOENT;
 472                 break;
 473         }
 474
 475         lck_mtx_unlock(&sysctl_lock);
 476
 477         return ret;
 478 }
 479
 480
 481 /***  sysctl definitions  ***/
 482
 483 /* root kperf node */
 484 SYSCTL_NODE(, OID_AUTO, kpc, CTLFLAG_RW|CTLFLAG_LOCKED, 0,
 485             "kpc");
 486
 487 /* values */
 488 SYSCTL_PROC(_kpc, OID_AUTO, classes,
 489             CTLTYPE_INT|CTLFLAG_RD|CTLFLAG_ANYBODY,
 490             (void*)REQ_CLASSES,
 491             sizeof(int), kpc_sysctl, "I", "Available classes");
 492
 493 SYSCTL_PROC(_kpc, OID_AUTO, counting,
 494             CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_ANYBODY,
 495             (void*)REQ_COUNTING,
 496             sizeof(int), kpc_sysctl, "I", "PMCs counting");
 497
 498 SYSCTL_PROC(_kpc, OID_AUTO, thread_counting,
 499             CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_ANYBODY,
 500             (void*)REQ_THREAD_COUNTING,
 501             sizeof(int), kpc_sysctl, "I", "Thread accumulation");
 502
 503 SYSCTL_PROC(_kpc, OID_AUTO, pmu_version,
 504             CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_ANYBODY,
 505             (void *)REQ_PMU_VERSION,
 506             sizeof(int), kpc_sysctl, "I", "PMU version for hardware");
 507
 508 /* faux values */
 509 SYSCTL_PROC(_kpc, OID_AUTO, config_count,
 510             CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_ANYBODY,
 511             (void*)REQ_CONFIG_COUNT,
 512             sizeof(int), kpc_sysctl, "S", "Config count");
 513
 514 SYSCTL_PROC(_kpc, OID_AUTO, counter_count,
 515             CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_ANYBODY,
 516             (void*)REQ_COUNTER_COUNT,
 517             sizeof(int), kpc_sysctl, "S", "Counter count");
 518
 519 SYSCTL_PROC(_kpc, OID_AUTO, sw_inc,
 520             CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_ANYBODY,
 521             (void*)REQ_SW_INC,
 522             sizeof(int), kpc_sysctl, "S", "Software increment");
 523
 524 /* arrays */
 525 SYSCTL_PROC(_kpc, OID_AUTO, thread_counters,
 526             CTLFLAG_RD|CTLFLAG_WR|CTLFLAG_ANYBODY,
 527             (void*)REQ_THREAD_COUNTERS,
 528             sizeof(uint64_t), kpc_sysctl,
 529             "QU", "Current thread counters");
 530
 531 SYSCTL_PROC(_kpc, OID_AUTO, counters,
 532             CTLFLAG_RD|CTLFLAG_WR|CTLFLAG_ANYBODY,
 533             (void*)REQ_COUNTERS,
 534             sizeof(uint64_t), kpc_sysctl,
 535             "QU", "Current counters");
 536
 537 SYSCTL_PROC(_kpc, OID_AUTO, shadow_counters,
 538             CTLFLAG_RD|CTLFLAG_WR|CTLFLAG_ANYBODY,
 539             (void*)REQ_SHADOW_COUNTERS,
 540             sizeof(uint64_t), kpc_sysctl,
 541             "QU", "Current shadow counters");
 542
 543 SYSCTL_PROC(_kpc, OID_AUTO, config,
 544             CTLFLAG_RD|CTLFLAG_WR|CTLFLAG_ANYBODY,
 545             (void*)REQ_CONFIG,
 546             sizeof(uint64_t), kpc_sysctl,
 547             "QU", "Set counter configs");
 548
 549 SYSCTL_PROC(_kpc, OID_AUTO, period,
 550             CTLFLAG_RD|CTLFLAG_WR|CTLFLAG_ANYBODY,
 551             (void*)REQ_PERIOD,
 552             sizeof(uint64_t), kpc_sysctl,
 553             "QU", "Set counter periods");
 554
 555 SYSCTL_PROC(_kpc, OID_AUTO, actionid,
 556             CTLFLAG_RD|CTLFLAG_WR|CTLFLAG_ANYBODY,
 557             (void*)REQ_ACTIONID,
 558             sizeof(uint32_t), kpc_sysctl,
 559             "QU", "Set counter actionids");
 560
 561