osfmk/kern/kpc_common.c

   1 /*
   2  * Copyright (c) 2012 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <mach/mach_types.h>
  30 #include <machine/machine_routines.h>
  31 #include <kern/processor.h>
  32 #include <kern/kalloc.h>
  33 #include <sys/errno.h>
  34 #include <sys/vm.h>
  35 #include <kperf/buffer.h>
  36 #include <kern/thread.h>
  37 #if defined(__arm64__) || defined(__arm__)
  38 #include <arm/cpu_data_internal.h>
  39 #endif
  40
  41 #include <kern/kpc.h>
  42
  43 #include <kperf/kperf.h>
  44 #include <kperf/sample.h>
  45 #include <kperf/context.h>
  46 #include <kperf/action.h>
  47
  48 uint32_t kpc_actionid[KPC_MAX_COUNTERS];
  49
  50 #define COUNTERBUF_SIZE_PER_CPU (KPC_MAX_COUNTERS * sizeof(uint64_t))
  51 #define COUNTERBUF_SIZE (machine_info.logical_cpu_max * \
  52                          COUNTERBUF_SIZE_PER_CPU)
  53
  54 /* locks */
  55 static lck_grp_attr_t *kpc_config_lckgrp_attr = NULL;
  56 static lck_grp_t      *kpc_config_lckgrp = NULL;
  57 static lck_mtx_t       kpc_config_lock;
  58
  59 /* state specifying if all counters have been requested by kperf */
  60 static boolean_t force_all_ctrs = FALSE;
  61
  62 /* power manager */
  63 static kpc_pm_handler_t kpc_pm_handler;
  64 static boolean_t kpc_pm_has_custom_config;
  65 static uint64_t kpc_pm_pmc_mask;
  66 #if MACH_ASSERT
  67 static bool kpc_calling_pm = false;
  68 #endif /* MACH_ASSERT */
  69
  70 boolean_t kpc_context_switch_active = FALSE;
  71 bool kpc_supported = true;
  72
  73 void
  74 kpc_common_init(void)
  75 {
  76         kpc_config_lckgrp_attr = lck_grp_attr_alloc_init();
  77         kpc_config_lckgrp = lck_grp_alloc_init("kpc", kpc_config_lckgrp_attr);
  78         lck_mtx_init(&kpc_config_lock, kpc_config_lckgrp, LCK_ATTR_NULL);
  79 }
  80
  81 boolean_t
  82 kpc_register_cpu(struct cpu_data *cpu_data)
  83 {
  84         assert(cpu_data);
  85         assert(cpu_data->cpu_kpc_buf[0] == NULL);
  86         assert(cpu_data->cpu_kpc_buf[1] == NULL);
  87         assert(cpu_data->cpu_kpc_shadow == NULL);
  88         assert(cpu_data->cpu_kpc_reload == NULL);
  89
  90         /*
  91          * Buffers allocated through kpc_counterbuf_alloc() are large enough to
  92          * store all PMCs values from all CPUs. This mimics the userspace API.
  93          * This does not suit well with the per-CPU kpc buffers, since:
  94          *      1. Buffers don't need to be this large.
  95          *      2. The actual number of CPUs is not known at this point.
  96          *
  97          * CPUs are asked to callout into kpc when being registered, we'll
  98          * allocate the memory here.
  99          */
 100
 101         if ((cpu_data->cpu_kpc_buf[0] = kalloc(COUNTERBUF_SIZE_PER_CPU)) == NULL) {
 102                 goto error;
 103         }
 104         if ((cpu_data->cpu_kpc_buf[1] = kalloc(COUNTERBUF_SIZE_PER_CPU)) == NULL) {
 105                 goto error;
 106         }
 107         if ((cpu_data->cpu_kpc_shadow = kalloc(COUNTERBUF_SIZE_PER_CPU)) == NULL) {
 108                 goto error;
 109         }
 110         if ((cpu_data->cpu_kpc_reload = kalloc(COUNTERBUF_SIZE_PER_CPU)) == NULL) {
 111                 goto error;
 112         }
 113
 114         memset(cpu_data->cpu_kpc_buf[0], 0, COUNTERBUF_SIZE_PER_CPU);
 115         memset(cpu_data->cpu_kpc_buf[1], 0, COUNTERBUF_SIZE_PER_CPU);
 116         memset(cpu_data->cpu_kpc_shadow, 0, COUNTERBUF_SIZE_PER_CPU);
 117         memset(cpu_data->cpu_kpc_reload, 0, COUNTERBUF_SIZE_PER_CPU);
 118
 119         /* success */
 120         return TRUE;
 121
 122 error:
 123         kpc_unregister_cpu(cpu_data);
 124         return FALSE;
 125 }
 126
 127 void
 128 kpc_unregister_cpu(struct cpu_data *cpu_data)
 129 {
 130         assert(cpu_data);
 131         if (cpu_data->cpu_kpc_buf[0] != NULL) {
 132                 kfree(cpu_data->cpu_kpc_buf[0], COUNTERBUF_SIZE_PER_CPU);
 133                 cpu_data->cpu_kpc_buf[0] = NULL;
 134         }
 135         if (cpu_data->cpu_kpc_buf[1] != NULL) {
 136                 kfree(cpu_data->cpu_kpc_buf[1], COUNTERBUF_SIZE_PER_CPU);
 137                 cpu_data->cpu_kpc_buf[1] = NULL;
 138         }
 139         if (cpu_data->cpu_kpc_shadow != NULL) {
 140                 kfree(cpu_data->cpu_kpc_shadow, COUNTERBUF_SIZE_PER_CPU);
 141                 cpu_data->cpu_kpc_shadow = NULL;
 142         }
 143         if (cpu_data->cpu_kpc_reload != NULL) {
 144                 kfree(cpu_data->cpu_kpc_reload, COUNTERBUF_SIZE_PER_CPU);
 145                 cpu_data->cpu_kpc_reload = NULL;
 146         }
 147 }
 148
 149
 150 static void
 151 kpc_task_set_forced_all_ctrs(task_t task, boolean_t state)
 152 {
 153         assert(task);
 154
 155         task_lock(task);
 156         if (state) {
 157                 task->t_kpc |= TASK_KPC_FORCED_ALL_CTRS;
 158         } else {
 159                 task->t_kpc &= ~TASK_KPC_FORCED_ALL_CTRS;
 160         }
 161         task_unlock(task);
 162 }
 163
 164 static boolean_t
 165 kpc_task_get_forced_all_ctrs(task_t task)
 166 {
 167         assert(task);
 168         return task->t_kpc & TASK_KPC_FORCED_ALL_CTRS ? TRUE : FALSE;
 169 }
 170
 171 int
 172 kpc_force_all_ctrs(task_t task, int val)
 173 {
 174         boolean_t new_state = val ? TRUE : FALSE;
 175         boolean_t old_state = kpc_get_force_all_ctrs();
 176
 177         /*
 178          * Refuse to do the operation if the counters are already forced by
 179          * another task.
 180          */
 181         if (kpc_get_force_all_ctrs() && !kpc_task_get_forced_all_ctrs(task)) {
 182                 return EACCES;
 183         }
 184
 185         /* nothing to do if the state is not changing */
 186         if (old_state == new_state) {
 187                 return 0;
 188         }
 189
 190         /* notify the power manager */
 191         if (kpc_pm_handler) {
 192 #if MACH_ASSERT
 193                 kpc_calling_pm = true;
 194 #endif /* MACH_ASSERT */
 195                 kpc_pm_handler( new_state ? FALSE : TRUE );
 196 #if MACH_ASSERT
 197                 kpc_calling_pm = false;
 198 #endif /* MACH_ASSERT */
 199         }
 200
 201         /*
 202          * This is a force -- ensure that counters are forced, even if power
 203          * management fails to acknowledge it.
 204          */
 205         if (force_all_ctrs != new_state) {
 206                 force_all_ctrs = new_state;
 207         }
 208
 209         /* update the task bits */
 210         kpc_task_set_forced_all_ctrs(task, new_state);
 211
 212         return 0;
 213 }
 214
 215 void
 216 kpc_pm_acknowledge(boolean_t available_to_pm)
 217 {
 218         /*
 219          * Force-all-counters should still be true when the counters are being
 220          * made available to power management and false when counters are going
 221          * to be taken away.
 222          */
 223         assert(force_all_ctrs == available_to_pm);
 224         /*
 225          * Make sure power management isn't playing games with us.
 226          */
 227         assert(kpc_calling_pm == true);
 228
 229         /*
 230          * Counters being available means no one is forcing all counters.
 231          */
 232         force_all_ctrs = available_to_pm ? FALSE : TRUE;
 233 }
 234
 235 int
 236 kpc_get_force_all_ctrs(void)
 237 {
 238         return force_all_ctrs;
 239 }
 240
 241 boolean_t
 242 kpc_multiple_clients(void)
 243 {
 244         return kpc_pm_handler != NULL;
 245 }
 246
 247 boolean_t
 248 kpc_controls_fixed_counters(void)
 249 {
 250         return !kpc_pm_handler || force_all_ctrs || !kpc_pm_has_custom_config;
 251 }
 252
 253 boolean_t
 254 kpc_controls_counter(uint32_t ctr)
 255 {
 256         uint64_t pmc_mask = 0ULL;
 257
 258         assert(ctr < (kpc_fixed_count() + kpc_configurable_count()));
 259
 260         if (ctr < kpc_fixed_count()) {
 261                 return kpc_controls_fixed_counters();
 262         }
 263
 264         /*
 265          * By default kpc manages all PMCs, but if the Power Manager registered
 266          * with custom_config=TRUE, the Power Manager manages its reserved PMCs.
 267          * However, kpc takes ownership back if a task acquired all PMCs via
 268          * force_all_ctrs.
 269          */
 270         pmc_mask = (1ULL << (ctr - kpc_fixed_count()));
 271         if ((pmc_mask & kpc_pm_pmc_mask) && kpc_pm_has_custom_config && !force_all_ctrs) {
 272                 return FALSE;
 273         }
 274
 275         return TRUE;
 276 }
 277
 278 uint32_t
 279 kpc_get_running(void)
 280 {
 281         uint64_t pmc_mask = 0;
 282         uint32_t cur_state = 0;
 283
 284         if (kpc_is_running_fixed()) {
 285                 cur_state |= KPC_CLASS_FIXED_MASK;
 286         }
 287
 288         pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_CONFIGURABLE_MASK);
 289         if (kpc_is_running_configurable(pmc_mask)) {
 290                 cur_state |= KPC_CLASS_CONFIGURABLE_MASK;
 291         }
 292
 293         pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_POWER_MASK);
 294         if ((pmc_mask != 0) && kpc_is_running_configurable(pmc_mask)) {
 295                 cur_state |= KPC_CLASS_POWER_MASK;
 296         }
 297
 298         return cur_state;
 299 }
 300
 301 /* may be called from an IPI */
 302 int
 303 kpc_get_curcpu_counters(uint32_t classes, int *curcpu, uint64_t *buf)
 304 {
 305         int enabled = 0, offset = 0;
 306         uint64_t pmc_mask = 0ULL;
 307
 308         assert(buf);
 309
 310         enabled = ml_set_interrupts_enabled(FALSE);
 311
 312         /* grab counters and CPU number as close as possible */
 313         if (curcpu) {
 314                 *curcpu = current_processor()->cpu_id;
 315         }
 316
 317         if (classes & KPC_CLASS_FIXED_MASK) {
 318                 kpc_get_fixed_counters(&buf[offset]);
 319                 offset += kpc_get_counter_count(KPC_CLASS_FIXED_MASK);
 320         }
 321
 322         if (classes & KPC_CLASS_CONFIGURABLE_MASK) {
 323                 pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_CONFIGURABLE_MASK);
 324                 kpc_get_configurable_counters(&buf[offset], pmc_mask);
 325                 offset += kpc_popcount(pmc_mask);
 326         }
 327
 328         if (classes & KPC_CLASS_POWER_MASK) {
 329                 pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_POWER_MASK);
 330                 kpc_get_configurable_counters(&buf[offset], pmc_mask);
 331                 offset += kpc_popcount(pmc_mask);
 332         }
 333
 334         ml_set_interrupts_enabled(enabled);
 335
 336         return offset;
 337 }
 338
 339 /* generic counter reading function, public api */
 340 int
 341 kpc_get_cpu_counters(boolean_t all_cpus, uint32_t classes,
 342     int *curcpu, uint64_t *buf)
 343 {
 344         assert(buf);
 345
 346         /*
 347          * Unlike reading the current CPU counters, reading counters from all
 348          * CPUs is architecture dependent. This allows kpc to make the most of
 349          * the platform if memory mapped registers is supported.
 350          */
 351         if (all_cpus) {
 352                 return kpc_get_all_cpus_counters(classes, curcpu, buf);
 353         } else {
 354                 return kpc_get_curcpu_counters(classes, curcpu, buf);
 355         }
 356 }
 357
 358 int
 359 kpc_get_shadow_counters(boolean_t all_cpus, uint32_t classes,
 360     int *curcpu, uint64_t *buf)
 361 {
 362         int curcpu_id = current_processor()->cpu_id;
 363         uint32_t cfg_count = kpc_configurable_count(), offset = 0;
 364         uint64_t pmc_mask = 0ULL;
 365         boolean_t enabled;
 366
 367         assert(buf);
 368
 369         enabled = ml_set_interrupts_enabled(FALSE);
 370
 371         curcpu_id = current_processor()->cpu_id;
 372         if (curcpu) {
 373                 *curcpu = curcpu_id;
 374         }
 375
 376         for (int cpu = 0; cpu < machine_info.logical_cpu_max; ++cpu) {
 377                 /* filter if the caller did not request all cpus */
 378                 if (!all_cpus && (cpu != curcpu_id)) {
 379                         continue;
 380                 }
 381
 382                 if (classes & KPC_CLASS_FIXED_MASK) {
 383                         uint32_t count = kpc_get_counter_count(KPC_CLASS_FIXED_MASK);
 384                         memcpy(&buf[offset], &FIXED_SHADOW_CPU(cpu, 0), count * sizeof(uint64_t));
 385                         offset += count;
 386                 }
 387
 388                 if (classes & KPC_CLASS_CONFIGURABLE_MASK) {
 389                         pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_CONFIGURABLE_MASK);
 390
 391                         for (uint32_t cfg_ctr = 0; cfg_ctr < cfg_count; ++cfg_ctr) {
 392                                 if ((1ULL << cfg_ctr) & pmc_mask) {
 393                                         buf[offset++] = CONFIGURABLE_SHADOW_CPU(cpu, cfg_ctr);
 394                                 }
 395                         }
 396                 }
 397
 398                 if (classes & KPC_CLASS_POWER_MASK) {
 399                         pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_POWER_MASK);
 400
 401                         for (uint32_t cfg_ctr = 0; cfg_ctr < cfg_count; ++cfg_ctr) {
 402                                 if ((1ULL << cfg_ctr) & pmc_mask) {
 403                                         buf[offset++] = CONFIGURABLE_SHADOW_CPU(cpu, cfg_ctr);
 404                                 }
 405                         }
 406                 }
 407         }
 408
 409         ml_set_interrupts_enabled(enabled);
 410
 411         return offset;
 412 }
 413
 414 uint32_t
 415 kpc_get_counter_count(uint32_t classes)
 416 {
 417         uint32_t count = 0;
 418
 419         if (classes & KPC_CLASS_FIXED_MASK) {
 420                 count += kpc_fixed_count();
 421         }
 422
 423         if (classes & (KPC_CLASS_CONFIGURABLE_MASK | KPC_CLASS_POWER_MASK)) {
 424                 uint64_t pmc_msk = kpc_get_configurable_pmc_mask(classes);
 425                 uint32_t pmc_cnt = kpc_popcount(pmc_msk);
 426                 count += pmc_cnt;
 427         }
 428
 429         return count;
 430 }
 431
 432 uint32_t
 433 kpc_get_config_count(uint32_t classes)
 434 {
 435         uint32_t count = 0;
 436
 437         if (classes & KPC_CLASS_FIXED_MASK) {
 438                 count += kpc_fixed_config_count();
 439         }
 440
 441         if (classes & (KPC_CLASS_CONFIGURABLE_MASK | KPC_CLASS_POWER_MASK)) {
 442                 uint64_t pmc_mask = kpc_get_configurable_pmc_mask(classes);
 443                 count += kpc_configurable_config_count(pmc_mask);
 444         }
 445
 446         if ((classes & KPC_CLASS_RAWPMU_MASK) && !kpc_multiple_clients()) {
 447                 count += kpc_rawpmu_config_count();
 448         }
 449
 450         return count;
 451 }
 452
 453 int
 454 kpc_get_config(uint32_t classes, kpc_config_t *current_config)
 455 {
 456         uint32_t count = 0;
 457
 458         assert(current_config);
 459
 460         if (classes & KPC_CLASS_FIXED_MASK) {
 461                 kpc_get_fixed_config(&current_config[count]);
 462                 count += kpc_get_config_count(KPC_CLASS_FIXED_MASK);
 463         }
 464
 465         if (classes & KPC_CLASS_CONFIGURABLE_MASK) {
 466                 uint64_t pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_CONFIGURABLE_MASK);
 467                 kpc_get_configurable_config(&current_config[count], pmc_mask);
 468                 count += kpc_get_config_count(KPC_CLASS_CONFIGURABLE_MASK);
 469         }
 470
 471         if (classes & KPC_CLASS_POWER_MASK) {
 472                 uint64_t pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_POWER_MASK);
 473                 kpc_get_configurable_config(&current_config[count], pmc_mask);
 474                 count += kpc_get_config_count(KPC_CLASS_POWER_MASK);
 475         }
 476
 477         if (classes & KPC_CLASS_RAWPMU_MASK) {
 478                 // Client shouldn't ask for config words that aren't available.
 479                 // Most likely, they'd misinterpret the returned buffer if we
 480                 // allowed this.
 481                 if (kpc_multiple_clients()) {
 482                         return EPERM;
 483                 }
 484                 kpc_get_rawpmu_config(&current_config[count]);
 485                 count += kpc_get_config_count(KPC_CLASS_RAWPMU_MASK);
 486         }
 487
 488         return 0;
 489 }
 490
 491 int
 492 kpc_set_config(uint32_t classes, kpc_config_t *configv)
 493 {
 494         int ret = 0;
 495         struct kpc_config_remote mp_config = {
 496                 .classes = classes, .configv = configv,
 497                 .pmc_mask = kpc_get_configurable_pmc_mask(classes)
 498         };
 499
 500         assert(configv);
 501
 502         /* don't allow RAWPMU configuration when sharing counters */
 503         if ((classes & KPC_CLASS_RAWPMU_MASK) && kpc_multiple_clients()) {
 504                 return EPERM;
 505         }
 506
 507         /* no clients have the right to modify both classes */
 508         if ((classes & (KPC_CLASS_CONFIGURABLE_MASK)) &&
 509             (classes & (KPC_CLASS_POWER_MASK))) {
 510                 return EPERM;
 511         }
 512
 513         lck_mtx_lock(&kpc_config_lock);
 514
 515         /* translate the power class for the machine layer */
 516         if (classes & KPC_CLASS_POWER_MASK) {
 517                 mp_config.classes |= KPC_CLASS_CONFIGURABLE_MASK;
 518         }
 519
 520         ret = kpc_set_config_arch( &mp_config );
 521
 522         lck_mtx_unlock(&kpc_config_lock);
 523
 524         return ret;
 525 }
 526
 527 uint32_t
 528 kpc_get_counterbuf_size(void)
 529 {
 530         return COUNTERBUF_SIZE;
 531 }
 532
 533 /* allocate a buffer large enough for all possible counters */
 534 uint64_t *
 535 kpc_counterbuf_alloc(void)
 536 {
 537         uint64_t *buf = NULL;
 538
 539         buf = kalloc_tag(COUNTERBUF_SIZE, VM_KERN_MEMORY_DIAG);
 540         if (buf) {
 541                 bzero(buf, COUNTERBUF_SIZE);
 542         }
 543
 544         return buf;
 545 }
 546
 547 void
 548 kpc_counterbuf_free(uint64_t *buf)
 549 {
 550         if (buf) {
 551                 kfree(buf, COUNTERBUF_SIZE);
 552         }
 553 }
 554
 555 void
 556 kpc_sample_kperf(uint32_t actionid)
 557 {
 558         struct kperf_sample sbuf;
 559
 560         BUF_DATA(PERF_KPC_HNDLR | DBG_FUNC_START);
 561
 562         thread_t thread = current_thread();
 563         task_t task = get_threadtask(thread);
 564
 565         struct kperf_context ctx = {
 566                 .cur_thread = thread,
 567                 .cur_task = task,
 568                 .cur_pid = task_pid(task),
 569                 .trigger_type = TRIGGER_TYPE_PMI,
 570                 .trigger_id = 0,
 571         };
 572
 573         int r = kperf_sample(&sbuf, &ctx, actionid, SAMPLE_FLAG_PEND_USER);
 574
 575         BUF_INFO(PERF_KPC_HNDLR | DBG_FUNC_END, r);
 576 }
 577
 578
 579 int
 580 kpc_set_period(uint32_t classes, uint64_t *val)
 581 {
 582         struct kpc_config_remote mp_config = {
 583                 .classes = classes, .configv = val,
 584                 .pmc_mask = kpc_get_configurable_pmc_mask(classes)
 585         };
 586
 587         assert(val);
 588
 589         /* no clients have the right to modify both classes */
 590         if ((classes & (KPC_CLASS_CONFIGURABLE_MASK)) &&
 591             (classes & (KPC_CLASS_POWER_MASK))) {
 592                 return EPERM;
 593         }
 594
 595         lck_mtx_lock(&kpc_config_lock);
 596
 597 #ifdef FIXED_COUNTER_SHADOW
 598         if ((classes & KPC_CLASS_FIXED_MASK) && !kpc_controls_fixed_counters()) {
 599                 lck_mtx_unlock(&kpc_config_lock);
 600                 return EPERM;
 601         }
 602 # else
 603         if (classes & KPC_CLASS_FIXED_MASK) {
 604                 lck_mtx_unlock(&kpc_config_lock);
 605                 return EINVAL;
 606         }
 607 #endif
 608
 609         /* translate the power class for the machine layer */
 610         if (classes & KPC_CLASS_POWER_MASK) {
 611                 mp_config.classes |= KPC_CLASS_CONFIGURABLE_MASK;
 612         }
 613
 614         kprintf("setting period %u\n", classes);
 615         kpc_set_period_arch( &mp_config );
 616
 617         lck_mtx_unlock(&kpc_config_lock);
 618
 619         return 0;
 620 }
 621
 622 int
 623 kpc_get_period(uint32_t classes, uint64_t *val)
 624 {
 625         uint32_t count = 0;
 626         uint64_t pmc_mask = 0ULL;
 627
 628         assert(val);
 629
 630         lck_mtx_lock(&kpc_config_lock);
 631
 632         if (classes & KPC_CLASS_FIXED_MASK) {
 633                 /* convert reload values to periods */
 634                 count = kpc_get_counter_count(KPC_CLASS_FIXED_MASK);
 635                 for (uint32_t i = 0; i < count; ++i) {
 636                         *val++ = kpc_fixed_max() - FIXED_RELOAD(i);
 637                 }
 638         }
 639
 640         if (classes & KPC_CLASS_CONFIGURABLE_MASK) {
 641                 pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_CONFIGURABLE_MASK);
 642
 643                 /* convert reload values to periods */
 644                 count = kpc_configurable_count();
 645                 for (uint32_t i = 0; i < count; ++i) {
 646                         if ((1ULL << i) & pmc_mask) {
 647                                 *val++ = kpc_configurable_max() - CONFIGURABLE_RELOAD(i);
 648                         }
 649                 }
 650         }
 651
 652         if (classes & KPC_CLASS_POWER_MASK) {
 653                 pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_POWER_MASK);
 654
 655                 /* convert reload values to periods */
 656                 count = kpc_configurable_count();
 657                 for (uint32_t i = 0; i < count; ++i) {
 658                         if ((1ULL << i) & pmc_mask) {
 659                                 *val++ = kpc_configurable_max() - CONFIGURABLE_RELOAD(i);
 660                         }
 661                 }
 662         }
 663
 664         lck_mtx_unlock(&kpc_config_lock);
 665
 666         return 0;
 667 }
 668
 669 int
 670 kpc_set_actionid(uint32_t classes, uint32_t *val)
 671 {
 672         uint32_t count = 0;
 673         uint64_t pmc_mask = 0ULL;
 674
 675         assert(val);
 676
 677         /* NOTE: what happens if a pmi occurs while actionids are being
 678          * set is undefined. */
 679         lck_mtx_lock(&kpc_config_lock);
 680
 681         if (classes & KPC_CLASS_FIXED_MASK) {
 682                 count = kpc_get_counter_count(KPC_CLASS_FIXED_MASK);
 683                 memcpy(&FIXED_ACTIONID(0), val, count * sizeof(uint32_t));
 684                 val += count;
 685         }
 686
 687         if (classes & KPC_CLASS_CONFIGURABLE_MASK) {
 688                 pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_CONFIGURABLE_MASK);
 689
 690                 count = kpc_configurable_count();
 691                 for (uint32_t i = 0; i < count; ++i) {
 692                         if ((1ULL << i) & pmc_mask) {
 693                                 CONFIGURABLE_ACTIONID(i) = *val++;
 694                         }
 695                 }
 696         }
 697
 698         if (classes & KPC_CLASS_POWER_MASK) {
 699                 pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_POWER_MASK);
 700
 701                 count = kpc_configurable_count();
 702                 for (uint32_t i = 0; i < count; ++i) {
 703                         if ((1ULL << i) & pmc_mask) {
 704                                 CONFIGURABLE_ACTIONID(i) = *val++;
 705                         }
 706                 }
 707         }
 708
 709         lck_mtx_unlock(&kpc_config_lock);
 710
 711         return 0;
 712 }
 713
 714 int
 715 kpc_get_actionid(uint32_t classes, uint32_t *val)
 716 {
 717         uint32_t count = 0;
 718         uint64_t pmc_mask = 0ULL;
 719
 720         assert(val);
 721
 722         lck_mtx_lock(&kpc_config_lock);
 723
 724         if (classes & KPC_CLASS_FIXED_MASK) {
 725                 count = kpc_get_counter_count(KPC_CLASS_FIXED_MASK);
 726                 memcpy(val, &FIXED_ACTIONID(0), count * sizeof(uint32_t));
 727                 val += count;
 728         }
 729
 730         if (classes & KPC_CLASS_CONFIGURABLE_MASK) {
 731                 pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_CONFIGURABLE_MASK);
 732
 733                 count = kpc_configurable_count();
 734                 for (uint32_t i = 0; i < count; ++i) {
 735                         if ((1ULL << i) & pmc_mask) {
 736                                 *val++ = CONFIGURABLE_ACTIONID(i);
 737                         }
 738                 }
 739         }
 740
 741         if (classes & KPC_CLASS_POWER_MASK) {
 742                 pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_POWER_MASK);
 743
 744                 count = kpc_configurable_count();
 745                 for (uint32_t i = 0; i < count; ++i) {
 746                         if ((1ULL << i) & pmc_mask) {
 747                                 *val++ = CONFIGURABLE_ACTIONID(i);
 748                         }
 749                 }
 750         }
 751
 752         lck_mtx_unlock(&kpc_config_lock);
 753
 754         return 0;
 755 }
 756
 757 int
 758 kpc_set_running(uint32_t classes)
 759 {
 760         uint32_t all_cfg_classes = KPC_CLASS_CONFIGURABLE_MASK | KPC_CLASS_POWER_MASK;
 761         struct kpc_running_remote mp_config = {
 762                 .classes = classes, .cfg_target_mask = 0ULL, .cfg_state_mask = 0ULL
 763         };
 764
 765         /* target all available PMCs */
 766         mp_config.cfg_target_mask = kpc_get_configurable_pmc_mask(all_cfg_classes);
 767
 768         /* translate the power class for the machine layer */
 769         if (classes & KPC_CLASS_POWER_MASK) {
 770                 mp_config.classes |= KPC_CLASS_CONFIGURABLE_MASK;
 771         }
 772
 773         /* generate the state of each configurable PMCs */
 774         mp_config.cfg_state_mask = kpc_get_configurable_pmc_mask(classes);
 775
 776         return kpc_set_running_arch(&mp_config);
 777 }
 778
 779 boolean_t
 780 kpc_register_pm_handler(kpc_pm_handler_t handler)
 781 {
 782         return kpc_reserve_pm_counters(0x38, handler, TRUE);
 783 }
 784
 785 boolean_t
 786 kpc_reserve_pm_counters(uint64_t pmc_mask, kpc_pm_handler_t handler,
 787     boolean_t custom_config)
 788 {
 789         uint64_t all_mask = (1ULL << kpc_configurable_count()) - 1;
 790         uint64_t req_mask = 0ULL;
 791
 792         /* pre-condition */
 793         assert(handler != NULL);
 794         assert(kpc_pm_handler == NULL);
 795
 796         /* check number of counters requested */
 797         req_mask = (pmc_mask & all_mask);
 798         assert(kpc_popcount(req_mask) <= kpc_configurable_count());
 799
 800         /* save the power manager states */
 801         kpc_pm_has_custom_config = custom_config;
 802         kpc_pm_pmc_mask = req_mask;
 803         kpc_pm_handler = handler;
 804
 805         printf("kpc: pm registered pmc_mask=%llx custom_config=%d\n",
 806             req_mask, custom_config);
 807
 808         /* post-condition */
 809         {
 810                 uint32_t cfg_count = kpc_get_counter_count(KPC_CLASS_CONFIGURABLE_MASK);
 811                 uint32_t pwr_count = kpc_popcount(kpc_pm_pmc_mask);
 812 #pragma unused(cfg_count, pwr_count)
 813                 assert((cfg_count + pwr_count) == kpc_configurable_count());
 814         }
 815
 816         return force_all_ctrs ? FALSE : TRUE;
 817 }
 818
 819 void
 820 kpc_release_pm_counters(void)
 821 {
 822         /* pre-condition */
 823         assert(kpc_pm_handler != NULL);
 824
 825         /* release the counters */
 826         kpc_pm_has_custom_config = FALSE;
 827         kpc_pm_pmc_mask = 0ULL;
 828         kpc_pm_handler = NULL;
 829
 830         printf("kpc: pm released counters\n");
 831
 832         /* post-condition */
 833         assert(kpc_get_counter_count(KPC_CLASS_CONFIGURABLE_MASK) == kpc_configurable_count());
 834 }
 835
 836 uint8_t
 837 kpc_popcount(uint64_t value)
 838 {
 839         return __builtin_popcountll(value);
 840 }
 841
 842 uint64_t
 843 kpc_get_configurable_pmc_mask(uint32_t classes)
 844 {
 845         uint32_t configurable_count = kpc_configurable_count();
 846         uint64_t cfg_mask = 0ULL, pwr_mask = 0ULL, all_cfg_pmcs_mask = 0ULL;
 847
 848         /* not configurable classes or no configurable counters */
 849         if (((classes & (KPC_CLASS_CONFIGURABLE_MASK | KPC_CLASS_POWER_MASK)) == 0) ||
 850             (configurable_count == 0)) {
 851                 goto exit;
 852         }
 853
 854         assert(configurable_count < 64);
 855         all_cfg_pmcs_mask = (1ULL << configurable_count) - 1;
 856
 857         if (classes & KPC_CLASS_CONFIGURABLE_MASK) {
 858                 if (force_all_ctrs == TRUE) {
 859                         cfg_mask |= all_cfg_pmcs_mask;
 860                 } else {
 861                         cfg_mask |= (~kpc_pm_pmc_mask) & all_cfg_pmcs_mask;
 862                 }
 863         }
 864
 865         /*
 866          * The power class exists iff:
 867          *      - No tasks acquired all PMCs
 868          *      - PM registered and uses kpc to interact with PMCs
 869          */
 870         if ((force_all_ctrs == FALSE) &&
 871             (kpc_pm_handler != NULL) &&
 872             (kpc_pm_has_custom_config == FALSE) &&
 873             (classes & KPC_CLASS_POWER_MASK)) {
 874                 pwr_mask |= kpc_pm_pmc_mask & all_cfg_pmcs_mask;
 875         }
 876
 877 exit:
 878         /* post-conditions */
 879         assert(((cfg_mask | pwr_mask) & (~all_cfg_pmcs_mask)) == 0 );
 880         assert( kpc_popcount(cfg_mask | pwr_mask) <= kpc_configurable_count());
 881         assert((cfg_mask & pwr_mask) == 0ULL );
 882
 883         return cfg_mask | pwr_mask;
 884 }