osfmk/arm/kpc_arm.c

   1 /*
   2  * Copyright (c) 2012 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <mach/mach_types.h>
  30 #include <machine/machine_routines.h>
  31 #include <kern/processor.h>
  32 #include <kern/thread.h>
  33 #include <sys/errno.h>
  34 #include <arm/cpu_data_internal.h>
  35 #include <arm/cpu_internal.h>
  36 #include <kern/kpc.h>
  37
  38 #ifdef ARMA7
  39 /* PMU v2 based implementation for A7 */
  40 static uint32_t saved_PMXEVTYPER[MAX_CPUS][KPC_ARM_TOTAL_COUNT];
  41 static uint32_t saved_PMCNTENSET[MAX_CPUS];
  42 static uint64_t saved_counter[MAX_CPUS][KPC_ARM_TOTAL_COUNT];
  43 static uint32_t saved_PMOVSR[MAX_CPUS];
  44
  45 static uint32_t kpc_configured = 0;
  46 static uint32_t kpc_xcall_sync;
  47 static uint64_t kpc_running_cfg_pmc_mask = 0;
  48 static uint32_t kpc_running_classes = 0;
  49 static uint32_t kpc_reload_sync;
  50 static uint32_t kpc_enabled_counters = 0;
  51
  52 static int first_time = 1;
  53
  54 /* Private */
  55
  56 static uint64_t get_counter_config(uint32_t counter);
  57
  58
  59 static boolean_t
  60 enable_counter(uint32_t counter)
  61 {
  62         boolean_t enabled;
  63         uint32_t PMCNTENSET;
  64         /* Cycle counter is MSB; configurable counters reside in LSBs */
  65         uint32_t mask = (counter == 0) ? (1 << 31) : (1 << (counter - 1));
  66
  67         /* Enabled? */
  68         __asm__ volatile ("mrc p15, 0, %0, c9, c12, 1;" : "=r" (PMCNTENSET));
  69
  70         enabled = (PMCNTENSET & mask);
  71         if (!enabled) {
  72                 /* Counter interrupt enable (PMINTENSET) */
  73                 __asm__ volatile ("mcr p15, 0, %0, c9, c14, 1;" : : "r" (mask));
  74
  75                 /* Individual counter enable set (PMCNTENSET) */
  76                 __asm__ volatile ("mcr p15, 0, %0, c9, c12, 1;" : : "r" (mask));
  77
  78                 kpc_enabled_counters++;
  79
  80                 /* 1st enabled counter? Set the master enable bit in PMCR */
  81                 if (kpc_enabled_counters == 1) {
  82                         uint32_t PMCR = 1;
  83                         __asm__ volatile ("mcr p15, 0, %0, c9, c12, 0;" : : "r" (PMCR));
  84                 }
  85         }
  86
  87         return enabled;
  88 }
  89
  90 static boolean_t
  91 disable_counter(uint32_t counter)
  92 {
  93         boolean_t enabled;
  94         uint32_t PMCNTENCLR;
  95         /* Cycle counter is MSB; configurable counters reside in LSBs */
  96         uint32_t mask = (counter == 0) ? (1 << 31) : (1 << (counter - 1));
  97
  98         /* Enabled? */
  99         __asm__ volatile ("mrc p15, 0, %0, c9, c12, 2;" : "=r" (PMCNTENCLR));
 100
 101         enabled = (PMCNTENCLR & mask);
 102         if (enabled) {
 103                 /* Individual counter enable clear (PMCNTENCLR) */
 104                 __asm__ volatile ("mcr p15, 0, %0, c9, c12, 2;" : : "r" (mask));
 105
 106                 /* Counter interrupt disable (PMINTENCLR) */
 107                 __asm__ volatile ("mcr p15, 0, %0, c9, c14, 2;" : : "r" (mask));
 108
 109                 kpc_enabled_counters--;
 110
 111                 /* Last enabled counter? Clear the master enable bit in PMCR */
 112                 if (kpc_enabled_counters == 0) {
 113                         uint32_t PMCR = 0;
 114                         __asm__ volatile ("mcr p15, 0, %0, c9, c12, 0;" : : "r" (PMCR));
 115                 }
 116         }
 117
 118         return enabled;
 119 }
 120
 121 static uint64_t
 122 read_counter(uint32_t counter)
 123 {
 124         uint32_t low = 0;
 125
 126         switch (counter) {
 127         case 0:
 128                 /* Fixed counter */
 129                 __asm__ volatile ("mrc p15, 0, %0, c9, c13, 0;" : "=r" (low));
 130                 break;
 131         case 1:
 132         case 2:
 133         case 3:
 134         case 4:
 135                 /* Configurable. Set PMSELR... */
 136                 __asm__ volatile ("mcr p15, 0, %0, c9, c12, 5;" : : "r" (counter - 1));
 137                 /* ...then read PMXEVCNTR */
 138                 __asm__ volatile ("mrc p15, 0, %0, c9, c13, 2;" : "=r" (low));
 139                 break;
 140         default:
 141                 /* ??? */
 142                 break;
 143         }
 144
 145         return (uint64_t)low;
 146 }
 147
 148 static void
 149 write_counter(uint32_t counter, uint64_t value)
 150 {
 151         uint32_t low = value & 0xFFFFFFFF;
 152
 153         switch (counter) {
 154         case 0:
 155                 /* Fixed counter */
 156                 __asm__ volatile ("mcr p15, 0, %0, c9, c13, 0;" : : "r" (low));
 157                 break;
 158         case 1:
 159         case 2:
 160         case 3:
 161         case 4:
 162                 /* Configurable. Set PMSELR... */
 163                 __asm__ volatile ("mcr p15, 0, %0, c9, c12, 5;" : : "r" (counter - 1));
 164                 /* ...then write PMXEVCNTR */
 165                 __asm__ volatile ("mcr p15, 0, %0, c9, c13, 2;" : : "r" (low));
 166                 break;
 167         default:
 168                 /* ??? */
 169                 break;
 170         }
 171 }
 172
 173 static uint64_t
 174 kpc_reload_counter(int ctr)
 175 {
 176         uint64_t old = read_counter(ctr);
 177         write_counter(ctr, FIXED_RELOAD(ctr));
 178         return old;
 179 }
 180
 181 static void
 182 set_running_fixed(boolean_t on)
 183 {
 184         int i;
 185         boolean_t enabled;
 186         int n = KPC_ARM_FIXED_COUNT;
 187
 188         enabled = ml_set_interrupts_enabled(FALSE);
 189
 190         for (i = 0; i < n; i++) {
 191                 if (on) {
 192                         enable_counter(i);
 193                 } else {
 194                         disable_counter(i);
 195                 }
 196         }
 197
 198         ml_set_interrupts_enabled(enabled);
 199 }
 200
 201 static void
 202 set_running_configurable(uint64_t target_mask, uint64_t state_mask)
 203 {
 204         uint32_t cfg_count = kpc_configurable_count(), offset = kpc_fixed_count();
 205         boolean_t enabled;
 206
 207         enabled = ml_set_interrupts_enabled(FALSE);
 208
 209         for (uint32_t i = 0; i < cfg_count; ++i) {
 210                 if (((1ULL << i) & target_mask) == 0) {
 211                         continue;
 212                 }
 213                 assert(kpc_controls_counter(offset + i));
 214
 215                 if ((1ULL << i) & state_mask) {
 216                         enable_counter(offset + i);
 217                 } else {
 218                         disable_counter(offset + i);
 219                 }
 220         }
 221
 222         ml_set_interrupts_enabled(enabled);
 223 }
 224
 225 static uintptr_t
 226 get_interrupted_pc(bool *kernel_out)
 227 {
 228         struct arm_saved_state *state = getCpuDatap()->cpu_int_state;
 229         if (!state) {
 230                 return 0;
 231         }
 232
 233         bool kernel = !PSR_IS_USER(get_saved_state_cpsr(state));
 234         *kernel_out = kernel;
 235         uintptr_t pc = get_saved_state_pc(state);
 236         if (kernel) {
 237                 pc = VM_KERNEL_UNSLIDE(pc);
 238         }
 239         return pc;
 240 }
 241
 242 void kpc_pmi_handler(cpu_id_t source);
 243 void
 244 kpc_pmi_handler(cpu_id_t source)
 245 {
 246         uint64_t extra;
 247         int ctr;
 248         int enabled;
 249
 250         enabled = ml_set_interrupts_enabled(FALSE);
 251
 252         /* The pmi must be delivered to the CPU that generated it */
 253         if (source != getCpuDatap()->interrupt_nub) {
 254                 panic("pmi from IOCPU %p delivered to IOCPU %p", source, getCpuDatap()->interrupt_nub);
 255         }
 256
 257         for (ctr = 0;
 258             ctr < (KPC_ARM_FIXED_COUNT + KPC_ARM_CONFIGURABLE_COUNT);
 259             ctr++) {
 260                 uint32_t PMOVSR;
 261                 uint32_t mask;
 262
 263                 /* check the counter for overflow */
 264                 if (ctr == 0) {
 265                         mask = 1 << 31;
 266                 } else {
 267                         mask = 1 << (ctr - 1);
 268                 }
 269
 270                 /* read PMOVSR */
 271                 __asm__ volatile ("mrc p15, 0, %0, c9, c12, 3;" : "=r" (PMOVSR));
 272
 273                 if (PMOVSR & mask) {
 274                         extra = kpc_reload_counter(ctr);
 275
 276                         FIXED_SHADOW(ctr)
 277                                 += (kpc_fixed_max() - FIXED_RELOAD(ctr) + 1 /* wrap */) + extra;
 278
 279                         if (FIXED_ACTIONID(ctr)) {
 280                                 bool kernel = false;
 281                                 uintptr_t pc = get_interrupted_pc(&kernel);
 282                                 kpc_sample_kperf(FIXED_ACTIONID(ctr), ctr, get_counter_config(ctr),
 283                                     FIXED_SHADOW(ctr), pc, kernel ? KPC_KERNEL_PC : 0);
 284                         }
 285
 286                         /* clear PMOVSR bit */
 287                         __asm__ volatile ("mcr p15, 0, %0, c9, c12, 3;" : : "r" (mask));
 288                 }
 289         }
 290
 291         ml_set_interrupts_enabled(enabled);
 292 }
 293
 294 static void
 295 kpc_set_running_xcall( void *vstate )
 296 {
 297         struct kpc_running_remote *mp_config = (struct kpc_running_remote*) vstate;
 298         assert(mp_config);
 299
 300         if (kpc_controls_fixed_counters()) {
 301                 set_running_fixed(mp_config->classes & KPC_CLASS_FIXED_MASK);
 302         }
 303
 304         set_running_configurable(mp_config->cfg_target_mask,
 305             mp_config->cfg_state_mask);
 306
 307         if (os_atomic_dec(&kpc_xcall_sync, relaxed) == 0) {
 308                 thread_wakeup((event_t) &kpc_xcall_sync);
 309         }
 310 }
 311
 312 static uint64_t
 313 get_counter_config(uint32_t counter)
 314 {
 315         uint32_t config = 0;
 316
 317         switch (counter) {
 318         case 0:
 319                 /* Fixed counter accessed via top bit... */
 320                 counter = 31;
 321                 /* Write PMSELR.SEL */
 322                 __asm__ volatile ("mcr p15, 0, %0, c9, c12, 5;" : : "r" (counter));
 323                 /* Read PMXEVTYPER */
 324                 __asm__ volatile ("mcr p15, 0, %0, c9, c13, 1;" : "=r" (config));
 325                 break;
 326         case 1:
 327         case 2:
 328         case 3:
 329         case 4:
 330                 /* Offset */
 331                 counter -= 1;
 332                 /* Write PMSELR.SEL to select the configurable counter */
 333                 __asm__ volatile ("mcr p15, 0, %0, c9, c12, 5;" : : "r" (counter));
 334                 /* Read PMXEVTYPER to get the config */
 335                 __asm__ volatile ("mrc p15, 0, %0, c9, c13, 1;" : "=r" (config));
 336                 break;
 337         default:
 338                 break;
 339         }
 340
 341         return config;
 342 }
 343
 344 static void
 345 set_counter_config(uint32_t counter, uint64_t config)
 346 {
 347         switch (counter) {
 348         case 0:
 349                 /* Write PMSELR.SEL */
 350                 __asm__ volatile ("mcr p15, 0, %0, c9, c12, 5;" : : "r" (31));
 351                 /* Write PMXEVTYPER */
 352                 __asm__ volatile ("mcr p15, 0, %0, c9, c13, 1;" : : "r" (config & 0xFFFFFFFF));
 353                 break;
 354         case 1:
 355         case 2:
 356         case 3:
 357         case 4:
 358                 /* Write PMSELR.SEL */
 359                 __asm__ volatile ("mcr p15, 0, %0, c9, c12, 5;" : : "r" (counter - 1));
 360                 /* Write PMXEVTYPER */
 361                 __asm__ volatile ("mcr p15, 0, %0, c9, c13, 1;" : : "r" (config & 0xFFFFFFFF));
 362                 break;
 363         default:
 364                 break;
 365         }
 366 }
 367
 368 /* Common */
 369
 370 void
 371 kpc_arch_init(void)
 372 {
 373         uint32_t PMCR;
 374         uint32_t event_counters;
 375
 376         /* read PMOVSR and determine the number of event counters */
 377         __asm__ volatile ("mrc p15, 0, %0, c9, c12, 0;" : "=r" (PMCR));
 378         event_counters = (PMCR >> 11) & 0x1F;
 379
 380         assert(event_counters >= KPC_ARM_CONFIGURABLE_COUNT);
 381 }
 382
 383 uint32_t
 384 kpc_get_classes(void)
 385 {
 386         return KPC_CLASS_FIXED_MASK | KPC_CLASS_CONFIGURABLE_MASK;
 387 }
 388
 389 uint32_t
 390 kpc_fixed_count(void)
 391 {
 392         return KPC_ARM_FIXED_COUNT;
 393 }
 394
 395 uint32_t
 396 kpc_configurable_count(void)
 397 {
 398         return KPC_ARM_CONFIGURABLE_COUNT;
 399 }
 400
 401 uint32_t
 402 kpc_fixed_config_count(void)
 403 {
 404         return KPC_ARM_FIXED_COUNT;
 405 }
 406
 407 uint32_t
 408 kpc_configurable_config_count(uint64_t pmc_mask)
 409 {
 410         assert(kpc_popcount(pmc_mask) <= kpc_configurable_count());
 411         return kpc_popcount(pmc_mask);
 412 }
 413
 414 int
 415 kpc_get_fixed_config(kpc_config_t *configv)
 416 {
 417         configv[0] = get_counter_config(0);
 418         return 0;
 419 }
 420
 421 uint64_t
 422 kpc_fixed_max(void)
 423 {
 424         return (1ULL << KPC_ARM_COUNTER_WIDTH) - 1;
 425 }
 426
 427 uint64_t
 428 kpc_configurable_max(void)
 429 {
 430         return (1ULL << KPC_ARM_COUNTER_WIDTH) - 1;
 431 }
 432
 433 int
 434 kpc_get_configurable_counters(uint64_t *counterv, uint64_t pmc_mask)
 435 {
 436         uint32_t cfg_count = kpc_configurable_count(), offset = kpc_fixed_count();
 437
 438         assert(counterv);
 439
 440         for (uint32_t i = 0; i < cfg_count; ++i) {
 441                 uint32_t PMOVSR;
 442                 uint32_t mask;
 443                 uint64_t ctr;
 444
 445                 if (((1ULL << i) & pmc_mask) == 0) {
 446                         continue;
 447                 }
 448                 ctr = read_counter(i + offset);
 449
 450                 /* check the counter for overflow */
 451                 mask = 1 << i;
 452
 453                 /* read PMOVSR */
 454                 __asm__ volatile ("mrc p15, 0, %0, c9, c12, 3;" : "=r" (PMOVSR));
 455
 456                 if (PMOVSR & mask) {
 457                         ctr = CONFIGURABLE_SHADOW(i) +
 458                             (kpc_configurable_max() - CONFIGURABLE_RELOAD(i) + 1 /* Wrap */) +
 459                             ctr;
 460                 } else {
 461                         ctr = CONFIGURABLE_SHADOW(i) +
 462                             (ctr - CONFIGURABLE_RELOAD(i));
 463                 }
 464
 465                 *counterv++ = ctr;
 466         }
 467
 468         return 0;
 469 }
 470
 471 int
 472 kpc_get_fixed_counters(uint64_t *counterv)
 473 {
 474         uint32_t PMOVSR;
 475         uint32_t mask;
 476         uint64_t ctr;
 477
 478         /* check the counter for overflow */
 479         mask = 1 << 31;
 480
 481         /* read PMOVSR */
 482         __asm__ volatile ("mrc p15, 0, %0, c9, c12, 3;" : "=r" (PMOVSR));
 483
 484         ctr = read_counter(0);
 485
 486         if (PMOVSR & mask) {
 487                 ctr = FIXED_SHADOW(0) +
 488                     (kpc_fixed_max() - FIXED_RELOAD(0) + 1 /* Wrap */) +
 489                     (ctr & 0xFFFFFFFF);
 490         } else {
 491                 ctr = FIXED_SHADOW(0) +
 492                     (ctr - FIXED_RELOAD(0));
 493         }
 494
 495         counterv[0] = ctr;
 496
 497         return 0;
 498 }
 499 boolean_t
 500 kpc_is_running_fixed(void)
 501 {
 502         return (kpc_running_classes & KPC_CLASS_FIXED_MASK) == KPC_CLASS_FIXED_MASK;
 503 }
 504
 505 boolean_t
 506 kpc_is_running_configurable(uint64_t pmc_mask)
 507 {
 508         assert(kpc_popcount(pmc_mask) <= kpc_configurable_count());
 509         return ((kpc_running_classes & KPC_CLASS_CONFIGURABLE_MASK) == KPC_CLASS_CONFIGURABLE_MASK) &&
 510                ((kpc_running_cfg_pmc_mask & pmc_mask) == pmc_mask);
 511 }
 512
 513 int
 514 kpc_set_running_arch(struct kpc_running_remote *mp_config)
 515 {
 516         unsigned int cpu;
 517
 518         assert(mp_config);
 519
 520         if (first_time) {
 521                 kprintf( "kpc: setting PMI handler\n" );
 522                 PE_cpu_perfmon_interrupt_install_handler(kpc_pmi_handler);
 523                 for (cpu = 0; cpu < real_ncpus; cpu++) {
 524                         PE_cpu_perfmon_interrupt_enable(cpu_datap(cpu)->cpu_id,
 525                             TRUE);
 526                 }
 527                 first_time = 0;
 528         }
 529
 530         /* dispatch to all CPUs */
 531         cpu_broadcast_xcall(&kpc_xcall_sync, TRUE, kpc_set_running_xcall,
 532             mp_config);
 533
 534         kpc_running_cfg_pmc_mask = mp_config->cfg_state_mask;
 535         kpc_running_classes = mp_config->classes;
 536         kpc_configured = 1;
 537
 538         return 0;
 539 }
 540
 541 static void
 542 save_regs(void)
 543 {
 544         int i;
 545         int cpuid = cpu_number();
 546         uint32_t PMCR = 0;
 547
 548         __asm__ volatile ("dmb ish");
 549
 550         /* Clear master enable */
 551         __asm__ volatile ("mcr p15, 0, %0, c9, c12, 0;" : : "r" (PMCR));
 552
 553         /* Save individual enable state */
 554         __asm__ volatile ("mrc p15, 0, %0, c9, c12, 1;" : "=r" (saved_PMCNTENSET[cpuid]));
 555
 556         /* Save PMOVSR */
 557         __asm__ volatile ("mrc p15, 0, %0, c9, c12, 3;" : "=r" (saved_PMOVSR[cpuid]));
 558
 559         /* Select fixed counter with PMSELR.SEL */
 560         __asm__ volatile ("mcr p15, 0, %0, c9, c12, 5;" : : "r" (31));
 561         /* Read PMXEVTYPER */
 562         __asm__ volatile ("mrc p15, 0, %0, c9, c13, 1;" : "=r" (saved_PMXEVTYPER[cpuid][0]));
 563
 564         /* Save configurable event selections */
 565         for (i = 0; i < 4; i++) {
 566                 /* Select counter with PMSELR.SEL */
 567                 __asm__ volatile ("mcr p15, 0, %0, c9, c12, 5;" : : "r" (i));
 568                 /* Read PMXEVTYPER */
 569                 __asm__ volatile ("mrc p15, 0, %0, c9, c13, 1;" : "=r" (saved_PMXEVTYPER[cpuid][i + 1]));
 570         }
 571
 572         /* Finally, save count for each counter */
 573         for (i = 0; i < 5; i++) {
 574                 saved_counter[cpuid][i] = read_counter(i);
 575         }
 576 }
 577
 578 static void
 579 restore_regs(void)
 580 {
 581         int i;
 582         int cpuid = cpu_number();
 583         uint64_t extra;
 584         uint32_t PMCR = 1;
 585
 586         /* Restore counter values */
 587         for (i = 0; i < 5; i++) {
 588                 /* did we overflow? if so handle it now since we won't get a pmi */
 589                 uint32_t mask;
 590
 591                 /* check the counter for overflow */
 592                 if (i == 0) {
 593                         mask = 1 << 31;
 594                 } else {
 595                         mask = 1 << (i - 1);
 596                 }
 597
 598                 if (saved_PMOVSR[cpuid] & mask) {
 599                         extra = kpc_reload_counter(i);
 600
 601                         /*
 602                          * CONFIGURABLE_* directly follows FIXED, so we can simply
 603                          * increment the index here. Although it's ugly.
 604                          */
 605                         FIXED_SHADOW(i)
 606                                 += (kpc_fixed_max() - FIXED_RELOAD(i) + 1 /* Wrap */) + extra;
 607
 608                         if (FIXED_ACTIONID(i)) {
 609                                 bool kernel = false;
 610                                 uintptr_t pc = get_interrupted_pc(&kernel);
 611                                 kpc_sample_kperf(FIXED_ACTIONID(i), i, get_counter_config(i),
 612                                     FIXED_SHADOW(i), pc, kernel ? KPC_KERNEL_PC : 0);
 613                         }
 614                 } else {
 615                         write_counter(i, saved_counter[cpuid][i]);
 616                 }
 617         }
 618
 619         /* Restore configuration - first, the fixed... */
 620         __asm__ volatile ("mcr p15, 0, %0, c9, c12, 5;" : : "r" (31));
 621         /* Write PMXEVTYPER */
 622         __asm__ volatile ("mcr p15, 0, %0, c9, c13, 1;" : : "r" (saved_PMXEVTYPER[cpuid][0]));
 623
 624         /* ...then the configurable */
 625         for (i = 0; i < 4; i++) {
 626                 /* Select counter with PMSELR.SEL */
 627                 __asm__ volatile ("mcr p15, 0, %0, c9, c12, 5;" : : "r" (i));
 628                 /* Write PMXEVTYPER */
 629                 __asm__ volatile ("mcr p15, 0, %0, c9, c13, 1;" : : "r" (saved_PMXEVTYPER[cpuid][i + 1]));
 630         }
 631
 632         /* Restore enable state */
 633         __asm__ volatile ("mcr p15, 0, %0, c9, c12, 1;" : : "r" (saved_PMCNTENSET[cpuid]));
 634
 635         /* Counter master re-enable */
 636         __asm__ volatile ("mcr p15, 0, %0, c9, c12, 0;" : : "r" (PMCR));
 637 }
 638
 639 static void
 640 kpc_set_reload_xcall(void *vmp_config)
 641 {
 642         struct kpc_config_remote *mp_config = vmp_config;
 643         uint32_t classes = 0, count = 0, offset = kpc_fixed_count();
 644         uint64_t *new_period = NULL, max = kpc_configurable_max();
 645         boolean_t enabled;
 646
 647         assert(mp_config);
 648         assert(mp_config->configv);
 649         classes = mp_config->classes;
 650         new_period = mp_config->configv;
 651
 652         enabled = ml_set_interrupts_enabled(FALSE);
 653
 654         if ((classes & KPC_CLASS_FIXED_MASK) && kpc_controls_fixed_counters()) {
 655                 /* update shadow counters */
 656                 kpc_get_fixed_counters(&FIXED_SHADOW(0));
 657
 658                 /* set the new period */
 659                 count = kpc_fixed_count();
 660                 for (uint32_t i = 0; i < count; ++i) {
 661                         if (*new_period == 0) {
 662                                 *new_period = kpc_fixed_max();
 663                         }
 664                         FIXED_RELOAD(i) = max - *new_period;
 665                         /* reload the counter if possible */
 666                         kpc_reload_counter(i);
 667                         /* next period value */
 668                         new_period++;
 669                 }
 670         }
 671
 672         if (classes & KPC_CLASS_CONFIGURABLE_MASK) {
 673                 /*
 674                  * Update _all_ shadow counters, this cannot be done for only
 675                  * selected PMCs. Otherwise, we would corrupt the configurable
 676                  * shadow buffer since the PMCs are muxed according to the pmc
 677                  * mask.
 678                  */
 679                 uint64_t all_cfg_mask = (1ULL << kpc_configurable_count()) - 1;
 680                 kpc_get_configurable_counters(&CONFIGURABLE_SHADOW(0), all_cfg_mask);
 681
 682                 /* set the new period */
 683                 count = kpc_configurable_count();
 684                 for (uint32_t i = 0; i < count; ++i) {
 685                         /* ignore the counter */
 686                         if (((1ULL << i) & mp_config->pmc_mask) == 0) {
 687                                 continue;
 688                         }
 689                         if (*new_period == 0) {
 690                                 *new_period = kpc_configurable_max();
 691                         }
 692                         CONFIGURABLE_RELOAD(i) = max - *new_period;
 693                         /* reload the counter */
 694                         kpc_reload_counter(offset + i);
 695                         /* next period value */
 696                         new_period++;
 697                 }
 698         }
 699
 700         ml_set_interrupts_enabled(enabled);
 701
 702         if (os_atomic_dec(&kpc_reload_sync, relaxed) == 0) {
 703                 thread_wakeup((event_t) &kpc_reload_sync);
 704         }
 705 }
 706
 707
 708 int
 709 kpc_set_period_arch(struct kpc_config_remote *mp_config)
 710 {
 711         /* dispatch to all CPUs */
 712         cpu_broadcast_xcall(&kpc_reload_sync, TRUE, kpc_set_reload_xcall, mp_config);
 713
 714         kpc_configured = 1;
 715
 716         return 0;
 717 }
 718
 719 int
 720 kpc_get_configurable_config(kpc_config_t *configv, uint64_t pmc_mask)
 721 {
 722         uint32_t cfg_count = kpc_configurable_count(), offset = kpc_fixed_count();
 723
 724         assert(configv);
 725
 726         for (uint32_t i = 0; i < cfg_count; ++i) {
 727                 if ((1ULL << i) & pmc_mask) {
 728                         *configv++ = get_counter_config(i + offset);
 729                 }
 730         }
 731
 732         return 0;
 733 }
 734
 735 static int
 736 kpc_set_configurable_config(kpc_config_t *configv, uint64_t pmc_mask)
 737 {
 738         uint32_t cfg_count = kpc_configurable_count(), offset = kpc_fixed_count();
 739         boolean_t enabled;
 740
 741         assert(configv);
 742
 743         enabled = ml_set_interrupts_enabled(FALSE);
 744
 745         for (uint32_t i = 0; i < cfg_count; ++i) {
 746                 if (((1ULL << i) & pmc_mask) == 0) {
 747                         continue;
 748                 }
 749                 assert(kpc_controls_counter(i + offset));
 750
 751                 set_counter_config(i + offset, *configv++);
 752         }
 753
 754         ml_set_interrupts_enabled(enabled);
 755
 756         return 0;
 757 }
 758
 759 static uint32_t kpc_config_sync;
 760 static void
 761 kpc_set_config_xcall(void *vmp_config)
 762 {
 763         struct kpc_config_remote *mp_config = vmp_config;
 764         kpc_config_t *new_config = NULL;
 765         uint32_t classes = 0ULL;
 766
 767         assert(mp_config);
 768         assert(mp_config->configv);
 769         classes = mp_config->classes;
 770         new_config = mp_config->configv;
 771
 772         if (classes & KPC_CLASS_CONFIGURABLE_MASK) {
 773                 kpc_set_configurable_config(new_config, mp_config->pmc_mask);
 774                 new_config += kpc_popcount(mp_config->pmc_mask);
 775         }
 776
 777         if (os_atomic_dec(&kpc_config_sync, relaxed) == 0) {
 778                 thread_wakeup((event_t) &kpc_config_sync);
 779         }
 780 }
 781
 782 int
 783 kpc_set_config_arch(struct kpc_config_remote *mp_config)
 784 {
 785         /* dispatch to all CPUs */
 786         cpu_broadcast_xcall(&kpc_config_sync, TRUE, kpc_set_config_xcall, mp_config);
 787
 788         kpc_configured = 1;
 789
 790         return 0;
 791 }
 792
 793 void
 794 kpc_idle(void)
 795 {
 796         if (kpc_configured) {
 797                 save_regs();
 798         }
 799 }
 800
 801 void
 802 kpc_idle_exit(void)
 803 {
 804         if (kpc_configured) {
 805                 restore_regs();
 806         }
 807 }
 808
 809 static uint32_t kpc_xread_sync;
 810 static void
 811 kpc_get_curcpu_counters_xcall(void *args)
 812 {
 813         struct kpc_get_counters_remote *handler = args;
 814         int offset = 0, r = 0;
 815
 816         assert(handler);
 817         assert(handler->buf);
 818
 819         offset = cpu_number() * handler->buf_stride;
 820         r = kpc_get_curcpu_counters(handler->classes, NULL, &handler->buf[offset]);
 821
 822         /* number of counters added by this CPU, needs to be atomic  */
 823         os_atomic_add(&(handler->nb_counters), r, relaxed);
 824
 825         if (os_atomic_dec(&kpc_xread_sync, relaxed) == 0) {
 826                 thread_wakeup((event_t) &kpc_xread_sync);
 827         }
 828 }
 829
 830 int
 831 kpc_get_all_cpus_counters(uint32_t classes, int *curcpu, uint64_t *buf)
 832 {
 833         int enabled = 0;
 834
 835         struct kpc_get_counters_remote hdl = {
 836                 .classes = classes, .nb_counters = 0,
 837                 .buf_stride = kpc_get_counter_count(classes),
 838                 .buf = buf
 839         };
 840
 841         assert(buf);
 842
 843         enabled = ml_set_interrupts_enabled(FALSE);
 844
 845         if (curcpu) {
 846                 *curcpu = cpu_number();
 847         }
 848         cpu_broadcast_xcall(&kpc_xread_sync, TRUE, kpc_get_curcpu_counters_xcall, &hdl);
 849
 850         ml_set_interrupts_enabled(enabled);
 851
 852         return hdl.nb_counters;
 853 }
 854
 855 int
 856 kpc_get_pmu_version(void)
 857 {
 858         return KPC_PMU_ARM_V2;
 859 }
 860
 861 int
 862 kpc_set_sw_inc( uint32_t mask )
 863 {
 864         /* Only works with the configurable counters set to count the increment event (0x0) */
 865
 866         /* Write to PMSWINC */
 867         __asm__ volatile ("mcr p15, 0, %0, c9, c12, 4;" : : "r" (mask));
 868
 869         return 0;
 870 }
 871
 872 #else /* !ARMA7 */
 873
 874 /* no kpc */
 875
 876 void
 877 kpc_arch_init(void)
 878 {
 879         /* No-op */
 880 }
 881
 882 uint32_t
 883 kpc_get_classes(void)
 884 {
 885         return 0;
 886 }
 887
 888 uint32_t
 889 kpc_fixed_count(void)
 890 {
 891         return 0;
 892 }
 893
 894 uint32_t
 895 kpc_configurable_count(void)
 896 {
 897         return 0;
 898 }
 899
 900 uint32_t
 901 kpc_fixed_config_count(void)
 902 {
 903         return 0;
 904 }
 905
 906 uint32_t
 907 kpc_configurable_config_count(uint64_t pmc_mask __unused)
 908 {
 909         return 0;
 910 }
 911
 912 int
 913 kpc_get_fixed_config(kpc_config_t *configv __unused)
 914 {
 915         return 0;
 916 }
 917
 918 uint64_t
 919 kpc_fixed_max(void)
 920 {
 921         return 0;
 922 }
 923
 924 uint64_t
 925 kpc_configurable_max(void)
 926 {
 927         return 0;
 928 }
 929
 930 int
 931 kpc_get_configurable_config(kpc_config_t *configv __unused, uint64_t pmc_mask __unused)
 932 {
 933         return ENOTSUP;
 934 }
 935
 936 int
 937 kpc_get_configurable_counters(uint64_t *counterv __unused, uint64_t pmc_mask __unused)
 938 {
 939         return ENOTSUP;
 940 }
 941
 942 int
 943 kpc_get_fixed_counters(uint64_t *counterv __unused)
 944 {
 945         return 0;
 946 }
 947
 948 boolean_t
 949 kpc_is_running_fixed(void)
 950 {
 951         return FALSE;
 952 }
 953
 954 boolean_t
 955 kpc_is_running_configurable(uint64_t pmc_mask __unused)
 956 {
 957         return FALSE;
 958 }
 959
 960 int
 961 kpc_set_running_arch(struct kpc_running_remote *mp_config __unused)
 962 {
 963         return ENOTSUP;
 964 }
 965
 966 int
 967 kpc_set_period_arch(struct kpc_config_remote *mp_config __unused)
 968 {
 969         return ENOTSUP;
 970 }
 971
 972 int
 973 kpc_set_config_arch(struct kpc_config_remote *mp_config __unused)
 974 {
 975         return ENOTSUP;
 976 }
 977
 978 void
 979 kpc_idle(void)
 980 {
 981         // do nothing
 982 }
 983
 984 void
 985 kpc_idle_exit(void)
 986 {
 987         // do nothing
 988 }
 989
 990 int
 991 kpc_get_all_cpus_counters(uint32_t classes, int *curcpu, uint64_t *buf)
 992 {
 993 #pragma unused(classes)
 994 #pragma unused(curcpu)
 995 #pragma unused(buf)
 996
 997         return 0;
 998 }
 999
1000 int
1001 kpc_set_sw_inc( uint32_t mask __unused )
1002 {
1003         return ENOTSUP;
1004 }
1005
1006 int
1007 kpc_get_pmu_version(void)
1008 {
1009         return KPC_PMU_ERROR;
1010 }
1011
1012 #endif
1013
1014 /*
1015  * RAWPMU isn't implemented for any of the 32-bit ARMs.
1016  */
1017
1018 uint32_t
1019 kpc_rawpmu_config_count(void)
1020 {
1021         return 0;
1022 }
1023
1024 int
1025 kpc_get_rawpmu_config(__unused kpc_config_t *configv)
1026 {
1027         return 0;
1028 }