osfmk/x86_64/kpc_x86.c

   1 /*
   2  * Copyright (c) 2012 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <mach/mach_types.h>
  30 #include <machine/machine_routines.h>
  31 #include <kern/processor.h>
  32 #include <kern/kalloc.h>
  33 #include <i386/cpuid.h>
  34 #include <i386/proc_reg.h>
  35 #include <i386/mp.h>
  36 #include <sys/errno.h>
  37 #include <kperf/buffer.h>
  38
  39 #include <kern/kpc.h>
  40
  41 #include <kperf/kperf.h>
  42 #include <kperf/sample.h>
  43 #include <kperf/context.h>
  44 #include <kperf/action.h>
  45
  46 #include <kern/monotonic.h>
  47
  48 /* Fixed counter mask -- three counters, each with OS and USER */
  49 #define IA32_FIXED_CTR_ENABLE_ALL_CTRS_ALL_RINGS (0x333)
  50 #define IA32_FIXED_CTR_ENABLE_ALL_PMI (0x888)
  51
  52 #define IA32_PERFEVTSEL_PMI (1ull << 20)
  53 #define IA32_PERFEVTSEL_EN (1ull << 22)
  54
  55 /* Non-serialising */
  56 #define USE_RDPMC
  57
  58 #define RDPMC_FIXED_COUNTER_SELECTOR (1ULL<<30)
  59
  60 /* track the last config we enabled */
  61 static uint64_t kpc_running_cfg_pmc_mask = 0;
  62 static uint32_t kpc_running_classes = 0;
  63
  64 /* PMC / MSR accesses */
  65
  66 static uint64_t
  67 IA32_FIXED_CTR_CTRL(void)
  68 {
  69         return rdmsr64( MSR_IA32_PERF_FIXED_CTR_CTRL );
  70 }
  71
  72 #ifdef FIXED_COUNTER_RELOAD
  73 static void
  74 wrIA32_FIXED_CTRx(uint32_t ctr, uint64_t value)
  75 {
  76         return wrmsr64(MSR_IA32_PERF_FIXED_CTR0 + ctr, value);
  77 }
  78 #endif
  79
  80 static uint64_t
  81 IA32_PMCx(uint32_t ctr)
  82 {
  83 #ifdef USE_RDPMC
  84         return rdpmc64(ctr);
  85 #else /* !USE_RDPMC */
  86         return rdmsr64(MSR_IA32_PERFCTR0 + ctr);
  87 #endif /* !USE_RDPMC */
  88 }
  89
  90 static void
  91 wrIA32_PMCx(uint32_t ctr, uint64_t value)
  92 {
  93         return wrmsr64(MSR_IA32_PERFCTR0 + ctr, value);
  94 }
  95
  96 static uint64_t
  97 IA32_PERFEVTSELx(uint32_t ctr)
  98 {
  99         return rdmsr64(MSR_IA32_EVNTSEL0 + ctr);
 100 }
 101
 102 static void
 103 wrIA32_PERFEVTSELx(uint32_t ctr, uint64_t value)
 104 {
 105         wrmsr64(MSR_IA32_EVNTSEL0 + ctr, value);
 106 }
 107
 108
 109 /* internal functions */
 110
 111 boolean_t
 112 kpc_is_running_fixed(void)
 113 {
 114         return (kpc_running_classes & KPC_CLASS_FIXED_MASK) == KPC_CLASS_FIXED_MASK;
 115 }
 116
 117 boolean_t
 118 kpc_is_running_configurable(uint64_t pmc_mask)
 119 {
 120         assert(kpc_popcount(pmc_mask) <= kpc_configurable_count());
 121         return ((kpc_running_classes & KPC_CLASS_CONFIGURABLE_MASK) == KPC_CLASS_CONFIGURABLE_MASK) &&
 122                ((kpc_running_cfg_pmc_mask & pmc_mask) == pmc_mask);
 123 }
 124
 125 uint32_t
 126 kpc_fixed_count(void)
 127 {
 128         i386_cpu_info_t *info = NULL;
 129         info = cpuid_info();
 130         return info->cpuid_arch_perf_leaf.fixed_number;
 131 }
 132
 133 uint32_t
 134 kpc_configurable_count(void)
 135 {
 136         i386_cpu_info_t *info = NULL;
 137         info = cpuid_info();
 138         return info->cpuid_arch_perf_leaf.number;
 139 }
 140
 141 uint32_t
 142 kpc_fixed_config_count(void)
 143 {
 144         return KPC_X86_64_FIXED_CONFIGS;
 145 }
 146
 147 uint32_t
 148 kpc_configurable_config_count(uint64_t pmc_mask)
 149 {
 150         assert(kpc_popcount(pmc_mask) <= kpc_configurable_count());
 151         return kpc_popcount(pmc_mask);
 152 }
 153
 154 uint32_t
 155 kpc_rawpmu_config_count(void)
 156 {
 157         // RAW PMU access not implemented.
 158         return 0;
 159 }
 160
 161 int
 162 kpc_get_rawpmu_config(__unused kpc_config_t *configv)
 163 {
 164         return 0;
 165 }
 166
 167 static uint8_t
 168 kpc_fixed_width(void)
 169 {
 170         i386_cpu_info_t *info = NULL;
 171
 172         info = cpuid_info();
 173
 174         return info->cpuid_arch_perf_leaf.fixed_width;
 175 }
 176
 177 static uint8_t
 178 kpc_configurable_width(void)
 179 {
 180         i386_cpu_info_t *info = NULL;
 181
 182         info = cpuid_info();
 183
 184         return info->cpuid_arch_perf_leaf.width;
 185 }
 186
 187 uint64_t
 188 kpc_fixed_max(void)
 189 {
 190         return (1ULL << kpc_fixed_width()) - 1;
 191 }
 192
 193 uint64_t
 194 kpc_configurable_max(void)
 195 {
 196         return (1ULL << kpc_configurable_width()) - 1;
 197 }
 198
 199 #ifdef FIXED_COUNTER_SHADOW
 200 static uint64_t
 201 kpc_reload_fixed(int ctr)
 202 {
 203         uint64_t old = IA32_FIXED_CTRx(ctr);
 204         wrIA32_FIXED_CTRx(ctr, FIXED_RELOAD(ctr));
 205         return old;
 206 }
 207 #endif
 208
 209 static uint64_t
 210 kpc_reload_configurable(int ctr)
 211 {
 212         uint64_t cfg = IA32_PERFEVTSELx(ctr);
 213
 214         /* counters must be disabled before they can be written to */
 215         uint64_t old = IA32_PMCx(ctr);
 216         wrIA32_PERFEVTSELx(ctr, cfg & ~IA32_PERFEVTSEL_EN);
 217         wrIA32_PMCx(ctr, CONFIGURABLE_RELOAD(ctr));
 218         wrIA32_PERFEVTSELx(ctr, cfg);
 219         return old;
 220 }
 221
 222 void kpc_pmi_handler(void);
 223
 224 static void
 225 set_running_fixed(boolean_t on)
 226 {
 227         uint64_t global = 0, mask = 0, fixed_ctrl = 0;
 228         int i;
 229         boolean_t enabled;
 230
 231         if (on) {
 232                 /* these are per-thread in SMT */
 233                 fixed_ctrl = IA32_FIXED_CTR_ENABLE_ALL_CTRS_ALL_RINGS | IA32_FIXED_CTR_ENABLE_ALL_PMI;
 234         } else {
 235                 /* don't allow disabling fixed counters */
 236                 return;
 237         }
 238
 239         wrmsr64( MSR_IA32_PERF_FIXED_CTR_CTRL, fixed_ctrl );
 240
 241         enabled = ml_set_interrupts_enabled(FALSE);
 242
 243         /* rmw the global control */
 244         global = rdmsr64(MSR_IA32_PERF_GLOBAL_CTRL);
 245         for (i = 0; i < (int) kpc_fixed_count(); i++) {
 246                 mask |= (1ULL << (32 + i));
 247         }
 248
 249         if (on) {
 250                 global |= mask;
 251         } else {
 252                 global &= ~mask;
 253         }
 254
 255         wrmsr64(MSR_IA32_PERF_GLOBAL_CTRL, global);
 256
 257         ml_set_interrupts_enabled(enabled);
 258 }
 259
 260 static void
 261 set_running_configurable(uint64_t target_mask, uint64_t state_mask)
 262 {
 263         uint32_t cfg_count = kpc_configurable_count();
 264         uint64_t global = 0ULL, cfg = 0ULL, save = 0ULL;
 265         boolean_t enabled;
 266
 267         enabled = ml_set_interrupts_enabled(FALSE);
 268
 269         /* rmw the global control */
 270         global = rdmsr64(MSR_IA32_PERF_GLOBAL_CTRL);
 271
 272         /* need to save and restore counter since it resets when reconfigured */
 273         for (uint32_t i = 0; i < cfg_count; ++i) {
 274                 cfg = IA32_PERFEVTSELx(i);
 275                 save = IA32_PMCx(i);
 276                 wrIA32_PERFEVTSELx(i, cfg | IA32_PERFEVTSEL_PMI | IA32_PERFEVTSEL_EN);
 277                 wrIA32_PMCx(i, save);
 278         }
 279
 280         /* update the global control value */
 281         global &= ~target_mask; /* clear the targeted PMCs bits */
 282         global |= state_mask;   /* update the targeted PMCs bits with their new states */
 283         wrmsr64(MSR_IA32_PERF_GLOBAL_CTRL, global);
 284
 285         ml_set_interrupts_enabled(enabled);
 286 }
 287
 288 static void
 289 kpc_set_running_mp_call( void *vstate )
 290 {
 291         struct kpc_running_remote *mp_config = (struct kpc_running_remote*) vstate;
 292         assert(mp_config);
 293
 294         if (kpc_controls_fixed_counters()) {
 295                 set_running_fixed(mp_config->classes & KPC_CLASS_FIXED_MASK);
 296         }
 297
 298         set_running_configurable(mp_config->cfg_target_mask,
 299             mp_config->cfg_state_mask);
 300 }
 301
 302 int
 303 kpc_get_fixed_config(kpc_config_t *configv)
 304 {
 305         configv[0] = IA32_FIXED_CTR_CTRL();
 306         return 0;
 307 }
 308
 309 static int
 310 kpc_set_fixed_config(kpc_config_t *configv)
 311 {
 312         (void) configv;
 313
 314         /* NYI */
 315         return -1;
 316 }
 317
 318 int
 319 kpc_get_fixed_counters(uint64_t *counterv)
 320 {
 321 #if MONOTONIC
 322         mt_fixed_counts(counterv);
 323         return 0;
 324 #else /* MONOTONIC */
 325 #pragma unused(counterv)
 326         return ENOTSUP;
 327 #endif /* !MONOTONIC */
 328 }
 329
 330 int
 331 kpc_get_configurable_config(kpc_config_t *configv, uint64_t pmc_mask)
 332 {
 333         uint32_t cfg_count = kpc_configurable_count();
 334
 335         assert(configv);
 336
 337         for (uint32_t i = 0; i < cfg_count; ++i) {
 338                 if ((1ULL << i) & pmc_mask) {
 339                         *configv++  = IA32_PERFEVTSELx(i);
 340                 }
 341         }
 342         return 0;
 343 }
 344
 345 static int
 346 kpc_set_configurable_config(kpc_config_t *configv, uint64_t pmc_mask)
 347 {
 348         uint32_t cfg_count = kpc_configurable_count();
 349         uint64_t save;
 350
 351         for (uint32_t i = 0; i < cfg_count; i++) {
 352                 if (((1ULL << i) & pmc_mask) == 0) {
 353                         continue;
 354                 }
 355
 356                 /* need to save and restore counter since it resets when reconfigured */
 357                 save = IA32_PMCx(i);
 358
 359                 /*
 360                  * Some bits are not safe to set from user space.
 361                  * Allow these bits to be set:
 362                  *
 363                  *   0-7    Event select
 364                  *   8-15   UMASK
 365                  *   16     USR
 366                  *   17     OS
 367                  *   18     E
 368                  *   22     EN
 369                  *   23     INV
 370                  *   24-31  CMASK
 371                  *
 372                  * Excluding:
 373                  *
 374                  *   19     PC
 375                  *   20     INT
 376                  *   21     AnyThread
 377                  *   32     IN_TX
 378                  *   33     IN_TXCP
 379                  *   34-63  Reserved
 380                  */
 381                 wrIA32_PERFEVTSELx(i, *configv & 0xffc7ffffull);
 382                 wrIA32_PMCx(i, save);
 383
 384                 /* next configuration word */
 385                 configv++;
 386         }
 387
 388         return 0;
 389 }
 390
 391 int
 392 kpc_get_configurable_counters(uint64_t *counterv, uint64_t pmc_mask)
 393 {
 394         uint32_t cfg_count = kpc_configurable_count();
 395         uint64_t status, *it_counterv = counterv;
 396
 397         /* snap the counters */
 398         for (uint32_t i = 0; i < cfg_count; ++i) {
 399                 if ((1ULL << i) & pmc_mask) {
 400                         *it_counterv++ = CONFIGURABLE_SHADOW(i) +
 401                             (IA32_PMCx(i) - CONFIGURABLE_RELOAD(i));
 402                 }
 403         }
 404
 405         /* Grab the overflow bits */
 406         status = rdmsr64(MSR_IA32_PERF_GLOBAL_STATUS);
 407
 408         /* reset the iterator */
 409         it_counterv = counterv;
 410
 411         /*
 412          * If the overflow bit is set for a counter, our previous read may or may not have been
 413          * before the counter overflowed. Re-read any counter with it's overflow bit set so
 414          * we know for sure that it has overflowed. The reason this matters is that the math
 415          * is different for a counter that has overflowed.
 416          */
 417         for (uint32_t i = 0; i < cfg_count; ++i) {
 418                 if (((1ULL << i) & pmc_mask) &&
 419                     ((1ULL << i) & status)) {
 420                         *it_counterv++ = CONFIGURABLE_SHADOW(i) +
 421                             (kpc_configurable_max() - CONFIGURABLE_RELOAD(i)) + IA32_PMCx(i);
 422                 }
 423         }
 424
 425         return 0;
 426 }
 427
 428 static void
 429 kpc_get_curcpu_counters_mp_call(void *args)
 430 {
 431         struct kpc_get_counters_remote *handler = args;
 432         int offset = 0, r = 0;
 433
 434         assert(handler);
 435         assert(handler->buf);
 436
 437         offset = cpu_number() * handler->buf_stride;
 438         r = kpc_get_curcpu_counters(handler->classes, NULL, &handler->buf[offset]);
 439
 440         /* number of counters added by this CPU, needs to be atomic  */
 441         os_atomic_add(&(handler->nb_counters), r, relaxed);
 442 }
 443
 444 int
 445 kpc_get_all_cpus_counters(uint32_t classes, int *curcpu, uint64_t *buf)
 446 {
 447         int enabled = 0;
 448
 449         struct kpc_get_counters_remote hdl = {
 450                 .classes = classes, .nb_counters = 0,
 451                 .buf_stride = kpc_get_counter_count(classes), .buf = buf
 452         };
 453
 454         assert(buf);
 455
 456         enabled = ml_set_interrupts_enabled(FALSE);
 457
 458         if (curcpu) {
 459                 *curcpu = current_processor()->cpu_id;
 460         }
 461         mp_cpus_call(CPUMASK_ALL, ASYNC, kpc_get_curcpu_counters_mp_call, &hdl);
 462
 463         ml_set_interrupts_enabled(enabled);
 464
 465         return hdl.nb_counters;
 466 }
 467
 468 static void
 469 kpc_set_config_mp_call(void *vmp_config)
 470 {
 471         struct kpc_config_remote *mp_config = vmp_config;
 472         kpc_config_t *new_config = NULL;
 473         uint32_t classes = 0, count = 0;
 474         boolean_t enabled;
 475
 476         assert(mp_config);
 477         assert(mp_config->configv);
 478         classes = mp_config->classes;
 479         new_config = mp_config->configv;
 480
 481         enabled = ml_set_interrupts_enabled(FALSE);
 482
 483         if (classes & KPC_CLASS_FIXED_MASK) {
 484                 kpc_set_fixed_config(&new_config[count]);
 485                 count += kpc_get_config_count(KPC_CLASS_FIXED_MASK);
 486         }
 487
 488         if (classes & KPC_CLASS_CONFIGURABLE_MASK) {
 489                 kpc_set_configurable_config(&new_config[count], mp_config->pmc_mask);
 490                 count += kpc_popcount(mp_config->pmc_mask);
 491         }
 492
 493         ml_set_interrupts_enabled(enabled);
 494 }
 495
 496 static void
 497 kpc_set_reload_mp_call(void *vmp_config)
 498 {
 499         struct kpc_config_remote *mp_config = vmp_config;
 500         uint64_t *new_period = NULL, max = kpc_configurable_max();
 501         uint32_t classes = 0, count = 0;
 502         boolean_t enabled;
 503
 504         assert(mp_config);
 505         assert(mp_config->configv);
 506         classes = mp_config->classes;
 507         new_period = mp_config->configv;
 508
 509         enabled = ml_set_interrupts_enabled(FALSE);
 510
 511         if (classes & KPC_CLASS_CONFIGURABLE_MASK) {
 512                 /*
 513                  * Update _all_ shadow counters, this cannot be done for only
 514                  * selected PMCs. Otherwise, we would corrupt the configurable
 515                  * shadow buffer since the PMCs are muxed according to the pmc
 516                  * mask.
 517                  */
 518                 uint64_t all_cfg_mask = (1ULL << kpc_configurable_count()) - 1;
 519                 kpc_get_configurable_counters(&CONFIGURABLE_SHADOW(0), all_cfg_mask);
 520
 521                 /* set the new period */
 522                 count = kpc_configurable_count();
 523                 for (uint32_t i = 0; i < count; ++i) {
 524                         /* ignore the counter */
 525                         if (((1ULL << i) & mp_config->pmc_mask) == 0) {
 526                                 continue;
 527                         }
 528
 529                         if (*new_period == 0) {
 530                                 *new_period = kpc_configurable_max();
 531                         }
 532
 533                         CONFIGURABLE_RELOAD(i) = max - *new_period;
 534
 535                         /* reload the counter */
 536                         kpc_reload_configurable(i);
 537
 538                         /* clear overflow bit just in case */
 539                         wrmsr64(MSR_IA32_PERF_GLOBAL_OVF_CTRL, 1ull << i);
 540
 541                         /* next period value */
 542                         new_period++;
 543                 }
 544         }
 545
 546         ml_set_interrupts_enabled(enabled);
 547 }
 548
 549 int
 550 kpc_set_period_arch( struct kpc_config_remote *mp_config )
 551 {
 552         mp_cpus_call( CPUMASK_ALL, ASYNC, kpc_set_reload_mp_call, mp_config );
 553
 554         return 0;
 555 }
 556
 557
 558 /* interface functions */
 559
 560 void
 561 kpc_arch_init(void)
 562 {
 563         i386_cpu_info_t *info = cpuid_info();
 564         uint8_t version_id = info->cpuid_arch_perf_leaf.version;
 565         /*
 566          * kpc only supports Intel PMU versions 2 and above.
 567          */
 568         if (version_id < 2) {
 569                 kpc_supported = false;
 570         }
 571 }
 572
 573 uint32_t
 574 kpc_get_classes(void)
 575 {
 576         return KPC_CLASS_FIXED_MASK | KPC_CLASS_CONFIGURABLE_MASK;
 577 }
 578
 579 int
 580 kpc_set_running_arch(struct kpc_running_remote *mp_config)
 581 {
 582         assert(mp_config);
 583
 584         /* dispatch to all CPUs */
 585         mp_cpus_call(CPUMASK_ALL, ASYNC, kpc_set_running_mp_call, mp_config);
 586
 587         kpc_running_cfg_pmc_mask = mp_config->cfg_state_mask;
 588         kpc_running_classes = mp_config->classes;
 589
 590         return 0;
 591 }
 592
 593 int
 594 kpc_set_config_arch(struct kpc_config_remote *mp_config)
 595 {
 596         mp_cpus_call( CPUMASK_ALL, ASYNC, kpc_set_config_mp_call, mp_config );
 597
 598         return 0;
 599 }
 600
 601 /* PMI stuff */
 602 void
 603 kpc_pmi_handler(void)
 604 {
 605         uint64_t status, extra;
 606         uint32_t ctr;
 607         int enabled;
 608
 609         enabled = ml_set_interrupts_enabled(FALSE);
 610
 611         status = rdmsr64(MSR_IA32_PERF_GLOBAL_STATUS);
 612
 613 #ifdef FIXED_COUNTER_SHADOW
 614         for (ctr = 0; ctr < kpc_fixed_count(); ctr++) {
 615                 if ((1ULL << (ctr + 32)) & status) {
 616                         extra = kpc_reload_fixed(ctr);
 617
 618                         FIXED_SHADOW(ctr)
 619                                 += (kpc_fixed_max() - FIXED_RELOAD(ctr) + 1 /* Wrap */) + extra;
 620
 621                         BUF_INFO(PERF_KPC_FCOUNTER, ctr, FIXED_SHADOW(ctr), extra, FIXED_ACTIONID(ctr));
 622
 623                         if (FIXED_ACTIONID(ctr)) {
 624                                 kpc_sample_kperf(FIXED_ACTIONID(ctr));
 625                         }
 626                 }
 627         }
 628 #endif
 629
 630         for (ctr = 0; ctr < kpc_configurable_count(); ctr++) {
 631                 if ((1ULL << ctr) & status) {
 632                         extra = kpc_reload_configurable(ctr);
 633
 634                         CONFIGURABLE_SHADOW(ctr)
 635                                 += kpc_configurable_max() - CONFIGURABLE_RELOAD(ctr) + extra;
 636
 637                         /* kperf can grab the PMCs when it samples so we need to make sure the overflow
 638                          * bits are in the correct state before the call to kperf_sample */
 639                         wrmsr64(MSR_IA32_PERF_GLOBAL_OVF_CTRL, 1ull << ctr);
 640
 641                         BUF_INFO(PERF_KPC_COUNTER, ctr, CONFIGURABLE_SHADOW(ctr), extra, CONFIGURABLE_ACTIONID(ctr));
 642
 643                         if (CONFIGURABLE_ACTIONID(ctr)) {
 644                                 kpc_sample_kperf(CONFIGURABLE_ACTIONID(ctr));
 645                         }
 646                 }
 647         }
 648
 649         ml_set_interrupts_enabled(enabled);
 650 }
 651
 652 int
 653 kpc_set_sw_inc( uint32_t mask __unused )
 654 {
 655         return ENOTSUP;
 656 }
 657
 658 int
 659 kpc_get_pmu_version(void)
 660 {
 661         i386_cpu_info_t *info = cpuid_info();
 662
 663         uint8_t version_id = info->cpuid_arch_perf_leaf.version;
 664
 665         if (version_id == 3) {
 666                 return KPC_PMU_INTEL_V3;
 667         } else if (version_id == 2) {
 668                 return KPC_PMU_INTEL_V2;
 669         }
 670
 671         return KPC_PMU_ERROR;
 672 }