osfmk/i386/perfmon.c

   1 /*
   2  * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License.  The rights granted to you under the
  10  * License may not be used to create, or enable the creation or
  11  * redistribution of, unlawful or unlicensed copies of an Apple operating
  12  * system, or to circumvent, violate, or enable the circumvention or
  13  * violation of, any terms of an Apple operating system software license
  14  * agreement.
  15  *
  16  * Please obtain a copy of the License at
  17  * http://www.opensource.apple.com/apsl/ and read it before using this
  18  * file.
  19  *
  20  * The Original Code and all software distributed under the License are
  21  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  22  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  23  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  24  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  25  * Please see the License for the specific language governing rights and
  26  * limitations under the License.
  27  *
  28  * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
  29  */
  30
  31 #include <mach/std_types.h>
  32 #include <i386/cpu_data.h>
  33 #include <i386/cpu_number.h>
  34 #include <i386/perfmon.h>
  35 #include <i386/proc_reg.h>
  36 #include <i386/cpu_threads.h>
  37 #include <i386/mp.h>
  38 #include <i386/cpuid.h>
  39 #include <i386/lock.h>
  40 #include <vm/vm_kern.h>
  41
  42 #ifdef DEBUG
  43 #define DBG(x...)       kprintf(x)
  44 #else
  45 #define DBG(x...)
  46 #endif
  47
  48 /*
  49  * Table of ESCRs and addresses associated with performance counters/CCCRs.
  50  * See Intel SDM Vol 3, Table 15-4 (section 15.9):
  51  */
  52 static uint16_t pmc_escr_addr_table[18][8] = {
  53         [MSR_BPU_COUNTER0] {
  54                 [MSR_BSU_ESCR0]         0x3a0,
  55                 [MSR_FSB_ESCR0]         0x3a2,
  56                 [MSR_MOB_ESCR0]         0x3aa,
  57                 [MSR_PMH_ESCR0]         0x3ac,
  58                 [MSR_BPU_ESCR0]         0x3b2,
  59                 [MSR_IS_ESCR0]          0x3b4,
  60                 [MSR_ITLB_ESCR0]        0x3b6,
  61                 [MSR_IX_ESCR0]          0x3c8,
  62         },
  63         [MSR_BPU_COUNTER1] {
  64                 [MSR_BSU_ESCR0]         0x3a0,
  65                 [MSR_FSB_ESCR0]         0x3a2,
  66                 [MSR_MOB_ESCR0]         0x3aa,
  67                 [MSR_PMH_ESCR0]         0x3ac,
  68                 [MSR_BPU_ESCR0]         0x3b2,
  69                 [MSR_IS_ESCR0]          0x3b4,
  70                 [MSR_ITLB_ESCR0]        0x3b6,
  71                 [MSR_IX_ESCR0]          0x3c8,
  72         },
  73         [MSR_BPU_COUNTER2] {
  74                 [MSR_BSU_ESCR1]         0x3a1,
  75                 [MSR_FSB_ESCR1]         0x3a3,
  76                 [MSR_MOB_ESCR1]         0x3ab,
  77                 [MSR_PMH_ESCR1]         0x3ad,
  78                 [MSR_BPU_ESCR1]         0x3b3,
  79                 [MSR_IS_ESCR1]          0x3b5,
  80                 [MSR_ITLB_ESCR1]        0x3b7,
  81                 [MSR_IX_ESCR1]          0x3c9,
  82         },
  83         [MSR_BPU_COUNTER3] {
  84                 [MSR_BSU_ESCR1]         0x3a1,
  85                 [MSR_FSB_ESCR1]         0x3a3,
  86                 [MSR_MOB_ESCR1]         0x3ab,
  87                 [MSR_PMH_ESCR1]         0x3ad,
  88                 [MSR_BPU_ESCR1]         0x3b3,
  89                 [MSR_IS_ESCR1]          0x3b5,
  90                 [MSR_ITLB_ESCR1]        0x3b7,
  91                 [MSR_IX_ESCR1]          0x3c9,
  92         },
  93         [MSR_MS_COUNTER0] {
  94                 [MSR_MS_ESCR1]          0x3c1,
  95                 [MSR_TBPU_ESCR1]        0x3c3,
  96                 [MSR_TC_ESCR1]          0x3c5,
  97         },
  98         [MSR_MS_COUNTER1] {
  99                 [MSR_MS_ESCR1]          0x3c1,
 100                 [MSR_TBPU_ESCR1]        0x3c3,
 101                 [MSR_TC_ESCR1]          0x3c5,
 102         },
 103         [MSR_MS_COUNTER2] {
 104                 [MSR_MS_ESCR1]          0x3c1,
 105                 [MSR_TBPU_ESCR1]        0x3c3,
 106                 [MSR_TC_ESCR1]          0x3c5,
 107         },
 108         [MSR_MS_COUNTER3] {
 109                 [MSR_MS_ESCR1]          0x3c1,
 110                 [MSR_TBPU_ESCR1]        0x3c3,
 111                 [MSR_TC_ESCR1]          0x3c5,
 112         },
 113         [MSR_FLAME_COUNTER0] {
 114                 [MSR_FIRM_ESCR0]        0x3a4,
 115                 [MSR_FLAME_ESCR0]       0x3a6,
 116                 [MSR_DAC_ESCR0]         0x3a8,
 117                 [MSR_SAT_ESCR0]         0x3ae,
 118                 [MSR_U2L_ESCR0]         0x3b0,
 119         },
 120         [MSR_FLAME_COUNTER1] {
 121                 [MSR_FIRM_ESCR0]        0x3a4,
 122                 [MSR_FLAME_ESCR0]       0x3a6,
 123                 [MSR_DAC_ESCR0]         0x3a8,
 124                 [MSR_SAT_ESCR0]         0x3ae,
 125                 [MSR_U2L_ESCR0]         0x3b0,
 126         },
 127         [MSR_FLAME_COUNTER2] {
 128                 [MSR_FIRM_ESCR1]        0x3a5,
 129                 [MSR_FLAME_ESCR1]       0x3a7,
 130                 [MSR_DAC_ESCR1]         0x3a9,
 131                 [MSR_SAT_ESCR1]         0x3af,
 132                 [MSR_U2L_ESCR1]         0x3b1,
 133         },
 134         [MSR_FLAME_COUNTER3] {
 135                 [MSR_FIRM_ESCR1]        0x3a5,
 136                 [MSR_FLAME_ESCR1]       0x3a7,
 137                 [MSR_DAC_ESCR1]         0x3a9,
 138                 [MSR_SAT_ESCR1]         0x3af,
 139                 [MSR_U2L_ESCR1]         0x3b1,
 140         },
 141         [MSR_IQ_COUNTER0] {
 142                 [MSR_CRU_ESCR0]         0x3b8,
 143                 [MSR_CRU_ESCR2]         0x3cc,
 144                 [MSR_CRU_ESCR4]         0x3e0,
 145                 [MSR_IQ_ESCR0]          0x3ba,
 146                 [MSR_RAT_ESCR0]         0x3bc,
 147                 [MSR_SSU_ESCR0]         0x3be,
 148                 [MSR_AFL_ESCR0]         0x3ca,
 149         },
 150         [MSR_IQ_COUNTER1] {
 151                 [MSR_CRU_ESCR0]         0x3b8,
 152                 [MSR_CRU_ESCR2]         0x3cc,
 153                 [MSR_CRU_ESCR4]         0x3e0,
 154                 [MSR_IQ_ESCR0]          0x3ba,
 155                 [MSR_RAT_ESCR0]         0x3bc,
 156                 [MSR_SSU_ESCR0]         0x3be,
 157                 [MSR_AFL_ESCR0]         0x3ca,
 158         },
 159         [MSR_IQ_COUNTER2] {
 160                 [MSR_CRU_ESCR1]         0x3b9,
 161                 [MSR_CRU_ESCR3]         0x3cd,
 162                 [MSR_CRU_ESCR5]         0x3e1,
 163                 [MSR_IQ_ESCR1]          0x3bb,
 164                 [MSR_RAT_ESCR1]         0x3bd,
 165                 [MSR_AFL_ESCR1]         0x3cb,
 166         },
 167         [MSR_IQ_COUNTER3] {
 168                 [MSR_CRU_ESCR1]         0x3b9,
 169                 [MSR_CRU_ESCR3]         0x3cd,
 170                 [MSR_CRU_ESCR5]         0x3e1,
 171                 [MSR_IQ_ESCR1]          0x3bb,
 172                 [MSR_RAT_ESCR1]         0x3bd,
 173                 [MSR_AFL_ESCR1]         0x3cb,
 174         },
 175         [MSR_IQ_COUNTER4] {
 176                 [MSR_CRU_ESCR0]         0x3b8,
 177                 [MSR_CRU_ESCR2]         0x3cc,
 178                 [MSR_CRU_ESCR4]         0x3e0,
 179                 [MSR_IQ_ESCR0]          0x3ba,
 180                 [MSR_RAT_ESCR0]         0x3bc,
 181                 [MSR_SSU_ESCR0]         0x3be,
 182                 [MSR_AFL_ESCR0]         0x3ca,
 183         },
 184         [MSR_IQ_COUNTER5] {
 185                 [MSR_CRU_ESCR1]         0x3b9,
 186                 [MSR_CRU_ESCR3]         0x3cd,
 187                 [MSR_CRU_ESCR5]         0x3e1,
 188                 [MSR_IQ_ESCR1]          0x3bb,
 189                 [MSR_RAT_ESCR1]         0x3bd,
 190                 [MSR_AFL_ESCR1]         0x3cb,
 191         },
 192 };
 193 #define PMC_ESCR_ADDR(id,esid)  pmc_escr_addr_table[id][esid]
 194
 195 typedef struct {
 196         pmc_id_t        id_max;                 /* Maximum counter id */
 197         pmc_machine_t   machine_type;           /* P6 or P4/Xeon */
 198         uint32_t        msr_counter_base;       /* First counter MSR */
 199         uint32_t        msr_control_base;       /* First control MSR */
 200         boolean_t       reserved[18];           /* Max-sized arrays... */
 201         pmc_ovf_func_t  *ovf_func[18];
 202 #ifdef DEBUG
 203         pmc_cccr_t      cccr_shadow[18];        /* Last cccr values set */
 204         pmc_counter_t   counter_shadow[18];     /* Last counter values set */
 205         uint32_t        ovfs_unexpected[18];    /* Count of unexpected intrs */
 206 #endif
 207 } pmc_table_t;
 208
 209 static pmc_machine_t
 210 _pmc_machine_type(void)
 211 {
 212         i386_cpu_info_t *infop = cpuid_info();
 213
 214         if (strncmp(infop->cpuid_vendor, CPUID_VID_INTEL, sizeof(CPUID_VID_INTEL)) != 0)
 215                 return pmc_none;
 216
 217         if (!pmc_is_available())
 218                 return pmc_none;
 219
 220         switch (infop->cpuid_family) {
 221         case 0x6:
 222                 return pmc_P6;
 223         case 0xf:
 224                 return pmc_P4_Xeon;
 225         default:
 226                 return pmc_unknown;
 227         }
 228 }
 229
 230 static void
 231 pmc_p4_intr(void *state)
 232 {
 233         pmc_table_t     *pmc_table = (pmc_table_t *) cpu_core()->pmc;
 234         uint32_t        cccr_addr;
 235         pmc_cccr_t      cccr;
 236         pmc_id_t        id;
 237         int             my_logical_cpu = cpu_to_logical_cpu(cpu_number());
 238
 239         /*
 240          * Scan through table for reserved counters with overflow and
 241          * with a registered overflow function.
 242          */
 243         for (id = 0; id <= pmc_table->id_max; id++) {
 244                 if (!pmc_table->reserved[id])
 245                         continue;
 246                 cccr_addr = pmc_table->msr_control_base + id;
 247                 cccr.u_u64 = rdmsr64(cccr_addr);
 248 #ifdef DEBUG
 249                 pmc_table->cccr_shadow[id] = cccr;
 250                 *((uint64_t *) &pmc_table->counter_shadow[id]) =
 251                         rdmsr64(pmc_table->msr_counter_base + id);
 252 #endif
 253                 if (cccr.u_htt.ovf == 0)
 254                         continue;
 255                 if ((cccr.u_htt.ovf_pmi_t0 == 1 && my_logical_cpu == 0) ||
 256                     (cccr.u_htt.ovf_pmi_t1 == 1 && my_logical_cpu == 1)) {
 257                         if (pmc_table->ovf_func[id]) {
 258                                 (*pmc_table->ovf_func[id])(id, state);
 259                                 /* func expected to clear overflow */
 260                                 continue;
 261                         }
 262                 }
 263                 /* Clear overflow for unexpected interrupt */
 264 #ifdef DEBUG
 265                 pmc_table->ovfs_unexpected[id]++;
 266 #endif
 267         }
 268 }
 269
 270 static void
 271 pmc_p6_intr(void *state)
 272 {
 273         pmc_table_t     *pmc_table = (pmc_table_t *) cpu_core()->pmc;
 274         pmc_id_t        id;
 275
 276         /*
 277          * Can't determine which counter has overflow
 278          * so call all registered functions.
 279          */
 280         for (id = 0; id <= pmc_table->id_max; id++)
 281                 if (pmc_table->reserved[id] && pmc_table->ovf_func[id])
 282                         (*pmc_table->ovf_func[id])(id, state);
 283 }
 284
 285 int
 286 pmc_init(void)
 287 {
 288         int             ret;
 289         cpu_core_t      *my_core;
 290         pmc_table_t     *pmc_table;
 291         pmc_machine_t   pmc_type;
 292
 293         my_core = cpu_core();
 294         assert(my_core);
 295
 296         pmc_type = _pmc_machine_type();
 297         if (pmc_type == pmc_none) {
 298                 return KERN_FAILURE;
 299         }
 300
 301         pmc_table = (pmc_table_t *) my_core->pmc;
 302         if (pmc_table == NULL) {
 303                 ret = kmem_alloc(kernel_map,
 304                         (void *) &pmc_table, sizeof(pmc_table_t));
 305                 if (ret != KERN_SUCCESS)
 306                         panic("pmc_init() kmem_alloc returned %d\n", ret);
 307                 bzero((void *)pmc_table, sizeof(pmc_table_t));
 308
 309                 pmc_table->machine_type = pmc_type;
 310                 switch (pmc_type) {
 311                 case pmc_P4_Xeon:
 312                         pmc_table->id_max = 17;
 313                         pmc_table->msr_counter_base = MSR_COUNTER_ADDR(0);
 314                         pmc_table->msr_control_base = MSR_CCCR_ADDR(0);
 315                         lapic_set_pmi_func(&pmc_p4_intr);
 316                         break;
 317                 case pmc_P6:
 318                         pmc_table->id_max = 1;
 319                         pmc_table->msr_counter_base = MSR_P6_COUNTER_ADDR(0);
 320                         pmc_table->msr_control_base = MSR_P6_PES_ADDR(0);
 321                         lapic_set_pmi_func(&pmc_p6_intr);
 322                         break;
 323                 default:
 324                         break;
 325                 }
 326                 if (!atomic_cmpxchg((uint32_t *) &my_core->pmc,
 327                                     0, (uint32_t) pmc_table)) {
 328                         kmem_free(kernel_map,
 329                                   (vm_offset_t) pmc_table, sizeof(pmc_table_t));
 330                 }
 331         }
 332         DBG("pmc_init() done for cpu %d my_core->pmc=0x%x type=%d\n",
 333                 cpu_number(), my_core->pmc, pmc_type);
 334
 335         return KERN_SUCCESS;
 336 }
 337
 338 static inline pmc_table_t *
 339 pmc_table_valid(pmc_id_t id)
 340 {
 341         cpu_core_t      *my_core = cpu_core();
 342         pmc_table_t     *pmc_table;
 343
 344         assert(my_core);
 345
 346         pmc_table = (pmc_table_t *) my_core->pmc;
 347         return (pmc_table == NULL ||
 348                 id > pmc_table->id_max ||
 349                 !pmc_table->reserved[id]) ? NULL : pmc_table;
 350 }
 351
 352 int
 353 pmc_machine_type(pmc_machine_t *type)
 354 {
 355         cpu_core_t      *my_core = cpu_core();
 356         pmc_table_t     *pmc_table;
 357
 358         assert(my_core);
 359
 360         pmc_table = (pmc_table_t *) my_core->pmc;
 361         if (pmc_table == NULL)
 362                 return KERN_FAILURE;
 363
 364         *type = pmc_table->machine_type;
 365
 366         return KERN_SUCCESS;
 367 }
 368
 369 int
 370 pmc_reserve(pmc_id_t id)
 371 {
 372         cpu_core_t      *my_core = cpu_core();
 373         pmc_table_t     *pmc_table;
 374
 375         assert(my_core);
 376
 377         pmc_table = (pmc_table_t *) my_core->pmc;
 378         if (pmc_table == NULL)
 379                 return KERN_FAILURE;
 380         if (id > pmc_table->id_max)
 381                 return KERN_INVALID_ARGUMENT;
 382         if (pmc_table->reserved[id])
 383                 return KERN_FAILURE;
 384
 385         pmc_table->reserved[id] = TRUE;
 386
 387         return KERN_SUCCESS;
 388 }
 389
 390 boolean_t
 391 pmc_is_reserved(pmc_id_t id)
 392 {
 393         return pmc_table_valid(id) != NULL;
 394 }
 395
 396 int
 397 pmc_free(pmc_id_t id)
 398 {
 399         pmc_table_t     *pmc_table = pmc_table_valid(id);
 400
 401         if (pmc_table == NULL)
 402                 return KERN_INVALID_ARGUMENT;
 403
 404         pmc_cccr_write(id, 0x0ULL);
 405         pmc_table->reserved[id] = FALSE;
 406         pmc_table->ovf_func[id] = NULL;
 407
 408         return KERN_SUCCESS;
 409 }
 410
 411 int
 412 pmc_counter_read(pmc_id_t id, pmc_counter_t *val)
 413 {
 414         pmc_table_t     *pmc_table = pmc_table_valid(id);
 415
 416         if (pmc_table == NULL)
 417                 return KERN_INVALID_ARGUMENT;
 418
 419         *(uint64_t *)val = rdmsr64(pmc_table->msr_counter_base + id);
 420
 421         return KERN_SUCCESS;
 422 }
 423
 424 int
 425 pmc_counter_write(pmc_id_t id, pmc_counter_t *val)
 426 {
 427         pmc_table_t     *pmc_table = pmc_table_valid(id);
 428
 429         if (pmc_table == NULL)
 430                 return KERN_INVALID_ARGUMENT;
 431
 432         wrmsr64(pmc_table->msr_counter_base + id, *(uint64_t *)val);
 433
 434         return KERN_SUCCESS;
 435 }
 436
 437 int
 438 pmc_cccr_read(pmc_id_t id, pmc_cccr_t *cccr)
 439 {
 440         pmc_table_t     *pmc_table = pmc_table_valid(id);
 441
 442         if (pmc_table == NULL)
 443                 return KERN_INVALID_ARGUMENT;
 444
 445         if (pmc_table->machine_type != pmc_P4_Xeon)
 446                 return KERN_FAILURE;
 447
 448         *(uint64_t *)cccr = rdmsr64(pmc_table->msr_control_base + id);
 449
 450         return KERN_SUCCESS;
 451 }
 452
 453 int
 454 pmc_cccr_write(pmc_id_t id, pmc_cccr_t *cccr)
 455 {
 456         pmc_table_t     *pmc_table = pmc_table_valid(id);
 457
 458         if (pmc_table == NULL)
 459                 return KERN_INVALID_ARGUMENT;
 460
 461         if (pmc_table->machine_type != pmc_P4_Xeon)
 462                 return KERN_FAILURE;
 463
 464         wrmsr64(pmc_table->msr_control_base + id, *(uint64_t *)cccr);
 465
 466         return KERN_SUCCESS;
 467 }
 468
 469 int
 470 pmc_evtsel_read(pmc_id_t id, pmc_evtsel_t *evtsel)
 471 {
 472         pmc_table_t     *pmc_table = pmc_table_valid(id);
 473
 474         if (pmc_table == NULL)
 475                 return KERN_INVALID_ARGUMENT;
 476
 477         if (pmc_table->machine_type != pmc_P6)
 478                 return KERN_FAILURE;
 479
 480         *(uint64_t *)evtsel = rdmsr64(pmc_table->msr_control_base + id);
 481
 482         return KERN_SUCCESS;
 483 }
 484
 485 int
 486 pmc_evtsel_write(pmc_id_t id, pmc_evtsel_t *evtsel)
 487 {
 488         pmc_table_t     *pmc_table = pmc_table_valid(id);
 489
 490         if (pmc_table == NULL)
 491                 return KERN_INVALID_ARGUMENT;
 492
 493         if (pmc_table->machine_type != pmc_P4_Xeon)
 494                 return KERN_FAILURE;
 495
 496         wrmsr64(pmc_table->msr_control_base + id, *(uint64_t *)evtsel);
 497
 498         return KERN_SUCCESS;
 499 }
 500
 501 int
 502 pmc_escr_read(pmc_id_t id, pmc_escr_id_t esid, pmc_escr_t *escr)
 503 {
 504         uint32_t        addr;
 505         pmc_table_t     *pmc_table = pmc_table_valid(id);
 506
 507         if (pmc_table == NULL)
 508                 return KERN_INVALID_ARGUMENT;
 509
 510         if (pmc_table->machine_type != pmc_P4_Xeon)
 511                 return KERN_FAILURE;
 512
 513         if (esid > PMC_ESID_MAX)
 514                 return KERN_INVALID_ARGUMENT;
 515
 516         addr = PMC_ESCR_ADDR(id, esid);
 517         if (addr == 0)
 518                 return KERN_INVALID_ARGUMENT;
 519
 520         *(uint64_t *)escr = rdmsr64(addr);
 521
 522         return KERN_SUCCESS;
 523 }
 524
 525 int
 526 pmc_escr_write(pmc_id_t id, pmc_escr_id_t esid, pmc_escr_t *escr)
 527 {
 528         uint32_t        addr;
 529         pmc_table_t     *pmc_table = pmc_table_valid(id);
 530
 531         if (pmc_table == NULL)
 532                 return KERN_FAILURE;
 533
 534         if (pmc_table->machine_type != pmc_P4_Xeon)
 535                 return KERN_FAILURE;
 536
 537         if (esid > PMC_ESID_MAX)
 538                 return KERN_INVALID_ARGUMENT;
 539
 540         addr = PMC_ESCR_ADDR(id, esid);
 541         if (addr == 0)
 542                 return KERN_INVALID_ARGUMENT;
 543
 544         wrmsr64(addr, *(uint64_t *)escr);
 545
 546         return KERN_SUCCESS;
 547 }
 548
 549 int
 550 pmc_set_ovf_func(pmc_id_t id, pmc_ovf_func_t func)
 551 {
 552         pmc_table_t     *pmc_table = pmc_table_valid(id);
 553
 554         if (pmc_table == NULL)
 555                 return KERN_INVALID_ARGUMENT;
 556
 557         pmc_table->ovf_func[id] = func;
 558
 559         return KERN_SUCCESS;
 560 }