osfmk/i386/perfmon.c

   1 /*
   2  * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * The contents of this file constitute Original Code as defined in and
   7  * are subject to the Apple Public Source License Version 1.1 (the
   8  * "License").  You may not use this file except in compliance with the
   9  * License.  Please obtain a copy of the License at
  10  * http://www.apple.com/publicsource and read it before using this file.
  11  *
  12  * This Original Code and all software distributed under the License are
  13  * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  14  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  15  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
  17  * License for the specific language governing rights and limitations
  18  * under the License.
  19  *
  20  * @APPLE_LICENSE_HEADER_END@
  21  */
  22
  23 #include <mach/std_types.h>
  24 #include <i386/cpu_data.h>
  25 #include <i386/cpu_number.h>
  26 #include <i386/perfmon.h>
  27 #include <i386/proc_reg.h>
  28 #include <i386/cpu_threads.h>
  29 #include <i386/mp.h>
  30 #include <i386/cpuid.h>
  31 #include <i386/lock.h>
  32 #include <vm/vm_kern.h>
  33
  34 #ifdef DEBUG
  35 #define DBG(x...)       kprintf(x)
  36 #else
  37 #define DBG(x...)
  38 #endif
  39
  40 /*
  41  * Table of ESCRs and addresses associated with performance counters/CCCRs.
  42  * See Intel SDM Vol 3, Table 15-4 (section 15.9):
  43  */
  44 static uint16_t pmc_escr_addr_table[18][8] = {
  45         [MSR_BPU_COUNTER0] {
  46                 [MSR_BSU_ESCR0]         0x3a0,
  47                 [MSR_FSB_ESCR0]         0x3a2,
  48                 [MSR_MOB_ESCR0]         0x3aa,
  49                 [MSR_PMH_ESCR0]         0x3ac,
  50                 [MSR_BPU_ESCR0]         0x3b2,
  51                 [MSR_IS_ESCR0]          0x3b4,
  52                 [MSR_ITLB_ESCR0]        0x3b6,
  53                 [MSR_IX_ESCR0]          0x3c8,
  54         },
  55         [MSR_BPU_COUNTER1] {
  56                 [MSR_BSU_ESCR0]         0x3a0,
  57                 [MSR_FSB_ESCR0]         0x3a2,
  58                 [MSR_MOB_ESCR0]         0x3aa,
  59                 [MSR_PMH_ESCR0]         0x3ac,
  60                 [MSR_BPU_ESCR0]         0x3b2,
  61                 [MSR_IS_ESCR0]          0x3b4,
  62                 [MSR_ITLB_ESCR0]        0x3b6,
  63                 [MSR_IX_ESCR0]          0x3c8,
  64         },
  65         [MSR_BPU_COUNTER2] {
  66                 [MSR_BSU_ESCR1]         0x3a1,
  67                 [MSR_FSB_ESCR1]         0x3a3,
  68                 [MSR_MOB_ESCR1]         0x3ab,
  69                 [MSR_PMH_ESCR1]         0x3ad,
  70                 [MSR_BPU_ESCR1]         0x3b3,
  71                 [MSR_IS_ESCR1]          0x3b5,
  72                 [MSR_ITLB_ESCR1]        0x3b7,
  73                 [MSR_IX_ESCR1]          0x3c9,
  74         },
  75         [MSR_BPU_COUNTER3] {
  76                 [MSR_BSU_ESCR1]         0x3a1,
  77                 [MSR_FSB_ESCR1]         0x3a3,
  78                 [MSR_MOB_ESCR1]         0x3ab,
  79                 [MSR_PMH_ESCR1]         0x3ad,
  80                 [MSR_BPU_ESCR1]         0x3b3,
  81                 [MSR_IS_ESCR1]          0x3b5,
  82                 [MSR_ITLB_ESCR1]        0x3b7,
  83                 [MSR_IX_ESCR1]          0x3c9,
  84         },
  85         [MSR_MS_COUNTER0] {
  86                 [MSR_MS_ESCR1]          0x3c1,
  87                 [MSR_TBPU_ESCR1]        0x3c3,
  88                 [MSR_TC_ESCR1]          0x3c5,
  89         },
  90         [MSR_MS_COUNTER1] {
  91                 [MSR_MS_ESCR1]          0x3c1,
  92                 [MSR_TBPU_ESCR1]        0x3c3,
  93                 [MSR_TC_ESCR1]          0x3c5,
  94         },
  95         [MSR_MS_COUNTER2] {
  96                 [MSR_MS_ESCR1]          0x3c1,
  97                 [MSR_TBPU_ESCR1]        0x3c3,
  98                 [MSR_TC_ESCR1]          0x3c5,
  99         },
 100         [MSR_MS_COUNTER3] {
 101                 [MSR_MS_ESCR1]          0x3c1,
 102                 [MSR_TBPU_ESCR1]        0x3c3,
 103                 [MSR_TC_ESCR1]          0x3c5,
 104         },
 105         [MSR_FLAME_COUNTER0] {
 106                 [MSR_FIRM_ESCR0]        0x3a4,
 107                 [MSR_FLAME_ESCR0]       0x3a6,
 108                 [MSR_DAC_ESCR0]         0x3a8,
 109                 [MSR_SAT_ESCR0]         0x3ae,
 110                 [MSR_U2L_ESCR0]         0x3b0,
 111         },
 112         [MSR_FLAME_COUNTER1] {
 113                 [MSR_FIRM_ESCR0]        0x3a4,
 114                 [MSR_FLAME_ESCR0]       0x3a6,
 115                 [MSR_DAC_ESCR0]         0x3a8,
 116                 [MSR_SAT_ESCR0]         0x3ae,
 117                 [MSR_U2L_ESCR0]         0x3b0,
 118         },
 119         [MSR_FLAME_COUNTER2] {
 120                 [MSR_FIRM_ESCR1]        0x3a5,
 121                 [MSR_FLAME_ESCR1]       0x3a7,
 122                 [MSR_DAC_ESCR1]         0x3a9,
 123                 [MSR_SAT_ESCR1]         0x3af,
 124                 [MSR_U2L_ESCR1]         0x3b1,
 125         },
 126         [MSR_FLAME_COUNTER3] {
 127                 [MSR_FIRM_ESCR1]        0x3a5,
 128                 [MSR_FLAME_ESCR1]       0x3a7,
 129                 [MSR_DAC_ESCR1]         0x3a9,
 130                 [MSR_SAT_ESCR1]         0x3af,
 131                 [MSR_U2L_ESCR1]         0x3b1,
 132         },
 133         [MSR_IQ_COUNTER0] {
 134                 [MSR_CRU_ESCR0]         0x3b8,
 135                 [MSR_CRU_ESCR2]         0x3cc,
 136                 [MSR_CRU_ESCR4]         0x3e0,
 137                 [MSR_IQ_ESCR0]          0x3ba,
 138                 [MSR_RAT_ESCR0]         0x3bc,
 139                 [MSR_SSU_ESCR0]         0x3be,
 140                 [MSR_AFL_ESCR0]         0x3ca,
 141         },
 142         [MSR_IQ_COUNTER1] {
 143                 [MSR_CRU_ESCR0]         0x3b8,
 144                 [MSR_CRU_ESCR2]         0x3cc,
 145                 [MSR_CRU_ESCR4]         0x3e0,
 146                 [MSR_IQ_ESCR0]          0x3ba,
 147                 [MSR_RAT_ESCR0]         0x3bc,
 148                 [MSR_SSU_ESCR0]         0x3be,
 149                 [MSR_AFL_ESCR0]         0x3ca,
 150         },
 151         [MSR_IQ_COUNTER2] {
 152                 [MSR_CRU_ESCR1]         0x3b9,
 153                 [MSR_CRU_ESCR3]         0x3cd,
 154                 [MSR_CRU_ESCR5]         0x3e1,
 155                 [MSR_IQ_ESCR1]          0x3bb,
 156                 [MSR_RAT_ESCR1]         0x3bd,
 157                 [MSR_AFL_ESCR1]         0x3cb,
 158         },
 159         [MSR_IQ_COUNTER3] {
 160                 [MSR_CRU_ESCR1]         0x3b9,
 161                 [MSR_CRU_ESCR3]         0x3cd,
 162                 [MSR_CRU_ESCR5]         0x3e1,
 163                 [MSR_IQ_ESCR1]          0x3bb,
 164                 [MSR_RAT_ESCR1]         0x3bd,
 165                 [MSR_AFL_ESCR1]         0x3cb,
 166         },
 167         [MSR_IQ_COUNTER4] {
 168                 [MSR_CRU_ESCR0]         0x3b8,
 169                 [MSR_CRU_ESCR2]         0x3cc,
 170                 [MSR_CRU_ESCR4]         0x3e0,
 171                 [MSR_IQ_ESCR0]          0x3ba,
 172                 [MSR_RAT_ESCR0]         0x3bc,
 173                 [MSR_SSU_ESCR0]         0x3be,
 174                 [MSR_AFL_ESCR0]         0x3ca,
 175         },
 176         [MSR_IQ_COUNTER5] {
 177                 [MSR_CRU_ESCR1]         0x3b9,
 178                 [MSR_CRU_ESCR3]         0x3cd,
 179                 [MSR_CRU_ESCR5]         0x3e1,
 180                 [MSR_IQ_ESCR1]          0x3bb,
 181                 [MSR_RAT_ESCR1]         0x3bd,
 182                 [MSR_AFL_ESCR1]         0x3cb,
 183         },
 184 };
 185 #define PMC_ESCR_ADDR(id,esid)  pmc_escr_addr_table[id][esid]
 186
 187 typedef struct {
 188         pmc_id_t        id_max;                 /* Maximum counter id */
 189         pmc_machine_t   machine_type;           /* P6 or P4/Xeon */
 190         uint32_t        msr_counter_base;       /* First counter MSR */
 191         uint32_t        msr_control_base;       /* First control MSR */
 192         boolean_t       reserved[18];           /* Max-sized arrays... */
 193         pmc_ovf_func_t  *ovf_func[18];
 194 #ifdef DEBUG
 195         pmc_cccr_t      cccr_shadow[18];        /* Last cccr values set */
 196         pmc_counter_t   counter_shadow[18];     /* Last counter values set */
 197         uint32_t        ovfs_unexpected[18];    /* Count of unexpected intrs */
 198 #endif
 199 } pmc_table_t;
 200
 201 static pmc_machine_t
 202 _pmc_machine_type(void)
 203 {
 204         i386_cpu_info_t *infop = cpuid_info();
 205
 206         if (strncmp(infop->cpuid_vendor, CPUID_VID_INTEL, sizeof(CPUID_VID_INTEL)) != 0)
 207                 return pmc_none;
 208
 209         if (!pmc_is_available())
 210                 return pmc_none;
 211
 212         switch (infop->cpuid_family) {
 213         case 0x6:
 214                 return pmc_P6;
 215         case 0xf:
 216                 return pmc_P4_Xeon;
 217         default:
 218                 return pmc_unknown;
 219         }
 220 }
 221
 222 static void
 223 pmc_p4_intr(void *state)
 224 {
 225         pmc_table_t     *pmc_table = (pmc_table_t *) cpu_core()->pmc;
 226         uint32_t        cccr_addr;
 227         pmc_cccr_t      cccr;
 228         pmc_id_t        id;
 229         int             my_logical_cpu = cpu_to_logical_cpu(cpu_number());
 230
 231         /*
 232          * Scan through table for reserved counters with overflow and
 233          * with a registered overflow function.
 234          */
 235         for (id = 0; id <= pmc_table->id_max; id++) {
 236                 if (!pmc_table->reserved[id])
 237                         continue;
 238                 cccr_addr = pmc_table->msr_control_base + id;
 239                 cccr.u_u64 = rdmsr64(cccr_addr);
 240 #ifdef DEBUG
 241                 pmc_table->cccr_shadow[id] = cccr;
 242                 *((uint64_t *) &pmc_table->counter_shadow[id]) =
 243                         rdmsr64(pmc_table->msr_counter_base + id);
 244 #endif
 245                 if (cccr.u_htt.ovf == 0)
 246                         continue;
 247                 if ((cccr.u_htt.ovf_pmi_t0 == 1 && my_logical_cpu == 0) ||
 248                     (cccr.u_htt.ovf_pmi_t1 == 1 && my_logical_cpu == 1)) {
 249                         if (pmc_table->ovf_func[id]) {
 250                                 (*pmc_table->ovf_func[id])(id, state);
 251                                 /* func expected to clear overflow */
 252                                 continue;
 253                         }
 254                 }
 255                 /* Clear overflow for unexpected interrupt */
 256 #ifdef DEBUG
 257                 pmc_table->ovfs_unexpected[id]++;
 258 #endif
 259         }
 260 }
 261
 262 static void
 263 pmc_p6_intr(void *state)
 264 {
 265         pmc_table_t     *pmc_table = (pmc_table_t *) cpu_core()->pmc;
 266         pmc_id_t        id;
 267
 268         /*
 269          * Can't determine which counter has overflow
 270          * so call all registered functions.
 271          */
 272         for (id = 0; id <= pmc_table->id_max; id++)
 273                 if (pmc_table->reserved[id] && pmc_table->ovf_func[id])
 274                         (*pmc_table->ovf_func[id])(id, state);
 275 }
 276
 277 int
 278 pmc_init(void)
 279 {
 280         int             ret;
 281         cpu_core_t      *my_core;
 282         pmc_table_t     *pmc_table;
 283         pmc_machine_t   pmc_type;
 284
 285         my_core = cpu_core();
 286         assert(my_core);
 287
 288         pmc_type = _pmc_machine_type();
 289         if (pmc_type == pmc_none) {
 290                 return KERN_FAILURE;
 291         }
 292
 293         pmc_table = (pmc_table_t *) my_core->pmc;
 294         if (pmc_table == NULL) {
 295                 ret = kmem_alloc(kernel_map,
 296                         (void *) &pmc_table, sizeof(pmc_table_t));
 297                 if (ret != KERN_SUCCESS)
 298                         panic("pmc_init() kmem_alloc returned %d\n", ret);
 299                 bzero((void *)pmc_table, sizeof(pmc_table_t));
 300
 301                 pmc_table->machine_type = pmc_type;
 302                 switch (pmc_type) {
 303                 case pmc_P4_Xeon:
 304                         pmc_table->id_max = 17;
 305                         pmc_table->msr_counter_base = MSR_COUNTER_ADDR(0);
 306                         pmc_table->msr_control_base = MSR_CCCR_ADDR(0);
 307                         lapic_set_pmi_func(&pmc_p4_intr);
 308                         break;
 309                 case pmc_P6:
 310                         pmc_table->id_max = 1;
 311                         pmc_table->msr_counter_base = MSR_P6_COUNTER_ADDR(0);
 312                         pmc_table->msr_control_base = MSR_P6_PES_ADDR(0);
 313                         lapic_set_pmi_func(&pmc_p6_intr);
 314                         break;
 315                 default:
 316                         break;
 317                 }
 318                 if (!atomic_cmpxchg((uint32_t *) &my_core->pmc,
 319                                     0, (uint32_t) pmc_table)) {
 320                         kmem_free(kernel_map,
 321                                   (vm_offset_t) pmc_table, sizeof(pmc_table_t));
 322                 }
 323         }
 324         DBG("pmc_init() done for cpu %d my_core->pmc=0x%x type=%d\n",
 325                 cpu_number(), my_core->pmc, pmc_type);
 326
 327         return KERN_SUCCESS;
 328 }
 329
 330 static inline pmc_table_t *
 331 pmc_table_valid(pmc_id_t id)
 332 {
 333         cpu_core_t      *my_core = cpu_core();
 334         pmc_table_t     *pmc_table;
 335
 336         assert(my_core);
 337
 338         pmc_table = (pmc_table_t *) my_core->pmc;
 339         return (pmc_table == NULL ||
 340                 id > pmc_table->id_max ||
 341                 !pmc_table->reserved[id]) ? NULL : pmc_table;
 342 }
 343
 344 int
 345 pmc_machine_type(pmc_machine_t *type)
 346 {
 347         cpu_core_t      *my_core = cpu_core();
 348         pmc_table_t     *pmc_table;
 349
 350         assert(my_core);
 351
 352         pmc_table = (pmc_table_t *) my_core->pmc;
 353         if (pmc_table == NULL)
 354                 return KERN_FAILURE;
 355
 356         *type = pmc_table->machine_type;
 357
 358         return KERN_SUCCESS;
 359 }
 360
 361 int
 362 pmc_reserve(pmc_id_t id)
 363 {
 364         cpu_core_t      *my_core = cpu_core();
 365         pmc_table_t     *pmc_table;
 366
 367         assert(my_core);
 368
 369         pmc_table = (pmc_table_t *) my_core->pmc;
 370         if (pmc_table == NULL)
 371                 return KERN_FAILURE;
 372         if (id > pmc_table->id_max)
 373                 return KERN_INVALID_ARGUMENT;
 374         if (pmc_table->reserved[id])
 375                 return KERN_FAILURE;
 376
 377         pmc_table->reserved[id] = TRUE;
 378
 379         return KERN_SUCCESS;
 380 }
 381
 382 boolean_t
 383 pmc_is_reserved(pmc_id_t id)
 384 {
 385         return pmc_table_valid(id) != NULL;
 386 }
 387
 388 int
 389 pmc_free(pmc_id_t id)
 390 {
 391         pmc_table_t     *pmc_table = pmc_table_valid(id);
 392
 393         if (pmc_table == NULL)
 394                 return KERN_INVALID_ARGUMENT;
 395
 396         pmc_cccr_write(id, 0x0ULL);
 397         pmc_table->reserved[id] = FALSE;
 398         pmc_table->ovf_func[id] = NULL;
 399
 400         return KERN_SUCCESS;
 401 }
 402
 403 int
 404 pmc_counter_read(pmc_id_t id, pmc_counter_t *val)
 405 {
 406         pmc_table_t     *pmc_table = pmc_table_valid(id);
 407
 408         if (pmc_table == NULL)
 409                 return KERN_INVALID_ARGUMENT;
 410
 411         *(uint64_t *)val = rdmsr64(pmc_table->msr_counter_base + id);
 412
 413         return KERN_SUCCESS;
 414 }
 415
 416 int
 417 pmc_counter_write(pmc_id_t id, pmc_counter_t *val)
 418 {
 419         pmc_table_t     *pmc_table = pmc_table_valid(id);
 420
 421         if (pmc_table == NULL)
 422                 return KERN_INVALID_ARGUMENT;
 423
 424         wrmsr64(pmc_table->msr_counter_base + id, *(uint64_t *)val);
 425
 426         return KERN_SUCCESS;
 427 }
 428
 429 int
 430 pmc_cccr_read(pmc_id_t id, pmc_cccr_t *cccr)
 431 {
 432         pmc_table_t     *pmc_table = pmc_table_valid(id);
 433
 434         if (pmc_table == NULL)
 435                 return KERN_INVALID_ARGUMENT;
 436
 437         if (pmc_table->machine_type != pmc_P4_Xeon)
 438                 return KERN_FAILURE;
 439
 440         *(uint64_t *)cccr = rdmsr64(pmc_table->msr_control_base + id);
 441
 442         return KERN_SUCCESS;
 443 }
 444
 445 int
 446 pmc_cccr_write(pmc_id_t id, pmc_cccr_t *cccr)
 447 {
 448         pmc_table_t     *pmc_table = pmc_table_valid(id);
 449
 450         if (pmc_table == NULL)
 451                 return KERN_INVALID_ARGUMENT;
 452
 453         if (pmc_table->machine_type != pmc_P4_Xeon)
 454                 return KERN_FAILURE;
 455
 456         wrmsr64(pmc_table->msr_control_base + id, *(uint64_t *)cccr);
 457
 458         return KERN_SUCCESS;
 459 }
 460
 461 int
 462 pmc_evtsel_read(pmc_id_t id, pmc_evtsel_t *evtsel)
 463 {
 464         pmc_table_t     *pmc_table = pmc_table_valid(id);
 465
 466         if (pmc_table == NULL)
 467                 return KERN_INVALID_ARGUMENT;
 468
 469         if (pmc_table->machine_type != pmc_P6)
 470                 return KERN_FAILURE;
 471
 472         *(uint64_t *)evtsel = rdmsr64(pmc_table->msr_control_base + id);
 473
 474         return KERN_SUCCESS;
 475 }
 476
 477 int
 478 pmc_evtsel_write(pmc_id_t id, pmc_evtsel_t *evtsel)
 479 {
 480         pmc_table_t     *pmc_table = pmc_table_valid(id);
 481
 482         if (pmc_table == NULL)
 483                 return KERN_INVALID_ARGUMENT;
 484
 485         if (pmc_table->machine_type != pmc_P4_Xeon)
 486                 return KERN_FAILURE;
 487
 488         wrmsr64(pmc_table->msr_control_base + id, *(uint64_t *)evtsel);
 489
 490         return KERN_SUCCESS;
 491 }
 492
 493 int
 494 pmc_escr_read(pmc_id_t id, pmc_escr_id_t esid, pmc_escr_t *escr)
 495 {
 496         uint32_t        addr;
 497         pmc_table_t     *pmc_table = pmc_table_valid(id);
 498
 499         if (pmc_table == NULL)
 500                 return KERN_INVALID_ARGUMENT;
 501
 502         if (pmc_table->machine_type != pmc_P4_Xeon)
 503                 return KERN_FAILURE;
 504
 505         if (esid > PMC_ESID_MAX)
 506                 return KERN_INVALID_ARGUMENT;
 507
 508         addr = PMC_ESCR_ADDR(id, esid);
 509         if (addr == 0)
 510                 return KERN_INVALID_ARGUMENT;
 511
 512         *(uint64_t *)escr = rdmsr64(addr);
 513
 514         return KERN_SUCCESS;
 515 }
 516
 517 int
 518 pmc_escr_write(pmc_id_t id, pmc_escr_id_t esid, pmc_escr_t *escr)
 519 {
 520         uint32_t        addr;
 521         pmc_table_t     *pmc_table = pmc_table_valid(id);
 522
 523         if (pmc_table == NULL)
 524                 return KERN_FAILURE;
 525
 526         if (pmc_table->machine_type != pmc_P4_Xeon)
 527                 return KERN_FAILURE;
 528
 529         if (esid > PMC_ESID_MAX)
 530                 return KERN_INVALID_ARGUMENT;
 531
 532         addr = PMC_ESCR_ADDR(id, esid);
 533         if (addr == 0)
 534                 return KERN_INVALID_ARGUMENT;
 535
 536         wrmsr64(addr, *(uint64_t *)escr);
 537
 538         return KERN_SUCCESS;
 539 }
 540
 541 int
 542 pmc_set_ovf_func(pmc_id_t id, pmc_ovf_func_t func)
 543 {
 544         pmc_table_t     *pmc_table = pmc_table_valid(id);
 545
 546         if (pmc_table == NULL)
 547                 return KERN_INVALID_ARGUMENT;
 548
 549         pmc_table->ovf_func[id] = func;
 550
 551         return KERN_SUCCESS;
 552 }