]> git.saurik.com Git - apple/xnu.git/blob - osfmk/i386/perfmon.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-792.25.20.tar.gz
[apple/xnu.git] / osfmk / i386 / perfmon.c
1 /*
2 * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
11 *
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
18 * under the License.
19 *
20 * @APPLE_LICENSE_HEADER_END@
21 */
22
23 #include <mach/std_types.h>
24 #include <i386/cpu_data.h>
25 #include <i386/cpu_number.h>
26 #include <i386/perfmon.h>
27 #include <i386/proc_reg.h>
28 #include <i386/cpu_threads.h>
29 #include <i386/mp.h>
30 #include <i386/cpuid.h>
31 #include <i386/lock.h>
32 #include <vm/vm_kern.h>
33 #include <kern/task.h>
34
35 #if DEBUG
36 #define DBG(x...) kprintf(x)
37 #else
38 #define DBG(x...)
39 #endif
40
41 decl_simple_lock_data(,pmc_lock)
42 static task_t pmc_owner = TASK_NULL;
43 static int pmc_thread_count = 0;
44
45 /* PMC Facility Owner:
46 * TASK_NULL - no one owns it
47 * kernel_task - owned by pmc
48 * other task - owned by another task
49 */
50
51 /*
52 * Table of ESCRs and addresses associated with performance counters/CCCRs.
53 * See Intel SDM Vol 3, Table 15-4 (section 15.9):
54 */
55 static uint16_t pmc_escr_addr_table[18][8] = {
56 [MSR_BPU_COUNTER0] {
57 [MSR_BSU_ESCR0] 0x3a0,
58 [MSR_FSB_ESCR0] 0x3a2,
59 [MSR_MOB_ESCR0] 0x3aa,
60 [MSR_PMH_ESCR0] 0x3ac,
61 [MSR_BPU_ESCR0] 0x3b2,
62 [MSR_IS_ESCR0] 0x3b4,
63 [MSR_ITLB_ESCR0] 0x3b6,
64 [MSR_IX_ESCR0] 0x3c8,
65 },
66 [MSR_BPU_COUNTER1] {
67 [MSR_BSU_ESCR0] 0x3a0,
68 [MSR_FSB_ESCR0] 0x3a2,
69 [MSR_MOB_ESCR0] 0x3aa,
70 [MSR_PMH_ESCR0] 0x3ac,
71 [MSR_BPU_ESCR0] 0x3b2,
72 [MSR_IS_ESCR0] 0x3b4,
73 [MSR_ITLB_ESCR0] 0x3b6,
74 [MSR_IX_ESCR0] 0x3c8,
75 },
76 [MSR_BPU_COUNTER2] {
77 [MSR_BSU_ESCR1] 0x3a1,
78 [MSR_FSB_ESCR1] 0x3a3,
79 [MSR_MOB_ESCR1] 0x3ab,
80 [MSR_PMH_ESCR1] 0x3ad,
81 [MSR_BPU_ESCR1] 0x3b3,
82 [MSR_IS_ESCR1] 0x3b5,
83 [MSR_ITLB_ESCR1] 0x3b7,
84 [MSR_IX_ESCR1] 0x3c9,
85 },
86 [MSR_BPU_COUNTER3] {
87 [MSR_BSU_ESCR1] 0x3a1,
88 [MSR_FSB_ESCR1] 0x3a3,
89 [MSR_MOB_ESCR1] 0x3ab,
90 [MSR_PMH_ESCR1] 0x3ad,
91 [MSR_BPU_ESCR1] 0x3b3,
92 [MSR_IS_ESCR1] 0x3b5,
93 [MSR_ITLB_ESCR1] 0x3b7,
94 [MSR_IX_ESCR1] 0x3c9,
95 },
96 [MSR_MS_COUNTER0] {
97 [MSR_MS_ESCR1] 0x3c1,
98 [MSR_TBPU_ESCR1] 0x3c3,
99 [MSR_TC_ESCR1] 0x3c5,
100 },
101 [MSR_MS_COUNTER1] {
102 [MSR_MS_ESCR1] 0x3c1,
103 [MSR_TBPU_ESCR1] 0x3c3,
104 [MSR_TC_ESCR1] 0x3c5,
105 },
106 [MSR_MS_COUNTER2] {
107 [MSR_MS_ESCR1] 0x3c1,
108 [MSR_TBPU_ESCR1] 0x3c3,
109 [MSR_TC_ESCR1] 0x3c5,
110 },
111 [MSR_MS_COUNTER3] {
112 [MSR_MS_ESCR1] 0x3c1,
113 [MSR_TBPU_ESCR1] 0x3c3,
114 [MSR_TC_ESCR1] 0x3c5,
115 },
116 [MSR_FLAME_COUNTER0] {
117 [MSR_FIRM_ESCR0] 0x3a4,
118 [MSR_FLAME_ESCR0] 0x3a6,
119 [MSR_DAC_ESCR0] 0x3a8,
120 [MSR_SAT_ESCR0] 0x3ae,
121 [MSR_U2L_ESCR0] 0x3b0,
122 },
123 [MSR_FLAME_COUNTER1] {
124 [MSR_FIRM_ESCR0] 0x3a4,
125 [MSR_FLAME_ESCR0] 0x3a6,
126 [MSR_DAC_ESCR0] 0x3a8,
127 [MSR_SAT_ESCR0] 0x3ae,
128 [MSR_U2L_ESCR0] 0x3b0,
129 },
130 [MSR_FLAME_COUNTER2] {
131 [MSR_FIRM_ESCR1] 0x3a5,
132 [MSR_FLAME_ESCR1] 0x3a7,
133 [MSR_DAC_ESCR1] 0x3a9,
134 [MSR_SAT_ESCR1] 0x3af,
135 [MSR_U2L_ESCR1] 0x3b1,
136 },
137 [MSR_FLAME_COUNTER3] {
138 [MSR_FIRM_ESCR1] 0x3a5,
139 [MSR_FLAME_ESCR1] 0x3a7,
140 [MSR_DAC_ESCR1] 0x3a9,
141 [MSR_SAT_ESCR1] 0x3af,
142 [MSR_U2L_ESCR1] 0x3b1,
143 },
144 [MSR_IQ_COUNTER0] {
145 [MSR_CRU_ESCR0] 0x3b8,
146 [MSR_CRU_ESCR2] 0x3cc,
147 [MSR_CRU_ESCR4] 0x3e0,
148 [MSR_IQ_ESCR0] 0x3ba,
149 [MSR_RAT_ESCR0] 0x3bc,
150 [MSR_SSU_ESCR0] 0x3be,
151 [MSR_AFL_ESCR0] 0x3ca,
152 },
153 [MSR_IQ_COUNTER1] {
154 [MSR_CRU_ESCR0] 0x3b8,
155 [MSR_CRU_ESCR2] 0x3cc,
156 [MSR_CRU_ESCR4] 0x3e0,
157 [MSR_IQ_ESCR0] 0x3ba,
158 [MSR_RAT_ESCR0] 0x3bc,
159 [MSR_SSU_ESCR0] 0x3be,
160 [MSR_AFL_ESCR0] 0x3ca,
161 },
162 [MSR_IQ_COUNTER2] {
163 [MSR_CRU_ESCR1] 0x3b9,
164 [MSR_CRU_ESCR3] 0x3cd,
165 [MSR_CRU_ESCR5] 0x3e1,
166 [MSR_IQ_ESCR1] 0x3bb,
167 [MSR_RAT_ESCR1] 0x3bd,
168 [MSR_AFL_ESCR1] 0x3cb,
169 },
170 [MSR_IQ_COUNTER3] {
171 [MSR_CRU_ESCR1] 0x3b9,
172 [MSR_CRU_ESCR3] 0x3cd,
173 [MSR_CRU_ESCR5] 0x3e1,
174 [MSR_IQ_ESCR1] 0x3bb,
175 [MSR_RAT_ESCR1] 0x3bd,
176 [MSR_AFL_ESCR1] 0x3cb,
177 },
178 [MSR_IQ_COUNTER4] {
179 [MSR_CRU_ESCR0] 0x3b8,
180 [MSR_CRU_ESCR2] 0x3cc,
181 [MSR_CRU_ESCR4] 0x3e0,
182 [MSR_IQ_ESCR0] 0x3ba,
183 [MSR_RAT_ESCR0] 0x3bc,
184 [MSR_SSU_ESCR0] 0x3be,
185 [MSR_AFL_ESCR0] 0x3ca,
186 },
187 [MSR_IQ_COUNTER5] {
188 [MSR_CRU_ESCR1] 0x3b9,
189 [MSR_CRU_ESCR3] 0x3cd,
190 [MSR_CRU_ESCR5] 0x3e1,
191 [MSR_IQ_ESCR1] 0x3bb,
192 [MSR_RAT_ESCR1] 0x3bd,
193 [MSR_AFL_ESCR1] 0x3cb,
194 },
195 };
196 #define PMC_ESCR_ADDR(id,esid) pmc_escr_addr_table[id][esid]
197
198 typedef struct {
199 pmc_id_t id_max; /* Maximum counter id */
200 pmc_machine_t machine_type; /* P6 or P4/Xeon */
201 uint32_t msr_counter_base; /* First counter MSR */
202 uint32_t msr_control_base; /* First control MSR */
203 boolean_t reserved[18]; /* Max-sized arrays... */
204 pmc_ovf_func_t *ovf_func[18];
205 #ifdef DEBUG
206 pmc_cccr_t cccr_shadow[18]; /* Last cccr values set */
207 pmc_counter_t counter_shadow[18]; /* Last counter values set */
208 uint32_t ovfs_unexpected[18]; /* Count of unexpected intrs */
209 #endif
210 } pmc_table_t;
211
212 static pmc_machine_t
213 _pmc_machine_type(void)
214 {
215 i386_cpu_info_t *infop = cpuid_info();
216
217 if (strncmp(infop->cpuid_vendor, CPUID_VID_INTEL, sizeof(CPUID_VID_INTEL)) != 0)
218 return pmc_none;
219
220 if (!pmc_is_available())
221 return pmc_none;
222
223 switch (infop->cpuid_family) {
224 case 0x6:
225 return pmc_P6;
226 case 0xf:
227 return pmc_P4_Xeon;
228 default:
229 return pmc_unknown;
230 }
231 }
232
233 static void
234 pmc_p4_intr(void *state)
235 {
236 pmc_table_t *pmc_table = (pmc_table_t *) cpu_core()->pmc;
237 uint32_t cccr_addr;
238 pmc_cccr_t cccr;
239 pmc_id_t id;
240 int my_logical_cpu = cpu_to_logical_cpu(cpu_number());
241
242 /*
243 * Scan through table for reserved counters with overflow and
244 * with a registered overflow function.
245 */
246 for (id = 0; id <= pmc_table->id_max; id++) {
247 if (!pmc_table->reserved[id])
248 continue;
249 cccr_addr = pmc_table->msr_control_base + id;
250 cccr.u_u64 = rdmsr64(cccr_addr);
251 #ifdef DEBUG
252 pmc_table->cccr_shadow[id] = cccr;
253 *((uint64_t *) &pmc_table->counter_shadow[id]) =
254 rdmsr64(pmc_table->msr_counter_base + id);
255 #endif
256 if (cccr.u_htt.ovf == 0)
257 continue;
258 if ((cccr.u_htt.ovf_pmi_t0 == 1 && my_logical_cpu == 0) ||
259 (cccr.u_htt.ovf_pmi_t1 == 1 && my_logical_cpu == 1)) {
260 if (pmc_table->ovf_func[id]) {
261 (*pmc_table->ovf_func[id])(id, state);
262 /* func expected to clear overflow */
263 continue;
264 }
265 }
266 /* Clear overflow for unexpected interrupt */
267 #ifdef DEBUG
268 pmc_table->ovfs_unexpected[id]++;
269 #endif
270 }
271 }
272
273 static void
274 pmc_p6_intr(void *state)
275 {
276 pmc_table_t *pmc_table = (pmc_table_t *) cpu_core()->pmc;
277 pmc_id_t id;
278
279 /*
280 * Can't determine which counter has overflow
281 * so call all registered functions.
282 */
283 for (id = 0; id <= pmc_table->id_max; id++)
284 if (pmc_table->reserved[id] && pmc_table->ovf_func[id])
285 (*pmc_table->ovf_func[id])(id, state);
286 }
287
288 void *
289 pmc_alloc(void)
290 {
291 int ret;
292 pmc_table_t *pmc_table;
293 pmc_machine_t pmc_type;
294
295 pmc_type = _pmc_machine_type();
296 if (pmc_type == pmc_none) {
297 return NULL;
298 }
299
300 ret = kmem_alloc(kernel_map,
301 (void *) &pmc_table, sizeof(pmc_table_t));
302 if (ret != KERN_SUCCESS)
303 panic("pmc_init() kmem_alloc returned %d\n", ret);
304 bzero((void *)pmc_table, sizeof(pmc_table_t));
305
306 pmc_table->machine_type = pmc_type;
307 switch (pmc_type) {
308 case pmc_P4_Xeon:
309 pmc_table->id_max = 17;
310 pmc_table->msr_counter_base = MSR_COUNTER_ADDR(0);
311 pmc_table->msr_control_base = MSR_CCCR_ADDR(0);
312 lapic_set_pmi_func(&pmc_p4_intr);
313 break;
314 case pmc_P6:
315 pmc_table->id_max = 1;
316 pmc_table->msr_counter_base = MSR_P6_COUNTER_ADDR(0);
317 pmc_table->msr_control_base = MSR_P6_PES_ADDR(0);
318 lapic_set_pmi_func(&pmc_p6_intr);
319 break;
320 default:
321 break;
322 }
323 return (void *) pmc_table;
324 }
325
326
327 static inline pmc_table_t *
328 pmc_table_valid(pmc_id_t id)
329 {
330 cpu_core_t *my_core = cpu_core();
331 pmc_table_t *pmc_table;
332
333 assert(my_core);
334
335 pmc_table = (pmc_table_t *) my_core->pmc;
336 return (pmc_table == NULL ||
337 id > pmc_table->id_max ||
338 !pmc_table->reserved[id]) ? NULL : pmc_table;
339 }
340
341 int
342 pmc_machine_type(pmc_machine_t *type)
343 {
344 cpu_core_t *my_core = cpu_core();
345 pmc_table_t *pmc_table;
346
347 assert(my_core);
348
349 pmc_table = (pmc_table_t *) my_core->pmc;
350 if (pmc_table == NULL)
351 return KERN_FAILURE;
352
353 *type = pmc_table->machine_type;
354
355 return KERN_SUCCESS;
356 }
357
358 int
359 pmc_reserve(pmc_id_t id)
360 {
361 cpu_core_t *my_core = cpu_core();
362 pmc_table_t *pmc_table;
363
364 assert(my_core);
365
366 pmc_table = (pmc_table_t *) my_core->pmc;
367 if (pmc_table == NULL)
368 return KERN_FAILURE;
369 if (id > pmc_table->id_max)
370 return KERN_INVALID_ARGUMENT;
371 if (pmc_table->reserved[id])
372 return KERN_FAILURE;
373
374 pmc_table->reserved[id] = TRUE;
375
376 return KERN_SUCCESS;
377 }
378
379 boolean_t
380 pmc_is_reserved(pmc_id_t id)
381 {
382 return pmc_table_valid(id) != NULL;
383 }
384
385 int
386 pmc_free(pmc_id_t id)
387 {
388 pmc_table_t *pmc_table = pmc_table_valid(id);
389
390 if (pmc_table == NULL)
391 return KERN_INVALID_ARGUMENT;
392
393 pmc_cccr_write(id, 0x0ULL);
394 pmc_table->reserved[id] = FALSE;
395 pmc_table->ovf_func[id] = NULL;
396
397 return KERN_SUCCESS;
398 }
399
400 int
401 pmc_counter_read(pmc_id_t id, pmc_counter_t *val)
402 {
403 pmc_table_t *pmc_table = pmc_table_valid(id);
404
405 if (pmc_table == NULL)
406 return KERN_INVALID_ARGUMENT;
407
408 *(uint64_t *)val = rdmsr64(pmc_table->msr_counter_base + id);
409
410 return KERN_SUCCESS;
411 }
412
413 int
414 pmc_counter_write(pmc_id_t id, pmc_counter_t *val)
415 {
416 pmc_table_t *pmc_table = pmc_table_valid(id);
417
418 if (pmc_table == NULL)
419 return KERN_INVALID_ARGUMENT;
420
421 wrmsr64(pmc_table->msr_counter_base + id, *(uint64_t *)val);
422
423 return KERN_SUCCESS;
424 }
425
426 int
427 pmc_cccr_read(pmc_id_t id, pmc_cccr_t *cccr)
428 {
429 pmc_table_t *pmc_table = pmc_table_valid(id);
430
431 if (pmc_table == NULL)
432 return KERN_INVALID_ARGUMENT;
433
434 if (pmc_table->machine_type != pmc_P4_Xeon)
435 return KERN_FAILURE;
436
437 *(uint64_t *)cccr = rdmsr64(pmc_table->msr_control_base + id);
438
439 return KERN_SUCCESS;
440 }
441
442 int
443 pmc_cccr_write(pmc_id_t id, pmc_cccr_t *cccr)
444 {
445 pmc_table_t *pmc_table = pmc_table_valid(id);
446
447 if (pmc_table == NULL)
448 return KERN_INVALID_ARGUMENT;
449
450 if (pmc_table->machine_type != pmc_P4_Xeon)
451 return KERN_FAILURE;
452
453 wrmsr64(pmc_table->msr_control_base + id, *(uint64_t *)cccr);
454
455 return KERN_SUCCESS;
456 }
457
458 int
459 pmc_evtsel_read(pmc_id_t id, pmc_evtsel_t *evtsel)
460 {
461 pmc_table_t *pmc_table = pmc_table_valid(id);
462
463 if (pmc_table == NULL)
464 return KERN_INVALID_ARGUMENT;
465
466 if (pmc_table->machine_type != pmc_P6)
467 return KERN_FAILURE;
468
469 *(uint64_t *)evtsel = rdmsr64(pmc_table->msr_control_base + id);
470
471 return KERN_SUCCESS;
472 }
473
474 int
475 pmc_evtsel_write(pmc_id_t id, pmc_evtsel_t *evtsel)
476 {
477 pmc_table_t *pmc_table = pmc_table_valid(id);
478
479 if (pmc_table == NULL)
480 return KERN_INVALID_ARGUMENT;
481
482 if (pmc_table->machine_type != pmc_P4_Xeon)
483 return KERN_FAILURE;
484
485 wrmsr64(pmc_table->msr_control_base + id, *(uint64_t *)evtsel);
486
487 return KERN_SUCCESS;
488 }
489
490 int
491 pmc_escr_read(pmc_id_t id, pmc_escr_id_t esid, pmc_escr_t *escr)
492 {
493 uint32_t addr;
494 pmc_table_t *pmc_table = pmc_table_valid(id);
495
496 if (pmc_table == NULL)
497 return KERN_INVALID_ARGUMENT;
498
499 if (pmc_table->machine_type != pmc_P4_Xeon)
500 return KERN_FAILURE;
501
502 if (esid > PMC_ESID_MAX)
503 return KERN_INVALID_ARGUMENT;
504
505 addr = PMC_ESCR_ADDR(id, esid);
506 if (addr == 0)
507 return KERN_INVALID_ARGUMENT;
508
509 *(uint64_t *)escr = rdmsr64(addr);
510
511 return KERN_SUCCESS;
512 }
513
514 int
515 pmc_escr_write(pmc_id_t id, pmc_escr_id_t esid, pmc_escr_t *escr)
516 {
517 uint32_t addr;
518 pmc_table_t *pmc_table = pmc_table_valid(id);
519
520 if (pmc_table == NULL)
521 return KERN_FAILURE;
522
523 if (pmc_table->machine_type != pmc_P4_Xeon)
524 return KERN_FAILURE;
525
526 if (esid > PMC_ESID_MAX)
527 return KERN_INVALID_ARGUMENT;
528
529 addr = PMC_ESCR_ADDR(id, esid);
530 if (addr == 0)
531 return KERN_INVALID_ARGUMENT;
532
533 wrmsr64(addr, *(uint64_t *)escr);
534
535 return KERN_SUCCESS;
536 }
537
538 int
539 pmc_set_ovf_func(pmc_id_t id, pmc_ovf_func_t func)
540 {
541 pmc_table_t *pmc_table = pmc_table_valid(id);
542
543 if (pmc_table == NULL)
544 return KERN_INVALID_ARGUMENT;
545
546 pmc_table->ovf_func[id] = func;
547
548 return KERN_SUCCESS;
549 }
550
551 int
552 pmc_acquire(task_t task)
553 {
554 kern_return_t retval = KERN_SUCCESS;
555
556 simple_lock(&pmc_lock);
557
558 if(pmc_owner == task) {
559 DBG("pmc_acquire - "
560 "ACQUIRED: already owner\n");
561 retval = KERN_SUCCESS;
562 /* already own it */
563 } else if(pmc_owner == TASK_NULL) { /* no one owns it */
564 pmc_owner = task;
565 pmc_thread_count = 0;
566 DBG("pmc_acquire - "
567 "ACQUIRED: no current owner - made new owner\n");
568 retval = KERN_SUCCESS;
569 } else { /* someone already owns it */
570 if(pmc_owner == kernel_task) {
571 if(pmc_thread_count == 0) {
572 /* kernel owns it but no threads using it */
573 pmc_owner = task;
574 pmc_thread_count = 0;
575 DBG("pmc_acquire - "
576 "ACQUIRED: owned by kernel, no threads\n");
577 retval = KERN_SUCCESS;
578 } else {
579 DBG("pmc_acquire - "
580 "DENIED: owned by kernel, in use\n");
581 retval = KERN_RESOURCE_SHORTAGE;
582 }
583 } else { /* non-kernel owner */
584 DBG("pmc_acquire - "
585 "DENIED: owned by another task\n");
586 retval = KERN_RESOURCE_SHORTAGE;
587 }
588 }
589
590 simple_unlock(&pmc_lock);
591 return retval;
592 }
593
594 int
595 pmc_release(task_t task)
596 {
597 kern_return_t retval = KERN_SUCCESS;
598 task_t old_pmc_owner = pmc_owner;
599
600 simple_lock(&pmc_lock);
601
602 if(task != pmc_owner) {
603 retval = KERN_NO_ACCESS;
604 } else {
605 if(old_pmc_owner == kernel_task) {
606 if(pmc_thread_count>0) {
607 DBG("pmc_release - "
608 "NOT RELEASED: owned by kernel, in use\n");
609 retval = KERN_NO_ACCESS;
610 } else {
611 DBG("pmc_release - "
612 "RELEASED: was owned by kernel\n");
613 pmc_owner = TASK_NULL;
614 retval = KERN_SUCCESS;
615 }
616 } else {
617 DBG("pmc_release - "
618 "RELEASED: was owned by user\n");
619 pmc_owner = TASK_NULL;
620 retval = KERN_SUCCESS;
621 }
622 }
623
624 simple_unlock(&pmc_lock);
625 return retval;
626 }
627
mirror of XNU