]> git.saurik.com Git - apple/xnu.git/blob - osfmk/arm64/kpc.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-4903.221.2.tar.gz
[apple/xnu.git] / osfmk / arm64 / kpc.c
1 /*
2 * Copyright (c) 2012-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <arm/cpu_data_internal.h>
30 #include <arm/cpu_internal.h>
31 #include <kern/kalloc.h>
32 #include <kern/kpc.h>
33 #include <kern/thread.h>
34 #include <kern/processor.h>
35 #include <mach/mach_types.h>
36 #include <machine/machine_routines.h>
37 #include <stdint.h>
38 #include <sys/errno.h>
39
40 #if MONOTONIC
41 #include <kern/monotonic.h>
42 #endif /* MONOTONIC */
43
44 void kpc_pmi_handler(unsigned int ctr);
45
46 /*
47 * PMCs 8 and 9 were added to Hurricane and to maintain the existing bit
48 * positions of the other PMCs, their configuration bits start at position 32.
49 */
50 #define PMCR_PMC_8_9_OFFSET (32)
51 #define PMCR_PMC_8_9_SHIFT(PMC) (((PMC) - 8) + PMCR_PMC_8_9_OFFSET)
52 #define PMCR_PMC_SHIFT(PMC) (((PMC) <= 7) ? (PMC) : \
53 PMCR_PMC_8_9_SHIFT(PMC))
54
55 /*
56 * PMCR0 controls enabling, interrupts, and overflow of performance counters.
57 */
58
59 /* PMC is enabled */
60 #define PMCR0_PMC_ENABLE_MASK(PMC) (UINT64_C(0x1) << PMCR_PMC_SHIFT(PMC))
61 #define PMCR0_PMC_DISABLE_MASK(PMC) (~PMCR0_PMC_ENABLE_MASK(PMC))
62
63 /* how interrupts are generated on PMIs */
64 #define PMCR0_INTGEN_SHIFT (8)
65 #define PMCR0_INTGEN_MASK (UINT64_C(0x7) << PMCR0_INTGEN_SHIFT)
66 #define PMCR0_INTGEN_OFF (UINT64_C(0) << PMCR0_INTGEN_SHIFT)
67 #define PMCR0_INTGEN_PMI (UINT64_C(1) << PMCR0_INTGEN_SHIFT)
68 #define PMCR0_INTGEN_AIC (UINT64_C(2) << PMCR0_INTGEN_SHIFT)
69 #define PMCR0_INTGEN_DBG_HLT (UINT64_C(3) << PMCR0_INTGEN_SHIFT)
70 #define PMCR0_INTGEN_FIQ (UINT64_C(4) << PMCR0_INTGEN_SHIFT)
71
72 /* 10 unused */
73
74 /* set by hardware if PMI was generated */
75 #define PMCR0_PMAI_SHIFT (11)
76 #define PMCR0_PMAI_MASK (UINT64_C(1) << PMCR0_PMAI_SHIFT)
77
78 /* overflow on a PMC generates an interrupt */
79 #define PMCR0_PMI_OFFSET (12)
80 #define PMCR0_PMI_SHIFT(PMC) (PMCR0_PMI_OFFSET + PMCR_PMC_SHIFT(PMC))
81 #define PMCR0_PMI_ENABLE_MASK(PMC) (UINT64_C(1) << PMCR0_PMI_SHIFT(PMC))
82 #define PMCR0_PMI_DISABLE_MASK(PMC) (~PMCR0_PMI_ENABLE_MASK(PMC))
83
84 /* disable counting when a PMI is signaled (except for AIC interrupts) */
85 #define PMCR0_DISCNT_SHIFT (20)
86 #define PMCR0_DISCNT_ENABLE_MASK (UINT64_C(1) << PMCR0_DISCNT_SHIFT)
87 #define PMCR0_DISCNT_DISABLE_MASK (~PMCR0_DISCNT_ENABLE_MASK)
88
89 /* 21 unused */
90
91 /* block PMIs until ERET retires */
92 #define PMCR0_WFRFE_SHIFT (22)
93 #define PMCR0_WFRFE_ENABLE_MASK (UINT64_C(1) << PMCR0_WFRE_SHIFT)
94 #define PMCR0_WFRFE_DISABLE_MASK (~PMCR0_WFRFE_ENABLE_MASK)
95
96 /* count global L2C events */
97 #define PMCR0_L2CGLOBAL_SHIFT (23)
98 #define PMCR0_L2CGLOBAL_ENABLE_MASK (UINT64_C(1) << PMCR0_L2CGLOBAL_SHIFT)
99 #define PMCR0_L2CGLOBAL_DISABLE_MASK (~PMCR0_L2CGLOBAL_ENABLE_MASK)
100
101 /* allow user mode access to configuration registers */
102 #define PMCR0_USEREN_SHIFT (30)
103 #define PMCR0_USEREN_ENABLE_MASK (UINT64_C(1) << PMCR0_USEREN_SHIFT)
104 #define PMCR0_USEREN_DISABLE_MASK (~PMCR0_USEREN_ENABLE_MASK)
105
106 /* force the CPMU clocks in case of a clocking bug */
107 #define PMCR0_CLKEN_SHIFT (31)
108 #define PMCR0_CLKEN_ENABLE_MASK (UINT64_C(1) << PMCR0_USEREN_SHIFT)
109 #define PMCR0_CLKEN_DISABLE_MASK (~PMCR0_CLKEN_ENABLE_MASK)
110
111 /* 32 - 44 mirror the low bits for PMCs 8 and 9 */
112
113 /* PMCR1 enables counters in different processor modes */
114
115 #define PMCR1_EL0_A32_OFFSET (0)
116 #define PMCR1_EL0_A64_OFFSET (8)
117 #define PMCR1_EL1_A64_OFFSET (16)
118 #define PMCR1_EL3_A64_OFFSET (24)
119
120 #define PMCR1_EL0_A32_SHIFT(PMC) (PMCR1_EL0_A32_OFFSET + PMCR_PMC_SHIFT(PMC))
121 #define PMCR1_EL0_A64_SHIFT(PMC) (PMCR1_EL0_A64_OFFSET + PMCR_PMC_SHIFT(PMC))
122 #define PMCR1_EL1_A64_SHIFT(PMC) (PMCR1_EL1_A64_OFFSET + PMCR_PMC_SHIFT(PMC))
123 #define PMCR1_EL3_A64_SHIFT(PMC) (PMCR1_EL0_A64_OFFSET + PMCR_PMC_SHIFT(PMC))
124
125 #define PMCR1_EL0_A32_ENABLE_MASK(PMC) (UINT64_C(1) << PMCR1_EL0_A32_SHIFT(PMC))
126 #define PMCR1_EL0_A64_ENABLE_MASK(PMC) (UINT64_C(1) << PMCR1_EL0_A64_SHIFT(PMC))
127 #define PMCR1_EL1_A64_ENABLE_MASK(PMC) (UINT64_C(1) << PMCR1_EL1_A64_SHIFT(PMC))
128 /* PMCR1_EL3_A64 is not supported on PMCs 8 and 9 */
129 #if NO_MONITOR
130 #define PMCR1_EL3_A64_ENABLE_MASK(PMC) UINT64_C(0)
131 #else
132 #define PMCR1_EL3_A64_ENABLE_MASK(PMC) (UINT64_C(1) << PMCR1_EL3_A64_SHIFT(PMC))
133 #endif
134
135 #define PMCR1_EL_ALL_ENABLE_MASK(PMC) (PMCR1_EL0_A32_ENABLE_MASK(PMC) | \
136 PMCR1_EL0_A64_ENABLE_MASK(PMC) | \
137 PMCR1_EL1_A64_ENABLE_MASK(PMC) | \
138 PMCR1_EL3_A64_ENABLE_MASK(PMC))
139 #define PMCR1_EL_ALL_DISABLE_MASK(PMC) (~PMCR1_EL_ALL_ENABLE_MASK(PMC))
140
141 /* PMESR0 and PMESR1 are event selection registers */
142
143 /* PMESR0 selects which event is counted on PMCs 2, 3, 4, and 5 */
144 /* PMESR1 selects which event is counted on PMCs 6, 7, 8, and 9 */
145
146 #define PMESR_PMC_WIDTH (8)
147 #define PMESR_PMC_MASK (UINT8_MAX)
148 #define PMESR_SHIFT(PMC, OFF) (8 * ((PMC) - (OFF)))
149 #define PMESR_EVT_MASK(PMC, OFF) (PMESR_PMC_MASK << PMESR_SHIFT(PMC, OFF))
150 #define PMESR_EVT_CLEAR(PMC, OFF) (~PMESR_EVT_MASK(PMC, OFF))
151
152 #define PMESR_EVT_DECODE(PMESR, PMC, OFF) \
153 (((PMESR) >> PMESR_SHIFT(PMC, OFF)) & PMESR_PMC_MASK)
154 #define PMESR_EVT_ENCODE(EVT, PMC, OFF) \
155 (((EVT) & PMESR_PMC_MASK) << PMESR_SHIFT(PMC, OFF))
156
157 /* system registers in the CPMU */
158
159 #define SREG_PMCR0 "S3_1_c15_c0_0"
160 #define SREG_PMCR1 "S3_1_c15_c1_0"
161 #define SREG_PMCR2 "S3_1_c15_c2_0"
162 #define SREG_PMCR3 "S3_1_c15_c3_0"
163 #define SREG_PMCR4 "S3_1_c15_c4_0"
164 #define SREG_PMESR0 "S3_1_c15_c5_0"
165 #define SREG_PMESR1 "S3_1_c15_c6_0"
166 #define SREG_PMSR "S3_1_c15_c13_0"
167 #define SREG_OPMAT0 "S3_1_c15_c7_0"
168 #define SREG_OPMAT1 "S3_1_c15_c8_0"
169 #define SREG_OPMSK0 "S3_1_c15_c9_0"
170 #define SREG_OPMSK1 "S3_1_c15_c10_0"
171
172 #define SREG_PMC0 "S3_2_c15_c0_0"
173 #define SREG_PMC1 "S3_2_c15_c1_0"
174 #define SREG_PMC2 "S3_2_c15_c2_0"
175 #define SREG_PMC3 "S3_2_c15_c3_0"
176 #define SREG_PMC4 "S3_2_c15_c4_0"
177 #define SREG_PMC5 "S3_2_c15_c5_0"
178 #define SREG_PMC6 "S3_2_c15_c6_0"
179 #define SREG_PMC7 "S3_2_c15_c7_0"
180 #define SREG_PMC8 "S3_2_c15_c9_0"
181 #define SREG_PMC9 "S3_2_c15_c10_0"
182
183 #if !defined(APPLECYCLONE)
184 #define SREG_PMMMAP "S3_2_c15_c15_0"
185 #define SREG_PMTRHLD2 "S3_2_c15_c14_0"
186 #define SREG_PMTRHLD4 "S3_2_c15_c13_0"
187 #define SREG_PMTRHLD6 "S3_2_c15_c12_0"
188 #endif
189
190 /*
191 * The low 8 bits of a configuration words select the event to program on
192 * PMESR{0,1}. Bits 16-19 are mapped to PMCR1 bits.
193 */
194 #define CFGWORD_EL0A32EN_MASK (0x10000)
195 #define CFGWORD_EL0A64EN_MASK (0x20000)
196 #define CFGWORD_EL1EN_MASK (0x40000)
197 #define CFGWORD_EL3EN_MASK (0x80000)
198 #define CFGWORD_ALLMODES_MASK (0xf0000)
199
200 /* ACC offsets for PIO */
201 #define ACC_CPMU_PMC0_OFFSET (0x200)
202 #define ACC_CPMU_PMC8_OFFSET (0x280)
203
204 /*
205 * Macros for reading and writing system registers.
206 *
207 * SR must be one of the SREG_* defines above.
208 */
209 #define SREG_WRITE(SR, V) __asm__ volatile("msr " SR ", %0 ; isb" : : "r"(V))
210 #define SREG_READ(SR) ({ uint64_t VAL; \
211 __asm__ volatile("mrs %0, " SR : "=r"(VAL)); \
212 VAL; })
213
214 /*
215 * Configuration registers that can be controlled by RAWPMU:
216 *
217 * All: PMCR2-4, OPMAT0-1, OPMSK0-1.
218 * Typhoon/Twister/Hurricane: PMMMAP, PMTRHLD2/4/6.
219 */
220 #if defined(APPLECYCLONE)
221 #define RAWPMU_CONFIG_COUNT 7
222 #else
223 #define RAWPMU_CONFIG_COUNT 11
224 #endif
225
226 /* TODO: allocate dynamically */
227 static uint64_t saved_PMCR[MAX_CPUS][2];
228 static uint64_t saved_PMESR[MAX_CPUS][2];
229 static uint64_t saved_RAWPMU[MAX_CPUS][RAWPMU_CONFIG_COUNT];
230 static uint64_t saved_counter[MAX_CPUS][KPC_MAX_COUNTERS];
231 static uint64_t kpc_running_cfg_pmc_mask = 0;
232 static uint32_t kpc_running_classes = 0;
233 static uint32_t kpc_configured = 0;
234
235 /*
236 * The whitelist is disabled by default on development/debug kernel. This can
237 * be changed via the kpc.disable_whitelist sysctl. The whitelist is enabled on
238 * release kernel and cannot be disabled.
239 */
240 #if DEVELOPMENT || DEBUG
241 static boolean_t whitelist_disabled = TRUE;
242 #else
243 static boolean_t whitelist_disabled = FALSE;
244 #endif
245
246 /* List of counter events that are allowed externally */
247 static kpc_config_t whitelist[] = {
248 0, /* NO_EVENT */
249
250 #if defined(APPLECYCLONE)
251 0x02, /* CORE_CYCLE */
252 0x19, /* BIU_UPSTREAM_CYCLE */
253 0x1a, /* BIU_DOWNSTREAM_CYCLE */
254 0x22, /* L2C_AGENT_LD */
255 0x23, /* L2C_AGENT_LD_MISS */
256 0x24, /* L2C_AGENT_ST */
257 0x25, /* L2C_AGENT_ST_MISS */
258 0x78, /* INST_A32 */
259 0x79, /* INST_THUMB */
260 0x7a, /* INST_A64 */
261 0x7b, /* INST_BRANCH */
262 0xb4, /* SYNC_DC_LOAD_MISS */
263 0xb5, /* SYNC_DC_STORE_MISS */
264 0xb6, /* SYNC_DTLB_MISS */
265 0xb9, /* SYNC_ST_HIT_YNGR_LD */
266 0xc0, /* SYNC_BR_ANY_MISP */
267 0xce, /* FED_IC_MISS_DEM */
268 0xcf, /* FED_ITLB_MISS */
269
270 #elif defined(APPLETYPHOON)
271 0x02, /* CORE_CYCLE */
272 0x13, /* BIU_UPSTREAM_CYCLE */
273 0x14, /* BIU_DOWNSTREAM_CYCLE */
274 0x1a, /* L2C_AGENT_LD */
275 0x1b, /* L2C_AGENT_LD_MISS */
276 0x1c, /* L2C_AGENT_ST */
277 0x1d, /* L2C_AGENT_ST_MISS */
278 0x8a, /* INST_A32 */
279 0x8b, /* INST_THUMB */
280 0x8c, /* INST_A64 */
281 0x8d, /* INST_BRANCH */
282 0xbf, /* SYNC_DC_LOAD_MISS */
283 0xc0, /* SYNC_DC_STORE_MISS */
284 0xc1, /* SYNC_DTLB_MISS */
285 0xc4, /* SYNC_ST_HIT_YNGR_LD */
286 0xcb, /* SYNC_BR_ANY_MISP */
287 0xd3, /* FED_IC_MISS_DEM */
288 0xd4, /* FED_ITLB_MISS */
289
290 #elif defined(APPLETWISTER) || defined(APPLEHURRICANE)
291 0x02, /* CORE_CYCLE */
292 0x1a, /* L2C_AGENT_LD */
293 0x1b, /* L2C_AGENT_LD_MISS */
294 0x1c, /* L2C_AGENT_ST */
295 0x1d, /* L2C_AGENT_ST_MISS */
296 0x8a, /* INST_A32 */
297 0x8b, /* INST_THUMB */
298 0x8c, /* INST_A64 */
299 0x8d, /* INST_BRANCH */
300 0xbf, /* SYNC_DC_LOAD_MISS */
301 0xc0, /* SYNC_DC_STORE_MISS */
302 0xc1, /* SYNC_DTLB_MISS */
303 0xc4, /* SYNC_ST_HIT_YNGR_LD */
304 0xcb, /* SYNC_BR_ANY_MISP */
305 0xd3, /* FED_IC_MISS_DEM */
306 0xd4, /* FED_ITLB_MISS */
307
308 #elif defined(APPLEMONSOON)
309 0x02, /* CORE_CYCLE */
310 0x8a, /* INST_A32 */
311 0x8b, /* INST_THUMB */
312 0x8c, /* INST_A64 */
313 0x8d, /* INST_BRANCH */
314 0xbf, /* SYNC_DC_LOAD_MISS */
315 0xc0, /* SYNC_DC_STORE_MISS */
316 0xc1, /* SYNC_DTLB_MISS */
317 0xc4, /* SYNC_ST_HIT_YNGR_LD */
318 0xcb, /* SYNC_BR_ANY_MISP */
319 0xd3, /* FED_IC_MISS_DEM */
320 0xd4, /* FED_ITLB_MISS */
321
322 #else
323 /* An unknown CPU gets a trivial { NO_EVENT } whitelist. */
324 #endif
325 };
326 #define WHITELIST_COUNT (sizeof(whitelist)/sizeof(*whitelist))
327
328 static boolean_t
329 config_in_whitelist(kpc_config_t cfg)
330 {
331 unsigned int i;
332
333 for (i = 0; i < WHITELIST_COUNT; i++) {
334 if (cfg == whitelist[i]) {
335 return TRUE;
336 }
337 }
338
339 return FALSE;
340 }
341
342 #ifdef KPC_DEBUG
343 static void dump_regs(void)
344 {
345 uint64_t val;
346 kprintf("PMCR0 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMCR0));
347 kprintf("PMCR1 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMCR1));
348 kprintf("PMCR2 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMCR2));
349 kprintf("PMCR3 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMCR3));
350 kprintf("PMCR4 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMCR4));
351 kprintf("PMESR0 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMESR0));
352 kprintf("PMESR1 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMESR1));
353
354 kprintf("PMC0 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC0));
355 kprintf("PMC1 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC1));
356 kprintf("PMC2 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC2));
357 kprintf("PMC3 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC3));
358 kprintf("PMC4 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC4));
359 kprintf("PMC5 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC5));
360 kprintf("PMC6 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC6));
361 kprintf("PMC7 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC7));
362
363 #if (KPC_ARM64_CONFIGURABLE_COUNT > 6)
364 kprintf("PMC8 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC8));
365 kprintf("PMC9 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC9));
366 #endif
367 }
368 #endif
369
370 static boolean_t
371 enable_counter(uint32_t counter)
372 {
373 int cpuid = cpu_number();
374 uint64_t pmcr0 = 0, intgen_type;
375 boolean_t counter_running, pmi_enabled, intgen_correct, enabled;
376
377 pmcr0 = SREG_READ(SREG_PMCR0) | 0x3 /* leave the fixed counters enabled for monotonic */;
378
379 counter_running = (pmcr0 & PMCR0_PMC_ENABLE_MASK(counter)) != 0;
380 pmi_enabled = (pmcr0 & PMCR0_PMI_ENABLE_MASK(counter)) != 0;
381
382 /* TODO this should use the PMI path rather than AIC for the interrupt
383 * as it is faster
384 */
385 intgen_type = PMCR0_INTGEN_AIC;
386 intgen_correct = (pmcr0 & PMCR0_INTGEN_MASK) == intgen_type;
387
388 enabled = counter_running && pmi_enabled && intgen_correct;
389
390 if (!enabled) {
391 pmcr0 |= PMCR0_PMC_ENABLE_MASK(counter);
392 pmcr0 |= PMCR0_PMI_ENABLE_MASK(counter);
393 pmcr0 &= ~PMCR0_INTGEN_MASK;
394 pmcr0 |= intgen_type;
395
396 SREG_WRITE(SREG_PMCR0, pmcr0);
397 }
398
399 saved_PMCR[cpuid][0] = pmcr0;
400 return enabled;
401 }
402
403 static boolean_t
404 disable_counter(uint32_t counter)
405 {
406 uint64_t pmcr0;
407 boolean_t enabled;
408 int cpuid = cpu_number();
409
410 if (counter < 2) {
411 return true;
412 }
413
414 pmcr0 = SREG_READ(SREG_PMCR0) | 0x3;
415 enabled = (pmcr0 & PMCR0_PMC_ENABLE_MASK(counter)) != 0;
416
417 if (enabled) {
418 pmcr0 &= PMCR0_PMC_DISABLE_MASK(counter);
419 SREG_WRITE(SREG_PMCR0, pmcr0);
420 }
421
422 saved_PMCR[cpuid][0] = pmcr0;
423 return enabled;
424 }
425
426 /*
427 * Enable counter in processor modes determined by configuration word.
428 */
429 static void
430 set_modes(uint32_t counter, kpc_config_t cfgword)
431 {
432 uint64_t bits = 0;
433 int cpuid = cpu_number();
434
435 if (cfgword & CFGWORD_EL0A32EN_MASK) {
436 bits |= PMCR1_EL0_A32_ENABLE_MASK(counter);
437 }
438 if (cfgword & CFGWORD_EL0A64EN_MASK) {
439 bits |= PMCR1_EL0_A64_ENABLE_MASK(counter);
440 }
441 if (cfgword & CFGWORD_EL1EN_MASK) {
442 bits |= PMCR1_EL1_A64_ENABLE_MASK(counter);
443 }
444 #if !NO_MONITOR
445 if (cfgword & CFGWORD_EL3EN_MASK) {
446 bits |= PMCR1_EL3_A64_ENABLE_MASK(counter);
447 }
448 #endif
449
450 /*
451 * Backwards compatibility: Writing a non-zero configuration word with
452 * all zeros in bits 16-19 is interpreted as enabling in all modes.
453 * This matches the behavior when the PMCR1 bits weren't exposed.
454 */
455 if (bits == 0 && cfgword != 0) {
456 bits = PMCR1_EL_ALL_ENABLE_MASK(counter);
457 }
458
459 uint64_t pmcr1 = SREG_READ(SREG_PMCR1);
460 pmcr1 &= PMCR1_EL_ALL_DISABLE_MASK(counter);
461 pmcr1 |= bits;
462 pmcr1 |= 0x30303; /* monotonic compatibility */
463 SREG_WRITE(SREG_PMCR1, pmcr1);
464 saved_PMCR[cpuid][1] = pmcr1;
465 }
466
467 static uint64_t
468 read_counter(uint32_t counter)
469 {
470 switch (counter) {
471 // case 0: return SREG_READ(SREG_PMC0);
472 // case 1: return SREG_READ(SREG_PMC1);
473 case 2: return SREG_READ(SREG_PMC2);
474 case 3: return SREG_READ(SREG_PMC3);
475 case 4: return SREG_READ(SREG_PMC4);
476 case 5: return SREG_READ(SREG_PMC5);
477 case 6: return SREG_READ(SREG_PMC6);
478 case 7: return SREG_READ(SREG_PMC7);
479 #if (KPC_ARM64_CONFIGURABLE_COUNT > 6)
480 case 8: return SREG_READ(SREG_PMC8);
481 case 9: return SREG_READ(SREG_PMC9);
482 #endif
483 default: return 0;
484 }
485 }
486
487 static void
488 write_counter(uint32_t counter, uint64_t value)
489 {
490 switch (counter) {
491 // case 0: SREG_WRITE(SREG_PMC0, value); break;
492 // case 1: SREG_WRITE(SREG_PMC1, value); break;
493 case 2: SREG_WRITE(SREG_PMC2, value); break;
494 case 3: SREG_WRITE(SREG_PMC3, value); break;
495 case 4: SREG_WRITE(SREG_PMC4, value); break;
496 case 5: SREG_WRITE(SREG_PMC5, value); break;
497 case 6: SREG_WRITE(SREG_PMC6, value); break;
498 case 7: SREG_WRITE(SREG_PMC7, value); break;
499 #if (KPC_ARM64_CONFIGURABLE_COUNT > 6)
500 case 8: SREG_WRITE(SREG_PMC8, value); break;
501 case 9: SREG_WRITE(SREG_PMC9, value); break;
502 #endif
503 default: break;
504 }
505 }
506
507 uint32_t
508 kpc_rawpmu_config_count(void)
509 {
510 return RAWPMU_CONFIG_COUNT;
511 }
512
513 int
514 kpc_get_rawpmu_config(kpc_config_t *configv)
515 {
516 configv[0] = SREG_READ(SREG_PMCR2);
517 configv[1] = SREG_READ(SREG_PMCR3);
518 configv[2] = SREG_READ(SREG_PMCR4);
519 configv[3] = SREG_READ(SREG_OPMAT0);
520 configv[4] = SREG_READ(SREG_OPMAT1);
521 configv[5] = SREG_READ(SREG_OPMSK0);
522 configv[6] = SREG_READ(SREG_OPMSK1);
523 #if RAWPMU_CONFIG_COUNT > 7
524 configv[7] = SREG_READ(SREG_PMMMAP);
525 configv[8] = SREG_READ(SREG_PMTRHLD2);
526 configv[9] = SREG_READ(SREG_PMTRHLD4);
527 configv[10] = SREG_READ(SREG_PMTRHLD6);
528 #endif
529 return 0;
530 }
531
532 static int
533 kpc_set_rawpmu_config(kpc_config_t *configv)
534 {
535 SREG_WRITE(SREG_PMCR2, configv[0]);
536 SREG_WRITE(SREG_PMCR3, configv[1]);
537 SREG_WRITE(SREG_PMCR4, configv[2]);
538 SREG_WRITE(SREG_OPMAT0, configv[3]);
539 SREG_WRITE(SREG_OPMAT1, configv[4]);
540 SREG_WRITE(SREG_OPMSK0, configv[5]);
541 SREG_WRITE(SREG_OPMSK1, configv[6]);
542 #if RAWPMU_CONFIG_COUNT > 7
543 SREG_WRITE(SREG_PMMMAP, configv[7]);
544 SREG_WRITE(SREG_PMTRHLD2, configv[8]);
545 SREG_WRITE(SREG_PMTRHLD4, configv[9]);
546 SREG_WRITE(SREG_PMTRHLD6, configv[10]);
547 #endif
548 return 0;
549 }
550
551 static void
552 save_regs(void)
553 {
554 int cpuid = cpu_number();
555
556 __asm__ volatile("dmb ish");
557
558 assert(ml_get_interrupts_enabled() == FALSE);
559
560 /* Save current PMCR0/1 values. PMCR2-4 are in the RAWPMU set. */
561 saved_PMCR[cpuid][0] = SREG_READ(SREG_PMCR0) | 0x3;
562
563 /* Save event selections. */
564 saved_PMESR[cpuid][0] = SREG_READ(SREG_PMESR0);
565 saved_PMESR[cpuid][1] = SREG_READ(SREG_PMESR1);
566
567 kpc_get_rawpmu_config(saved_RAWPMU[cpuid]);
568
569 /* Disable the counters. */
570 // SREG_WRITE(SREG_PMCR0, clear);
571
572 /* Finally, save state for each counter*/
573 for (int i = 2; i < KPC_ARM64_PMC_COUNT; i++) {
574 saved_counter[cpuid][i] = read_counter(i);
575 }
576 }
577
578 static void
579 restore_regs(void)
580 {
581 int cpuid = cpu_number();
582
583 /* Restore PMESR values. */
584 SREG_WRITE(SREG_PMESR0, saved_PMESR[cpuid][0]);
585 SREG_WRITE(SREG_PMESR1, saved_PMESR[cpuid][1]);
586
587 kpc_set_rawpmu_config(saved_RAWPMU[cpuid]);
588
589 /* Restore counter values */
590 for (int i = 2; i < KPC_ARM64_PMC_COUNT; i++) {
591 write_counter(i, saved_counter[cpuid][i]);
592 }
593
594 /* Restore PMCR0/1 values (with PMCR0 last to enable). */
595 SREG_WRITE(SREG_PMCR1, saved_PMCR[cpuid][1] | 0x30303);
596 SREG_WRITE(SREG_PMCR0, saved_PMCR[cpuid][0] | 0x3);
597 }
598
599 static uint64_t
600 get_counter_config(uint32_t counter)
601 {
602 uint64_t pmesr;
603
604 switch (counter) {
605 case 2: /* FALLTHROUGH */
606 case 3: /* FALLTHROUGH */
607 case 4: /* FALLTHROUGH */
608 case 5:
609 pmesr = PMESR_EVT_DECODE(SREG_READ(SREG_PMESR0), counter, 2);
610 break;
611 case 6: /* FALLTHROUGH */
612 case 7:
613 #if (KPC_ARM64_CONFIGURABLE_COUNT > 6)
614 /* FALLTHROUGH */
615 case 8: /* FALLTHROUGH */
616 case 9:
617 #endif
618 pmesr = PMESR_EVT_DECODE(SREG_READ(SREG_PMESR1), counter, 6);
619 break;
620 default:
621 pmesr = 0;
622 break;
623 }
624
625 kpc_config_t config = pmesr;
626
627 uint64_t pmcr1 = SREG_READ(SREG_PMCR1);
628
629 if (pmcr1 & PMCR1_EL0_A32_ENABLE_MASK(counter)) {
630 config |= CFGWORD_EL0A32EN_MASK;
631 }
632 if (pmcr1 & PMCR1_EL0_A64_ENABLE_MASK(counter)) {
633 config |= CFGWORD_EL0A64EN_MASK;
634 }
635 if (pmcr1 & PMCR1_EL1_A64_ENABLE_MASK(counter)) {
636 config |= CFGWORD_EL1EN_MASK;
637 #if NO_MONITOR
638 config |= CFGWORD_EL3EN_MASK;
639 #endif
640 }
641 #if !NO_MONITOR
642 if (pmcr1 & PMCR1_EL3_A64_ENABLE_MASK(counter)) {
643 config |= CFGWORD_EL3EN_MASK;
644 }
645 #endif
646
647 return config;
648 }
649
650 static void
651 set_counter_config(uint32_t counter, uint64_t config)
652 {
653 int cpuid = cpu_number();
654 uint64_t pmesr = 0;
655
656 switch (counter) {
657 case 2: /* FALLTHROUGH */
658 case 3: /* FALLTHROUGH */
659 case 4: /* FALLTHROUGH */
660 case 5:
661 pmesr = SREG_READ(SREG_PMESR0);
662 pmesr &= PMESR_EVT_CLEAR(counter, 2);
663 pmesr |= PMESR_EVT_ENCODE(config, counter, 2);
664 SREG_WRITE(SREG_PMESR0, pmesr);
665 saved_PMESR[cpuid][0] = pmesr;
666 break;
667
668 case 6: /* FALLTHROUGH */
669 case 7:
670 #if KPC_ARM64_CONFIGURABLE_COUNT > 6
671 /* FALLTHROUGH */
672 case 8: /* FALLTHROUGH */
673 case 9:
674 #endif
675 pmesr = SREG_READ(SREG_PMESR1);
676 pmesr &= PMESR_EVT_CLEAR(counter, 6);
677 pmesr |= PMESR_EVT_ENCODE(config, counter, 6);
678 SREG_WRITE(SREG_PMESR1, pmesr);
679 saved_PMESR[cpuid][1] = pmesr;
680 break;
681 default:
682 break;
683 }
684
685 set_modes(counter, config);
686 }
687
688 /* internal functions */
689
690 void
691 kpc_arch_init(void)
692 {
693 }
694
695 boolean_t
696 kpc_is_running_fixed(void)
697 {
698 return (kpc_running_classes & KPC_CLASS_FIXED_MASK) == KPC_CLASS_FIXED_MASK;
699 }
700
701 boolean_t
702 kpc_is_running_configurable(uint64_t pmc_mask)
703 {
704 assert(kpc_popcount(pmc_mask) <= kpc_configurable_count());
705 return ((kpc_running_classes & KPC_CLASS_CONFIGURABLE_MASK) == KPC_CLASS_CONFIGURABLE_MASK) &&
706 ((kpc_running_cfg_pmc_mask & pmc_mask) == pmc_mask);
707 }
708
709 uint32_t
710 kpc_fixed_count(void)
711 {
712 return KPC_ARM64_FIXED_COUNT;
713 }
714
715 uint32_t
716 kpc_configurable_count(void)
717 {
718 return KPC_ARM64_CONFIGURABLE_COUNT;
719 }
720
721 uint32_t
722 kpc_fixed_config_count(void)
723 {
724 return 0;
725 }
726
727 uint32_t
728 kpc_configurable_config_count(uint64_t pmc_mask)
729 {
730 assert(kpc_popcount(pmc_mask) <= kpc_configurable_count());
731 return kpc_popcount(pmc_mask);
732 }
733
734 int
735 kpc_get_fixed_config(kpc_config_t *configv __unused)
736 {
737 return 0;
738 }
739
740 uint64_t
741 kpc_fixed_max(void)
742 {
743 return (1ULL << KPC_ARM64_COUNTER_WIDTH) - 1;
744 }
745
746 uint64_t
747 kpc_configurable_max(void)
748 {
749 return (1ULL << KPC_ARM64_COUNTER_WIDTH) - 1;
750 }
751
752 static void
753 set_running_configurable(uint64_t target_mask, uint64_t state_mask)
754 {
755 uint32_t cfg_count = kpc_configurable_count(), offset = kpc_fixed_count();
756 boolean_t enabled;
757
758 enabled = ml_set_interrupts_enabled(FALSE);
759
760 for (uint32_t i = 0; i < cfg_count; ++i) {
761 if (((1ULL << i) & target_mask) == 0)
762 continue;
763 assert(kpc_controls_counter(offset + i));
764
765 if ((1ULL << i) & state_mask) {
766 enable_counter(offset + i);
767 } else {
768 disable_counter(offset + i);
769 }
770 }
771
772 ml_set_interrupts_enabled(enabled);
773 }
774
775 static uint32_t kpc_xcall_sync;
776 static void
777 kpc_set_running_xcall( void *vstate )
778 {
779 struct kpc_running_remote *mp_config = (struct kpc_running_remote*) vstate;
780 assert(mp_config);
781
782 set_running_configurable(mp_config->cfg_target_mask,
783 mp_config->cfg_state_mask);
784
785 if (hw_atomic_sub(&kpc_xcall_sync, 1) == 0)
786 thread_wakeup((event_t) &kpc_xcall_sync);
787 }
788
789 static uint32_t kpc_xread_sync;
790 static void
791 kpc_get_curcpu_counters_xcall(void *args)
792 {
793 struct kpc_get_counters_remote *handler = args;
794
795 assert(handler != NULL);
796 assert(handler->buf != NULL);
797
798 int offset = cpu_number() * handler->buf_stride;
799 int r = kpc_get_curcpu_counters(handler->classes, NULL, &handler->buf[offset]);
800
801 /* number of counters added by this CPU, needs to be atomic */
802 hw_atomic_add(&(handler->nb_counters), r);
803
804 if (hw_atomic_sub(&kpc_xread_sync, 1) == 0) {
805 thread_wakeup((event_t) &kpc_xread_sync);
806 }
807 }
808
809 int
810 kpc_get_all_cpus_counters(uint32_t classes, int *curcpu, uint64_t *buf)
811 {
812 assert(buf != NULL);
813
814 int enabled = ml_set_interrupts_enabled(FALSE);
815
816 /* grab counters and CPU number as close as possible */
817 if (curcpu) {
818 *curcpu = current_processor()->cpu_id;
819 }
820
821 struct kpc_get_counters_remote hdl = {
822 .classes = classes,
823 .nb_counters = 0,
824 .buf = buf,
825 .buf_stride = kpc_get_counter_count(classes)
826 };
827
828 cpu_broadcast_xcall(&kpc_xread_sync, TRUE, kpc_get_curcpu_counters_xcall, &hdl);
829 int offset = hdl.nb_counters;
830
831 (void)ml_set_interrupts_enabled(enabled);
832
833 return offset;
834 }
835
836 int
837 kpc_get_fixed_counters(uint64_t *counterv)
838 {
839 #if MONOTONIC
840 mt_fixed_counts(counterv);
841 return 0;
842 #else /* MONOTONIC */
843 #pragma unused(counterv)
844 return ENOTSUP;
845 #endif /* !MONOTONIC */
846 }
847
848 int
849 kpc_get_configurable_counters(uint64_t *counterv, uint64_t pmc_mask)
850 {
851 uint32_t cfg_count = kpc_configurable_count(), offset = kpc_fixed_count();
852 uint64_t ctr = 0ULL;
853
854 assert(counterv);
855
856 for (uint32_t i = 0; i < cfg_count; ++i) {
857 if (((1ULL << i) & pmc_mask) == 0)
858 continue;
859 ctr = read_counter(i + offset);
860
861 if (ctr & KPC_ARM64_COUNTER_OVF_MASK) {
862 ctr = CONFIGURABLE_SHADOW(i) +
863 (kpc_configurable_max() - CONFIGURABLE_RELOAD(i) + 1 /* Wrap */) +
864 (ctr & KPC_ARM64_COUNTER_MASK);
865 } else {
866 ctr = CONFIGURABLE_SHADOW(i) +
867 (ctr - CONFIGURABLE_RELOAD(i));
868 }
869
870 *counterv++ = ctr;
871 }
872
873 return 0;
874 }
875
876 int
877 kpc_get_configurable_config(kpc_config_t *configv, uint64_t pmc_mask)
878 {
879 uint32_t cfg_count = kpc_configurable_count(), offset = kpc_fixed_count();
880
881 assert(configv);
882
883 for (uint32_t i = 0; i < cfg_count; ++i)
884 if ((1ULL << i) & pmc_mask)
885 *configv++ = get_counter_config(i + offset);
886 return 0;
887 }
888
889 static int
890 kpc_set_configurable_config(kpc_config_t *configv, uint64_t pmc_mask)
891 {
892 uint32_t cfg_count = kpc_configurable_count(), offset = kpc_fixed_count();
893 boolean_t enabled;
894
895 assert(configv);
896
897 enabled = ml_set_interrupts_enabled(FALSE);
898
899 for (uint32_t i = 0; i < cfg_count; ++i) {
900 if (((1ULL << i) & pmc_mask) == 0)
901 continue;
902 assert(kpc_controls_counter(i + offset));
903
904 set_counter_config(i + offset, *configv++);
905 }
906
907 ml_set_interrupts_enabled(enabled);
908
909 return 0;
910 }
911
912 static uint32_t kpc_config_sync;
913 static void
914 kpc_set_config_xcall(void *vmp_config)
915 {
916 struct kpc_config_remote *mp_config = vmp_config;
917 kpc_config_t *new_config = NULL;
918 uint32_t classes = 0ULL;
919
920 assert(mp_config);
921 assert(mp_config->configv);
922 classes = mp_config->classes;
923 new_config = mp_config->configv;
924
925 if (classes & KPC_CLASS_CONFIGURABLE_MASK) {
926 kpc_set_configurable_config(new_config, mp_config->pmc_mask);
927 new_config += kpc_popcount(mp_config->pmc_mask);
928 }
929
930 if (classes & KPC_CLASS_RAWPMU_MASK) {
931 kpc_set_rawpmu_config(new_config);
932 new_config += RAWPMU_CONFIG_COUNT;
933 }
934
935 if (hw_atomic_sub(&kpc_config_sync, 1) == 0)
936 thread_wakeup((event_t) &kpc_config_sync);
937 }
938
939 static uint64_t
940 kpc_reload_counter(uint32_t ctr)
941 {
942 assert(ctr < (kpc_configurable_count() + kpc_fixed_count()));
943
944 /* don't reload counters reserved for power management */
945 if (!kpc_controls_counter(ctr))
946 return 0ULL;
947
948 uint64_t old = read_counter(ctr);
949 write_counter(ctr, FIXED_RELOAD(ctr));
950 return old & KPC_ARM64_COUNTER_MASK;
951 }
952
953 static uint32_t kpc_reload_sync;
954 static void
955 kpc_set_reload_xcall(void *vmp_config)
956 {
957 struct kpc_config_remote *mp_config = vmp_config;
958 uint32_t classes = 0, count = 0, offset = kpc_fixed_count();
959 uint64_t *new_period = NULL, max = kpc_configurable_max();
960 boolean_t enabled;
961
962 assert(mp_config);
963 assert(mp_config->configv);
964 classes = mp_config->classes;
965 new_period = mp_config->configv;
966
967 enabled = ml_set_interrupts_enabled(FALSE);
968
969 if (classes & KPC_CLASS_CONFIGURABLE_MASK) {
970 /*
971 * Update _all_ shadow counters, this cannot be done for only
972 * selected PMCs. Otherwise, we would corrupt the configurable
973 * shadow buffer since the PMCs are muxed according to the pmc
974 * mask.
975 */
976 uint64_t all_cfg_mask = (1ULL << kpc_configurable_count()) - 1;
977 kpc_get_configurable_counters(&CONFIGURABLE_SHADOW(0), all_cfg_mask);
978
979 /* set the new period */
980 count = kpc_configurable_count();
981 for (uint32_t i = 0; i < count; ++i) {
982 /* ignore the counter */
983 if (((1ULL << i) & mp_config->pmc_mask) == 0)
984 continue;
985 if (*new_period == 0)
986 *new_period = kpc_configurable_max();
987 CONFIGURABLE_RELOAD(i) = max - *new_period;
988 /* reload the counter */
989 kpc_reload_counter(offset + i);
990 /* next period value */
991 new_period++;
992 }
993 }
994
995 ml_set_interrupts_enabled(enabled);
996
997 if (hw_atomic_sub(&kpc_reload_sync, 1) == 0)
998 thread_wakeup((event_t) &kpc_reload_sync);
999 }
1000
1001 void
1002 kpc_pmi_handler(unsigned int ctr)
1003 {
1004 uint64_t extra = kpc_reload_counter(ctr);
1005
1006 FIXED_SHADOW(ctr) += (kpc_fixed_max() - FIXED_RELOAD(ctr) + 1 /* Wrap */) + extra;
1007
1008 if (FIXED_ACTIONID(ctr)) {
1009 kpc_sample_kperf(FIXED_ACTIONID(ctr));
1010 }
1011 }
1012
1013 uint32_t
1014 kpc_get_classes(void)
1015 {
1016 return KPC_CLASS_FIXED_MASK | KPC_CLASS_CONFIGURABLE_MASK | KPC_CLASS_RAWPMU_MASK;
1017 }
1018
1019 int
1020 kpc_set_running_arch(struct kpc_running_remote *mp_config)
1021 {
1022 assert(mp_config != NULL);
1023
1024 /* dispatch to all CPUs */
1025 cpu_broadcast_xcall(&kpc_xcall_sync, TRUE, kpc_set_running_xcall, mp_config);
1026
1027 kpc_running_cfg_pmc_mask = mp_config->cfg_state_mask;
1028 kpc_running_classes = mp_config->classes;
1029 kpc_configured = 1;
1030
1031 return 0;
1032 }
1033
1034 int
1035 kpc_set_period_arch(struct kpc_config_remote *mp_config)
1036 {
1037 assert(mp_config);
1038
1039 /* dispatch to all CPUs */
1040 cpu_broadcast_xcall(&kpc_reload_sync, TRUE, kpc_set_reload_xcall, mp_config);
1041
1042 kpc_configured = 1;
1043
1044 return 0;
1045 }
1046
1047 int
1048 kpc_set_config_arch(struct kpc_config_remote *mp_config)
1049 {
1050 uint32_t count = kpc_popcount(mp_config->pmc_mask);
1051
1052 assert(mp_config);
1053 assert(mp_config->configv);
1054
1055 /* check config against whitelist for external devs */
1056 for (uint32_t i = 0; i < count; ++i) {
1057 if (!whitelist_disabled && !config_in_whitelist(mp_config->configv[i])) {
1058 return EPERM;
1059 }
1060 }
1061
1062 /* dispatch to all CPUs */
1063 cpu_broadcast_xcall(&kpc_config_sync, TRUE, kpc_set_config_xcall, mp_config);
1064
1065 kpc_configured = 1;
1066
1067 return 0;
1068 }
1069
1070 void
1071 kpc_idle(void)
1072 {
1073 if (kpc_configured) {
1074 save_regs();
1075 }
1076 }
1077
1078 void
1079 kpc_idle_exit(void)
1080 {
1081 if (kpc_configured) {
1082 restore_regs();
1083 }
1084 }
1085
1086 int
1087 kpc_set_sw_inc( uint32_t mask __unused )
1088 {
1089 return ENOTSUP;
1090 }
1091
1092 int
1093 kpc_disable_whitelist( int val )
1094 {
1095 whitelist_disabled = val;
1096 return 0;
1097 }
1098
1099 int
1100 kpc_get_whitelist_disabled( void )
1101 {
1102 return whitelist_disabled;
1103 }
1104
1105 int
1106 kpc_get_pmu_version(void)
1107 {
1108 return KPC_PMU_ARM_APPLE;
1109 }
mirror of XNU